def main(): database = RetrievalModel(args=args) logging.info("db param size = %fMB", utils.count_parameters_in_MB(database)) logging.info(stat(database,(3,224,224))) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, args.code_size, args.layers, args.auxiliary, genotype) model.drop_path_prob = args.drop_path_prob logging.info("query param size = %fMB", utils.count_parameters_in_MB(model)) logging.info(stat(model,(3,224,224)))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) args.seed = np.random.randint(0, 100) logging.info("----------seed = {}-----------".format(args.seed)) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) logging.info("Arc = {}".format(genotype)) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs),eta_min=args.learning_rate_min) Best_valid_acc = 0.0 for epoch in range(args.epochs): logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) scheduler.step() if valid_acc >= Best_valid_acc: Best_valid_acc = valid_acc logging.info('Best_valid_acc %f', Best_valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True cudnn.enabled = True logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # equal to: genotype = genotypes.DARTS_v2 genotype = eval("genotypes.%s" % args.arch) print('Load genotype:', genotype) model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() utils.load(model, args.exp_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader( test_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True torch.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.wd) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batchsz, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc: %f', valid_acc) train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc: %f', train_acc) utils.save(model, os.path.join(args.save, 'trained.pt')) print('saved to: trained.pt')
def main(): torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled = True logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) model = Network(args.init_ch, 10, args.layers, True, genotype).cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) checkpoint = torch.load(args.checkpoint + '/top1.pt') model.load_state_dict(checkpoint['model_state_dict']) criterion = nn.CrossEntropyLoss().cuda() CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124] CIFAR_STD = [0.24703233, 0.24348505, 0.26158768] valid_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(CIFAR_MEAN, CIFAR_STD), ]) valid_queue = torch.utils.data.DataLoader(dset.CIFAR10( root=args.data, train=False, transform=valid_transform), batch_size=args.batch_size, shuffle=True, num_workers=2, pin_memory=True) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc: %f', valid_acc)
def initialize_model(self): """ Initialize model, may change across different model. :return: """ args = self.args model = self.model_fn(args) if args.gpus > 0: if self.args.gpus == 1: model = model.cuda() self.parallel_model = model else: self.model = model self.parallel_model = nn.DataParallel(self.model).cuda() # IPython.embed(header='checking replicas and others.') else: self.parallel_model = model # rewrite the pointer model = self.parallel_model logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) # scheduler as Cosine. scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) return model, optimizer, scheduler
def main(): # logging.info('no gpu device available') # sys.exit(1) np.random.seed(args.seed) torch.manual_seed(args.seed) if torch.cuda.is_available(): torch.cuda.set_device(args.gpu) cudnn.benchmark = True cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) utils.load(model, args.model_path, strict=False) model = model.to_device() # to(torch._default_device) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.to_device() # to(torch._default_device) _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader( test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def generate_pop(DNA_SIZE=args.DNA_SIZE, POP_SIZE=args.POP_SIZE): logging.info('\n-----generating population------') block_pop = [] size_pop = [] for idx in range(POP_SIZE): param = 10.0 while param >= 8.0: # disgard network that larger than 3M block_cfg, size_cfg, ds_cfg = make_cfg(DNA_SIZE) model = nas.initial_network(block_cfg, size_cfg, ds_cfg) param = utils.count_parameters_in_MB(nas.net) logging.info('param size = {0:.2f}MB'.format(param)) block_pop.append(block_cfg) size_pop.append(size_cfg) logging.info('pop idx {} generated\n'.format(idx)) logging.info('block_pop shape: POP {}, DNA {}'.format( len(block_pop), len(block_pop[0]))) logging.info(' size_pop shape: POP {}, DNA {}'.format( len(size_pop), len(size_pop[0]))) logging.info('ds_cfg {}'.format(ds_cfg)) logging.info('Population block_pop {}'.format(block_pop)) logging.info('Population size_pop {}\n'.format(size_pop)) pop = {'block_pop': block_pop, 'size_pop': size_pop, 'ds_cfg': ds_cfg} return pop
def main(): if not torch.cuda.is_available(): sys.exit(1) ## step 1 construct the selected network genotype = eval("genotypes.%s" % args.selected_arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) ## step 2 load pretrained model parameter if args.cifar100: model = torch.nn.DataParallel(model) model = model.cuda() model.load_state_dict(torch.load(args.model_path)['net']) else: utils.load(model, args.model_path) model = torch.nn.DataParallel(model) model = model.cuda() model.module.drop_path_prob = 0 model.drop_path_prob = 0 print("param size = %fMB" % utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() ## step 3 load test data valid_queue = load_data_cifar(args) ## step 4. inference on test data valid_acc, valid_obj = infer(valid_queue, model, criterion) print('-----------------------------------------------') print('Average Valid_acc: %f ' % valid_acc) print('-----------------------------------------------')
def build_cifar10(model_state_dict=None, optimizer_state_dict=None, **kwargs): epoch = kwargs.pop('epoch') ratio = kwargs.pop('ratio') train_ds, valid_ds = utils.load_cifar10(args.child_batch_size, args.child_cutout_size) # subset random sample model = NASWSNetworkCIFAR(10, args.child_layers, args.child_nodes, args.child_channels, args.child_keep_prob, args.child_drop_path_keep_prob, args.child_use_aux_head, args.steps) model = model.cuda() train_criterion = nn.CrossEntropyLoss().cuda() eval_criterion = nn.CrossEntropyLoss().cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD( model.parameters(), args.child_lr_max, momentum=0.9, weight_decay=args.child_l2_reg, ) if model_state_dict is not None: model.load_state_dict(model_state_dict) if optimizer_state_dict is not None: optimizer.load_state_dict(optimizer_state_dict) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.child_epochs, args.child_lr_min, epoch) return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) #损失函数 criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) #优化器 optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) torch.cuda.set_device(args.gpu) cudnn.enabled = True logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=False, num_workers=2) model.drop_path_prob = 0.0 test_acc, test_obj = infer(test_queue, model, criterion) logging.info('Test_acc %f', test_acc)
def build_cifar100(model_state_dict=None, optimizer_state_dict=None, args=None, **kwargs): epoch = kwargs.pop('epoch') ratio = kwargs.pop('ratio') train_transform, valid_transform = utils._data_transforms_cifar10( args.cutout_size) train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=valid_transform) num_train = len(train_data) assert num_train == len(valid_data) indices = list(range(num_train)) split = int(np.floor(ratio * num_train)) np.random.shuffle(indices) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.child_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=16) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.child_eval_batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler( indices[split:num_train]), pin_memory=True, num_workers=16) model = NASWSNetworkCIFAR(100, args.child_layers, args.child_nodes, args.child_channels, args.child_keep_prob, args.child_drop_path_keep_prob, args.child_use_aux_head, args.steps) model = model.cuda() train_criterion = nn.CrossEntropyLoss().cuda() eval_criterion = nn.CrossEntropyLoss().cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD( model.parameters(), args.child_lr_max, momentum=0.9, weight_decay=args.child_l2_reg, ) if model_state_dict is not None: model.load_state_dict(model_state_dict) if optimizer_state_dict is not None: optimizer.load_state_dict(optimizer_state_dict) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, args.child_epochs, args.child_lr_min, epoch) return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) in_channels, num_classes, dataset_in_torch, stride_for_aux = utils.dataset_fields( args, train=False) # new genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, in_channels, stride_for_aux, num_classes, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) train_data, valid_data = utils.dataset_split_and_transform( dataset_in_torch, args, train=False) # new train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() number_of_classes = class_dict[args.dataset] in_channels = inp_channel_dict[args.dataset] model = Network(args.init_channels, number_of_classes, args.layers, criterion, in_channels) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) # Get transforms to apply on data train_transform, valid_transform = utils.get_data_transforms(args) # Get the training queue train_queue, valid_queue = get_training_queues(args, train_transform) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) architect = Architect(model, args) for epoch in range(args.epochs): scheduler.step() lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) genotype = model.genotype() logging.info('genotype = %s', genotype) print(F.softmax(model.alphas_normal, dim=-1)) print(F.softmax(model.alphas_reduce, dim=-1)) # training train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr) logging.info('train_acc %f', train_acc) # validation valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %s' % args.gpu) logging.info('small regime or regular regime = %s' % args.regime) logging.info("args = %s", args) # read graph info if args.regime: graph_info_dict3 = read_graph_info(args.graph_txt + "3.txt") graph_info_dict4 = read_graph_info(args.graph_txt + "4.txt") graph_info_dict5 = read_graph_info(args.graph_txt + "5.txt") graph_info_dicts = [ graph_info_dict3, graph_info_dict4, graph_info_dict5 ] else: graph_info_dict2 = read_graph_info(args.graph_txt + "2.txt") graph_info_dict3 = read_graph_info(args.graph_txt + "3.txt") graph_info_dict4 = read_graph_info(args.graph_txt + "4.txt") graph_info_dict5 = read_graph_info(args.graph_txt + "5.txt") graph_info_dicts = [ graph_info_dict2, graph_info_dict3, graph_info_dict4, graph_info_dict5 ] criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() model = RandomlyWiredNN(args.base_channels, NUM_CLASSES, args.output_channels, args.regime, graph_info_dicts) # When training with multi GPUs, first execute nn.DataParallel() and then load model in eval stage if args.parallel: model = nn.DataParallel(model).cuda() print("multi GPUs") else: model = model.cuda() print("single GPU") utils.load_model(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) logging.info("FLOPs = %fGB", utils.model_param_flops_in_MB(model, multiply_adds=False)) _, test_queue = preprocess_imagenet(args) valid_acc, valid_obj, valid_speed = infer(test_queue, model, criterion) logging.info('valid_acc %f', valid_acc) logging.info('valid_obj %f', valid_obj) logging.info('valid_speed_per_image %f', valid_speed)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) gpus = [int(i) for i in args.gpu.split(',')] # torch.cuda.set_device(gpus) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %s' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = torch.nn.DataParallel(model) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() test_transform = transforms.Compose([ ToTensor(), ]) DATASET_INDEX = args.DATASET_INDEX STEGANOGRAPHY = args.STEGANOGRAPHY EMBEDDING_RATE = args.EMBEDDING_RATE BOSSBASE_COVER_DIR = args.BOSSBASE_COVER_DIR BOSSBASE_STEGO_DIR = args.BOSSBASE_STEGO_DIR BOWS_COVER_DIR = args.BOWS2_COVER_DIR BOWS_STEGO_DIR = args.BOWS2_STEGO_DIR TEST_INDEX_PATH = 'index_list{}/bossbase_test_index.npy'.format( DATASET_INDEX) test_data = MyDataset(TEST_INDEX_PATH, BOSSBASE_COVER_DIR, BOSSBASE_STEGO_DIR, BOWS_COVER_DIR, BOWS_STEGO_DIR, test_transform) test_queue = DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) print(test_acc) logging.info('test_acc %f', test_acc)
def evaluate(individual, num_classes, num_epochs, batch_size, learning_rate): train_transform, test_transform = utils._data_transforms_cifar10() train_dataset = torchvision.datasets.CIFAR10(root='../../data', train=True, transform=train_transform) test_dataset = torchvision.datasets.CIFAR10(root='../../data', train=False, transform=test_transform) train_loader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=batch_size, shuffle=True, # pin_memory=True, # num_workers=2 ) test_loader = torch.utils.data.DataLoader( dataset=test_dataset, batch_size=batch_size, shuffle=False, # pin_memory=True, # num_workers=2 ) structure = Network(individual.structure, [(3, 32), (32, 128), (128, 128)], num_classes, (32, 32)).to(device) individual.size = utils.count_parameters_in_MB(structure) parameters = filter(lambda p: p.requires_grad, structure.parameters()) cudnn.enabled = True cudnn.benchmark = True criterion = torch.nn.CrossEntropyLoss().to(device) optimizer = torch.optim.SGD(parameters, lr=learning_rate, momentum=0.9, weight_decay=3e-4) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs, eta_min=0.0) best_acc = 0 for epoch in range(num_epochs): print('epoch[{}/{}]:'.format(epoch + 1, num_epochs)) train(train_loader, structure, criterion, optimizer) scheduler.step() valid_acc = test(test_loader, structure, criterion) print() if valid_acc > best_acc: best_acc = valid_acc individual.accuracy = best_acc
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) torch.cuda.set_device(args.gpu) cudnn.enabled = True logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) if args.dataset in LARGE_DATASETS: model = NetworkLarge(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) else: model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils.data_transforms(args.dataset, args.cutout, args.cutout_length) if args.dataset == "CIFAR100": test_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=test_transform) elif args.dataset == "CIFAR10": test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) elif args.dataset == "sport8": dset_cls = dset.ImageFolder val_path = '%s/Sport8/test' % args.data test_data = dset_cls(root=val_path, transform=test_transform) elif args.dataset == "mit67": dset_cls = dset.ImageFolder val_path = '%s/MIT67/test' % args.data test_data = dset_cls(root=val_path, transform=test_transform) elif args.dataset == "flowers102": dset_cls = dset.ImageFolder val_path = '%s/flowers102/test' % args.tmp_data_dir test_data = dset_cls(root=val_path, transform=test_transform) test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=False, num_workers=2) model.drop_path_prob = 0.0 test_acc, test_obj = infer(test_queue, model, criterion) logging.info('Test_acc %f', test_acc)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) args.seed += 1 logging.info("args = %s", args) arch = utils.decode_arch(args.arch) model = Network(arch, CIFAR_CLASSES) model = model.cuda() params = utils.count_parameters_in_MB(model) logging.info("param size = %fMB", params / 1e6) flops = utils.count_FLOPs(model) logging.info("FLOPs = %d", flops) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10() train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=True) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=False) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) best_acc = 0 for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) if valid_acc > best_acc: best_acc = valid_acc logging.info('valid_acc %f', valid_acc) return params, flops, best_acc
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) in_channels = 3 num_classes = 10 stride_for_aux = 3 genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, in_channels, stride_for_aux, num_classes, args.layers, args.auxiliary, genotype) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() _, test_transform = utils._data_transforms_cifar10(args) #test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform) #version = 'v6' #images, labels = utils_test.load_new_test_data(version) #num_images = images.shape[0] data_10 = np.load( "/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_data.npy") labels_10 = np.load( '/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_labels.npy') # tensor_x = torch.stack([torch.Tensor(i) for i in data_10]) # transform to torch tensors tensor_y = torch.stack([torch.Tensor(i) for i in labels_10]) test_data = utils_d.TensorDataset(tensor_x, tensor_y) # create your datset test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion) logging.info('test_acc %f', test_acc)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype,output_height=args.img_cropped_height,output_width=args.img_cropped_width) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.MSELoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_trainsforms_denosining_dataset(args) train_data = DENOISE_DATASET(root=args.data,train_folder=args.train_data,label_folder=args.label_data,train=True, transform=train_transform,target_transform=train_transform ) #valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) num_train = len(train_data) indices = list(range(num_train)) split = int(np.floor(args.train_portion * num_train)) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_obj %f', train_obj) valid_obj = infer(valid_queue, model, criterion) logging.info('valid_obj %f', valid_obj) utils.save(model, os.path.join(args.save, './weights.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.init_channels, CLASSES, args.layers, args.auxiliary, genotype) if args.parallel: model = nn.DataParallel(model).cuda() else: model = model.cuda() model.load_state_dict( torch.load(args.model_path, map_location='cuda:0')['state_dict']) print("param size = {:.1f}MB".format( floor(utils.count_parameters_in_MB(model), 1))) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() validdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) valid_data = dset.ImageFolder( validdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4) model.drop_path_prob = args.drop_path_prob input = torch.randn(1, 3, 224, 224) input = input.cuda() valid_acc_top1, valid_acc_top5, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc_top1 %f', valid_acc_top1) logging.info('valid_acc_top5 %f', valid_acc_top5)
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) genotype = eval("genotypes.%s" % args.arch) model = Network(args.rnn_cell, args.init_channels, CIFAR_CLASSES, args.layers, args.nhid, genotype, ds="svhn") model = model.cuda() print("params:", print_param_size(model)) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_svhn_train(args) train_d = dset.SVHN(root=args.data, split="train", download=True, transform=train_transform) extra_d = dset.SVHN(root=args.data, split="extra", download=True, transform=train_transform) train_data = torch.utils.data.ConcatDataset([train_d, extra_d]) valid_data = dset.SVHN(root=args.data, split="test", download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=4) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=4) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, int(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights_train.pt'))
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled=True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # 得到train_search里学好的normal cell 和reduction cell,genotypes.DARTS就是选的学好的DARTS_V2 genotype = eval("genotypes.%s" % args.arch)#DARTS_V2 = Genotype(normal=[('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('skip_connect', 0), ('dil_conv_3x3', 2)], normal_concat=[2, 3, 4, 5], reduce=[('max_pool_3x3', 0), ('max_pool_3x3', 1), ('skip_connect', 2), ('max_pool_3x3', 1), ('max_pool_3x3', 0), ('skip_connect', 2), ('skip_connect', 2), ('max_pool_3x3', 1)], reduce_concat=[2, 3, 4, 5]) # 这里的Network用的是model.py的NetworkCIFAR model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype) model = model.cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() criterion = criterion.cuda() optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay ) train_transform, valid_transform = utils._data_transforms_cifar10(args) train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2) valid_queue = torch.utils.data.DataLoader( valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs)) for epoch in range(args.epochs): scheduler.step() logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) model.drop_path_prob = args.drop_path_prob * epoch / args.epochs train_acc, train_obj = train(train_queue, model, criterion, optimizer) logging.info('train_acc %f', train_acc) valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('valid_acc %f', valid_acc) utils.save(model, os.path.join(args.save, 'weights.pt'))
def build_cifar100(model_state_dict, optimizer_state_dict, **kwargs): epoch = kwargs.pop('epoch') train_transform, valid_transform = utils._data_transforms_cifar10( args.cutout_size) train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform) valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform) train_queue = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=16) valid_queue = torch.utils.data.DataLoader(valid_data, batch_size=args.eval_batch_size, shuffle=False, pin_memory=True, num_workers=16) model = NASNetworkCIFAR(args, 100, args.layers, args.nodes, args.channels, args.keep_prob, args.drop_path_keep_prob, args.use_aux_head, args.steps, args.arch) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) logging.info("multi adds = %fM", model.multi_adds / 1000000) if model_state_dict is not None: model.load_state_dict(model_state_dict) if torch.cuda.device_count() > 1: logging.info("Use %d %s", torch.cuda.device_count(), "GPUs !") model = nn.DataParallel(model) model = model.cuda() train_criterion = nn.CrossEntropyLoss().cuda() eval_criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD( model.parameters(), args.lr_max, momentum=0.9, weight_decay=args.l2_reg, ) if optimizer_state_dict is not None: optimizer.load_state_dict(optimizer_state_dict) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), args.lr_min, epoch) return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def get_current_model_size(self): model = ModelForModelSizeMeasure(self._C, self._num_classes, self._layers, self._criterion, self.alphas_normal, self.alphas_reduce, self._steps, self._multiplier, self._stem_multiplier) size = count_parameters_in_MB(model) # This need to be further checked with cuda stuff del model return size
def calculate_batch_size(batch_model, memory, genotype: Genotype, init_channels: int, classes: int, layers: int, auxiliary: bool) -> int: model = Network(init_channels, classes, layers, auxiliary, genotype) model_size_mb = utils.count_parameters_in_MB(model) batch_size = 0 consumption = 0 while consumption < (memory - 500): batch_size += 1 consumption = batch_model.predict([[model_size_mb, batch_size]]) log.info("Predicted consumption is %d MB", consumption) return batch_size
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) try: genotype = eval("genotypes.%s" % args.arch) except (AttributeError, SyntaxError): genotype = genotypes.load_genotype_from_file(args.arch) test_data, OUTPUT_DIM, IN_CHANNELS, is_regression = load_dataset( args, train=False) model = Network(args.init_channels, OUTPUT_DIM, args.layers, args.auxiliary, genotype, num_channels=IN_CHANNELS) model = model.cuda() utils.load(model, args.model_path) logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) criterion = nn.CrossEntropyLoss() if not is_regression else nn.MSELoss() criterion = criterion.cuda() test_queue = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2) model.drop_path_prob = args.drop_path_prob test_acc, test_obj = infer(test_queue, model, criterion, is_regression=is_regression) logging.info('test_acc (R^2 for regression) %f', test_acc) weights_foldername = os.path.dirname(args.model_path) with open(os.path.join(weights_foldername, "test.txt"), "w") as f: f.write(str(test_acc))
def __init__(self, context: PyTorchTrialContext) -> None: self.context = context self.data_config = context.get_data_config() self.hparams = context.get_hparams() self.criterion = torch.nn.functional.cross_entropy # The last epoch is only used for logging. self._last_epoch = -1 self.results = { "loss": float("inf"), "top1_accuracy": 0, "top5_accuracy": 0 } # Define the model genotype = self.get_genotype_from_hps() self.model = self.context.wrap_model( Network( self.hparams["init_channels"], 10, # num_classes self.hparams["layers"], self.hparams["auxiliary"], genotype, )) print("param size = {} MB".format( utils.count_parameters_in_MB(self.model))) size = 0 for p in self.model.parameters(): size += p.nelement() print("param count: {}".format(size)) # Apply constraints if desired if "use_constraints" in self.hparams and self.hparams[ "use_constraints"]: apply_constraints(self.hparams, size) # Define the optimizer self.optimizer = self.context.wrap_optimizer( torch.optim.SGD( self.model.parameters(), lr=self.context.get_hparam("learning_rate"), momentum=self.context.get_hparam("momentum"), weight_decay=self.context.get_hparam("weight_decay"), )) # Define the LR scheduler self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( self.optimizer, self.context.get_hparam("train_epochs"), ) step_mode = LRScheduler.StepMode.STEP_EVERY_EPOCH self.wrapped_scheduler = self.context.wrap_lr_scheduler( self.scheduler, step_mode=step_mode)