class Nasbench201(Dataset): """Nasbench201 Dataset.""" def __init__(self): """Construct the Nasbench201 class.""" super(Nasbench201, self).__init__() self.args.data_path = FileOps.download_dataset(self.args.data_path) self.nasbench201_api = API('self.args.data_path') def query(self, arch_str, dataset): """Query an item from the dataset according to the given arch_str and dataset . :arch_str: arch_str to define the topology of the cell :type path: str :dataset: dataset type :type dataset: str :return: an item of the dataset, which contains the network info and its results like accuracy, flops and etc :rtype: dict """ if dataset not in VALID_DATASET: raise ValueError( "Only cifar10, cifar100, and Imagenet dataset is supported.") ops_list = self.nasbench201_api.str2lists(arch_str) for op in ops_list: if op not in VALID_OPS: raise ValueError( "{} is not in the nasbench201 space.".format(op)) index = self.nasbench201_api.query_index_by_arch(arch_str) results = self.nasbench201_api.query_by_index(index, dataset) return results
def main(xargs): assert torch.cuda.is_available(), 'CUDA is not available.' torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True torch.set_num_threads(xargs.workers) prepare_seed(xargs.rand_seed) if os.path.isdir(xargs.save_dir): if click.confirm( '\nSave directory already exists in {}. Erase?'.format( xargs.save_dir, default=False)): os.system('rm -r ' + xargs.save_dir) assert not os.path.exists(xargs.save_dir) os.mkdir(xargs.save_dir) logger = prepare_logger(args) writer = SummaryWriter(xargs.save_dir) perturb_alpha = None if xargs.perturb: perturb_alpha = random_alpha train_data, valid_data, xshape, class_num = get_datasets( xargs.dataset, xargs.data_path, -1) # config_path = 'configs/nas-benchmark/algos/DARTS.config' config = load_config(xargs.config_path, { 'class_num': class_num, 'xshape': xshape }, logger) search_loader, _, valid_loader = get_nas_search_loaders( train_data, valid_data, xargs.dataset, 'configs/nas-benchmark/', config.batch_size, xargs.workers) logger.log( '||||||| {:10s} ||||||| Search-Loader-Num={:}, Valid-Loader-Num={:}, batch size={:}' .format(xargs.dataset, len(search_loader), len(valid_loader), config.batch_size)) logger.log('||||||| {:10s} ||||||| Config={:}'.format( xargs.dataset, config)) search_space = get_search_spaces('cell', xargs.search_space_name) if xargs.model_config is None: model_config = dict2config( { 'name': xargs.model, 'C': xargs.channel, 'N': xargs.num_cells, 'max_nodes': xargs.max_nodes, 'num_classes': class_num, 'space': search_space, 'affine': bool(xargs.affine), 'track_running_stats': bool(xargs.track_running_stats) }, None) else: model_config = load_config( xargs.model_config, { 'num_classes': class_num, 'space': search_space, 'affine': bool(xargs.affine), 'track_running_stats': bool(xargs.track_running_stats) }, None) search_model = get_cell_based_tiny_net(model_config) # logger.log('search-model :\n{:}'.format(search_model)) w_optimizer, w_scheduler, criterion = get_optim_scheduler( search_model.get_weights(), config, xargs.weight_learning_rate) a_optimizer = torch.optim.Adam(search_model.get_alphas(), lr=xargs.arch_learning_rate, betas=(0.5, 0.999), weight_decay=xargs.arch_weight_decay) logger.log('w-optimizer : {:}'.format(w_optimizer)) logger.log('a-optimizer : {:}'.format(a_optimizer)) logger.log('w-scheduler : {:}'.format(w_scheduler)) logger.log('criterion : {:}'.format(criterion)) flop, param = get_model_infos(search_model, xshape) # logger.log('{:}'.format(search_model)) logger.log('FLOP = {:.2f} M, Params = {:.2f} MB'.format(flop, param)) if xargs.arch_nas_dataset is None: api = None else: api = API(xargs.arch_nas_dataset) logger.log('{:} create API = {:} done'.format(time_string(), api)) last_info, model_base_path, model_best_path = logger.path( 'info'), logger.path('model'), logger.path('best') network, criterion = torch.nn.DataParallel( search_model).cuda(), criterion.cuda() if last_info.exists(): # automatically resume from previous checkpoint logger.log("=> loading checkpoint of the last-info '{:}' start".format( last_info)) last_info = torch.load(last_info) start_epoch = last_info['epoch'] checkpoint = torch.load(last_info['last_checkpoint']) genotypes = checkpoint['genotypes'] valid_accuracies = checkpoint['valid_accuracies'] search_model.load_state_dict(checkpoint['search_model']) w_scheduler.load_state_dict(checkpoint['w_scheduler']) w_optimizer.load_state_dict(checkpoint['w_optimizer']) a_optimizer.load_state_dict(checkpoint['a_optimizer']) logger.log( "=> loading checkpoint of the last-info '{:}' start with {:}-th epoch." .format(last_info, start_epoch)) else: logger.log("=> do not find the last-info file : {:}".format(last_info)) start_epoch, valid_accuracies, genotypes = 0, { 'best': -1 }, { -1: search_model.genotype() } # start training # start_time, search_time, epoch_time, total_epoch = time.time(), AverageMeter(), AverageMeter(), config.epochs + config.warmup start_time, search_time, epoch_time = time.time(), AverageMeter( ), AverageMeter() total_epoch = config.epochs + config.warmup assert 0 < xargs.early_stop_epoch <= total_epoch - 1 for epoch in range(start_epoch, total_epoch): if epoch >= xargs.early_stop_epoch: logger.log(f"Early stop @ {epoch} epoch.") break if xargs.perturb: epsilon_alpha = 0.03 + (xargs.epsilon_alpha - 0.03) * epoch / total_epoch logger.log(f'epoch {epoch} epsilon_alpha {epsilon_alpha}') else: epsilon_alpha = None w_scheduler.update(epoch, 0.0) need_time = 'Time Left: {:}'.format( convert_secs2time(epoch_time.val * (total_epoch - epoch), True)) epoch_str = '{:03d}-{:03d}'.format(epoch, total_epoch) logger.log('\n[Search the {:}-th epoch] {:}, LR={:}'.format( epoch_str, need_time, min(w_scheduler.get_lr()))) search_w_loss, search_w_top1, search_w_top5, search_a_loss, search_a_top1, search_a_top5 = search_func( search_loader, network, criterion, w_scheduler, w_optimizer, a_optimizer, epoch_str, xargs.print_freq, logger, xargs.gradient_clip, perturb_alpha, epsilon_alpha) search_time.update(time.time() - start_time) logger.log( '[{:}] searching : loss={:.2f}, accuracy@1={:.2f}%, accuracy@5={:.2f}%, time-cost={:.1f} s' .format(epoch_str, search_w_loss, search_w_top1, search_w_top5, search_time.sum)) valid_a_loss, valid_a_top1, valid_a_top5 = valid_func( valid_loader, network, criterion) writer.add_scalar('search/weight_loss', search_w_loss, epoch) writer.add_scalar('search/weight_top1_acc', search_w_top1, epoch) writer.add_scalar('search/weight_top5_acc', search_w_top5, epoch) writer.add_scalar('search/arch_loss', search_a_loss, epoch) writer.add_scalar('search/arch_top1_acc', search_a_top1, epoch) writer.add_scalar('search/arch_top5_acc', search_a_top5, epoch) writer.add_scalar('evaluate/loss', valid_a_loss, epoch) writer.add_scalar('evaluate/top1_acc', valid_a_top1, epoch) writer.add_scalar('evaluate/top5_acc', valid_a_top5, epoch) logger.log( '[{:}] evaluate : loss={:.2f}, accuracy@1={:.2f}%, accuracy@5={:.2f}%' .format(epoch_str, valid_a_loss, valid_a_top1, valid_a_top5)) writer.add_scalar('entropy', search_model.entropy, epoch) per_edge_dict = get_per_egde_value_dict(search_model.arch_parameters) for edge_name, edge_val in per_edge_dict.items(): writer.add_scalars(f"cell/{edge_name}", edge_val, epoch) # check the best accuracy valid_accuracies[epoch] = valid_a_top1 if valid_a_top1 > valid_accuracies['best']: valid_accuracies['best'] = valid_a_top1 genotypes['best'] = search_model.genotype() find_best = True else: find_best = False genotypes[epoch] = search_model.genotype() logger.log('<<<--->>> The {:}-th epoch : {:}'.format( epoch_str, genotypes[epoch])) # save checkpoint save_path = save_checkpoint( { 'epoch': epoch + 1, 'args': deepcopy(xargs), 'search_model': search_model.state_dict(), 'w_optimizer': w_optimizer.state_dict(), 'a_optimizer': a_optimizer.state_dict(), 'w_scheduler': w_scheduler.state_dict(), 'genotypes': genotypes, 'valid_accuracies': valid_accuracies }, model_base_path, logger) save_checkpoint( { 'epoch': epoch + 1, 'args': deepcopy(args), 'last_checkpoint': save_path, }, logger.path('info'), logger) if xargs.snapshoot > 0 and epoch % xargs.snapshoot == 0: save_checkpoint( { 'epoch': epoch + 1, 'args': deepcopy(args), 'search_model': search_model.state_dict(), }, os.path.join(str(logger.model_dir), f"checkpoint_epoch{epoch}.pth"), logger) if find_best: logger.log( '<<<--->>> The {:}-th epoch : find the highest validation accuracy : {:.2f}%.' .format(epoch_str, valid_a_top1)) copy_checkpoint(model_base_path, model_best_path, logger) with torch.no_grad(): logger.log('{:}'.format(search_model.show_alphas())) if api is not None: logger.log('{:}'.format(api.query_by_arch(genotypes[epoch]))) index = api.query_index_by_arch(genotypes[epoch]) info = api.query_meta_info_by_index( index) # This is an instance of `ArchResults` res_metrics = info.get_metrics( f'{xargs.dataset}', 'ori-test') # This is a dict with metric names as keys # cost_metrics = info.get_comput_costs('cifar10') writer.add_scalar(f'{xargs.dataset}_ground_acc_ori-test', res_metrics['accuracy'], epoch) writer.add_scalar(f'{xargs.dataset}_search_acc', valid_a_top1, epoch) if xargs.dataset.lower() != 'cifar10': writer.add_scalar( f'{xargs.dataset}_ground_acc_x-test', info.get_metrics(f'{xargs.dataset}', 'x-test')['accuracy'], epoch) if find_best: valid_accuracies['best_gt'] = res_metrics['accuracy'] writer.add_scalar(f"{xargs.dataset}_cur_best_gt_acc_ori-test", valid_accuracies['best_gt'], epoch) # measure elapsed time epoch_time.update(time.time() - start_time) start_time = time.time() logger.log('\n' + '-' * 100) logger.log('{:} : run {:} epochs, cost {:.1f} s, last-geno is {:}.'.format( args.model, xargs.early_stop_epoch, search_time.sum, genotypes[xargs.early_stop_epoch - 1])) if api is not None: logger.log('{:}'.format( api.query_by_arch(genotypes[xargs.early_stop_epoch - 1]))) logger.close()
#model_config = {'C': 16, 'N': 5, 'num_classes': num_classes, 'max_nodes': 4, 'search_space': NAS_BENCH_201, 'affine': False} model = TinyNetwork(C = args.init_channels, N = args.num_cells, max_nodes = args.max_nodes, num_classes = num_classes, search_space = NAS_BENCH_201, affine = False, track_running_stats = args.track_running_stats) model = model.to(device) #logging.info(model) optimizer, _, criterion = get_optim_scheduler(parameters=model.get_weights(), config=config) criterion = criterion.cuda() logging.info(f'optimizer: {optimizer}\nCriterion: {criterion}') # logging the initialized architecture best_arch_per_epoch = [] arch_str = model.genotype().tostr() arch_index = api.query_index_by_arch(model.genotype()) if args.dataset == 'cifar10': test_acc = get_arch_score(api, arch_index, 'cifar10', 200, acc_type) valid_acc = get_arch_score(api, arch_index, 'cifar10-valid', 200, val_acc_type) writer.add_scalar("test_acc", test_acc, 0) writer.add_scalar("valid_acc", valid_acc, 0) else: test_acc = get_arch_score(api, arch_index, args.dataset, 200, acc_type) valid_acc = get_arch_score(api, arch_index, args.dataset, 200, val_acc_type) writer.add_scalar("test_acc", test_acc, 0) writer.add_scalar("valid_acc", valid_acc, 0) tmp = (arch_str, test_acc, valid_acc) best_arch_per_epoch.append(tmp) ''' optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum = args.momentum, weight_decay = args.weight_decay)
def main(xargs): assert torch.cuda.is_available(), 'CUDA is not available.' torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True torch.set_num_threads(xargs.workers) prepare_seed(xargs.rand_seed) logger = prepare_logger(args) train_data, valid_data, xshape, class_num = get_datasets( xargs.dataset, xargs.data_path, -1) #config_path = 'configs/nas-benchmark/algos/GDAS.config' config = load_config(xargs.config_path, { 'class_num': class_num, 'xshape': xshape }, logger) search_loader, _, valid_loader = get_nas_search_loaders( train_data, valid_data, xargs.dataset, 'configs/nas-benchmark/', config.batch_size, xargs.workers) logger.log( '||||||| {:10s} ||||||| Search-Loader-Num={:}, batch size={:}'.format( xargs.dataset, len(search_loader), config.batch_size)) logger.log('||||||| {:10s} ||||||| Config={:}'.format( xargs.dataset, config)) search_space = get_search_spaces('cell', xargs.search_space_name) if xargs.model_config is None and not args.constrain: model_config = dict2config( { 'name': 'GDAS', 'C': xargs.channel, 'N': xargs.num_cells, 'max_nodes': xargs.max_nodes, 'num_classes': class_num, 'space': search_space, 'inp_size': 0, 'affine': False, 'track_running_stats': bool(xargs.track_running_stats) }, None) elif xargs.model_config is None: model_config = dict2config( { 'name': 'GDAS', 'C': xargs.channel, 'N': xargs.num_cells, 'max_nodes': xargs.max_nodes, 'num_classes': class_num, 'space': search_space, 'inp_size': 32, 'affine': False, 'track_running_stats': bool(xargs.track_running_stats) }, None) else: model_config = load_config( xargs.model_config, { 'num_classes': class_num, 'space': search_space, 'affine': False, 'track_running_stats': bool(xargs.track_running_stats) }, None) search_model = get_cell_based_tiny_net(model_config) #logger.log('search-model :\n{:}'.format(search_model)) logger.log('model-config : {:}'.format(model_config)) w_optimizer, w_scheduler, criterion = get_optim_scheduler( search_model.get_weights(), config) a_optimizer = torch.optim.Adam(search_model.get_alphas(), lr=xargs.arch_learning_rate, betas=(0.5, 0.999), weight_decay=xargs.arch_weight_decay) logger.log('w-optimizer : {:}'.format(w_optimizer)) logger.log('a-optimizer : {:}'.format(a_optimizer)) logger.log('w-scheduler : {:}'.format(w_scheduler)) logger.log('criterion : {:}'.format(criterion)) flop, param = get_model_infos(search_model, xshape) #logger.log('{:}'.format(search_model)) logger.log('FLOP = {:.2f} M, Params = {:.2f} MB'.format(flop, param)) logger.log('search-space [{:} ops] : {:}'.format(len(search_space), search_space)) if xargs.arch_nas_dataset is None: api = None else: api = API(xargs.arch_nas_dataset) logger.log('{:} create API = {:} done'.format(time_string(), api)) last_info, model_base_path, model_best_path = logger.path( 'info'), logger.path('model'), logger.path('best') network, criterion = torch.nn.DataParallel( search_model).cuda(), criterion.cuda() #network, criterion = search_model.cuda(), criterion.cuda() if last_info.exists(): # automatically resume from previous checkpoint logger.log("=> loading checkpoint of the last-info '{:}' start".format( last_info)) last_info = torch.load(last_info) start_epoch = last_info['epoch'] checkpoint = torch.load(last_info['last_checkpoint']) genotypes = checkpoint['genotypes'] valid_accuracies = checkpoint['valid_accuracies'] search_model.load_state_dict(checkpoint['search_model']) w_scheduler.load_state_dict(checkpoint['w_scheduler']) w_optimizer.load_state_dict(checkpoint['w_optimizer']) a_optimizer.load_state_dict(checkpoint['a_optimizer']) logger.log( "=> loading checkpoint of the last-info '{:}' start with {:}-th epoch." .format(last_info, start_epoch)) else: logger.log("=> do not find the last-info file : {:}".format(last_info)) start_epoch, valid_accuracies, genotypes = 0, { 'best': -1 }, { -1: search_model.genotype() } # start training start_time, search_time, epoch_time, total_epoch = time.time( ), AverageMeter(), AverageMeter(), config.epochs + config.warmup sampled_weights = [] for epoch in range(start_epoch, total_epoch + config.t_epochs): w_scheduler.update(epoch, 0.0) need_time = 'Time Left: {:}'.format( convert_secs2time( epoch_time.val * (total_epoch - epoch + config.t_epochs), True)) epoch_str = '{:03d}-{:03d}'.format(epoch, total_epoch) search_model.set_tau(xargs.tau_max - (xargs.tau_max - xargs.tau_min) * epoch / (total_epoch - 1)) logger.log('\n[Search the {:}-th epoch] {:}, tau={:}, LR={:}'.format( epoch_str, need_time, search_model.get_tau(), min(w_scheduler.get_lr()))) if epoch < total_epoch: search_w_loss, search_w_top1, search_w_top5, valid_a_loss , valid_a_top1 , valid_a_top5 \ = search_func(search_loader, network, criterion, w_scheduler, w_optimizer, a_optimizer, epoch_str, xargs.print_freq, logger, xargs.bilevel) else: search_w_loss, search_w_top1, search_w_top5, valid_a_loss , valid_a_top1 , valid_a_top5, arch_iter \ = train_func(search_loader, network, criterion, w_scheduler, w_optimizer, epoch_str, xargs.print_freq, sampled_weights[0], arch_iter, logger) search_time.update(time.time() - start_time) logger.log( '[{:}] searching : loss={:.2f}, accuracy@1={:.2f}%, accuracy@5={:.2f}%, time-cost={:.1f} s' .format(epoch_str, search_w_loss, search_w_top1, search_w_top5, search_time.sum)) logger.log( '[{:}] evaluate : loss={:.2f}, accuracy@1={:.2f}%, accuracy@5={:.2f}%' .format(epoch_str, valid_a_loss, valid_a_top1, valid_a_top5)) if (epoch + 1) % 50 == 0 and not config.t_epochs: weights = search_model.sample_weights(100) sampled_weights.append(weights) elif (epoch + 1) == total_epoch and config.t_epochs: weights = search_model.sample_weights(100) sampled_weights.append(weights) arch_iter = iter(weights) # validate with single arch single_weight = search_model.sample_weights(1)[0] single_valid_acc = AverageMeter() network.eval() for i in range(10): try: val_input, val_target = next(valid_iter) except Exception as e: valid_iter = iter(valid_loader) val_input, val_target = next(valid_iter) n_val = val_input.size(0) with torch.no_grad(): val_target = val_target.cuda(non_blocking=True) _, logits, _ = network(val_input, weights=single_weight) val_acc1, val_acc5 = obtain_accuracy(logits.data, val_target.data, topk=(1, 5)) single_valid_acc.update(val_acc1.item(), n_val) logger.log('[{:}] valid : accuracy = {:.2f}'.format( epoch_str, single_valid_acc.avg)) # check the best accuracy valid_accuracies[epoch] = valid_a_top1 if valid_a_top1 > valid_accuracies['best']: valid_accuracies['best'] = valid_a_top1 genotypes['best'] = search_model.genotype() find_best = True else: find_best = False if epoch < total_epoch: genotypes[epoch] = search_model.genotype() logger.log('<<<--->>> The {:}-th epoch : {:}'.format( epoch_str, genotypes[epoch])) # save checkpoint save_path = save_checkpoint( { 'epoch': epoch + 1, 'args': deepcopy(xargs), 'search_model': search_model.state_dict(), 'w_optimizer': w_optimizer.state_dict(), 'a_optimizer': a_optimizer.state_dict(), 'w_scheduler': w_scheduler.state_dict(), 'genotypes': genotypes, 'valid_accuracies': valid_accuracies }, model_base_path, logger) last_info = save_checkpoint( { 'epoch': epoch + 1, 'args': deepcopy(args), 'last_checkpoint': save_path, }, logger.path('info'), logger) if find_best: logger.log( '<<<--->>> The {:}-th epoch : find the highest validation accuracy : {:.2f}%.' .format(epoch_str, valid_a_top1)) copy_checkpoint(model_base_path, model_best_path, logger) with torch.no_grad(): logger.log('{:}'.format(search_model.show_alphas())) if api is not None and epoch < total_epoch: logger.log('{:}'.format(api.query_by_arch(genotypes[epoch]))) # measure elapsed time epoch_time.update(time.time() - start_time) start_time = time.time() network.eval() # Evaluate the architectures sampled throughout the search for i in range(len(sampled_weights) - 1): logger.log('Sample eval : epoch {}'.format((i + 1) * 50 - 1)) for w in sampled_weights[i]: sample_valid_acc = AverageMeter() for i in range(10): try: val_input, val_target = next(valid_iter) except Exception as e: valid_iter = iter(valid_loader) val_input, val_target = next(valid_iter) n_val = val_input.size(0) with torch.no_grad(): val_target = val_target.cuda(non_blocking=True) _, logits, _ = network(val_input, weights=w) val_acc1, val_acc5 = obtain_accuracy(logits.data, val_target.data, topk=(1, 5)) sample_valid_acc.update(val_acc1.item(), n_val) w_gene = search_model.genotype(w) if api is not None: ind = api.query_index_by_arch(w_gene) info = api.query_meta_info_by_index(ind) metrics = info.get_metrics('cifar10', 'ori-test') acc = metrics['accuracy'] else: acc = 0.0 logger.log( 'sample valid : val_acc = {:.2f} test_acc = {:.2f}'.format( sample_valid_acc.avg, acc)) # Evaluate the final sampling separately to find the top 10 architectures logger.log('Final sample eval') final_archs = [] for w in sampled_weights[-1]: sample_valid_acc = AverageMeter() for i in range(10): try: val_input, val_target = next(valid_iter) except Exception as e: valid_iter = iter(valid_loader) val_input, val_target = next(valid_iter) n_val = val_input.size(0) with torch.no_grad(): val_target = val_target.cuda(non_blocking=True) _, logits, _ = network(val_input, weights=w) val_acc1, val_acc5 = obtain_accuracy(logits.data, val_target.data, topk=(1, 5)) sample_valid_acc.update(val_acc1.item(), n_val) w_gene = search_model.genotype(w) if api is not None: ind = api.query_index_by_arch(w_gene) info = api.query_meta_info_by_index(ind) metrics = info.get_metrics('cifar10', 'ori-test') acc = metrics['accuracy'] else: acc = 0.0 logger.log('sample valid : val_acc = {:.2f} test_acc = {:.2f}'.format( sample_valid_acc.avg, acc)) final_archs.append((w, sample_valid_acc.avg)) top_10 = sorted(final_archs, key=lambda x: x[1], reverse=True)[:10] # Evaluate the top 10 architectures on the entire validation set logger.log('Evaluating top archs') for w, prev_acc in top_10: full_valid_acc = AverageMeter() for val_input, val_target in valid_loader: n_val = val_input.size(0) with torch.no_grad(): val_target = val_target.cuda(non_blocking=True) _, logits, _ = network(val_input, weights=w) val_acc1, val_acc5 = obtain_accuracy(logits.data, val_target.data, topk=(1, 5)) full_valid_acc.update(val_acc1.item(), n_val) w_gene = search_model.genotype(w) logger.log('genotype {}'.format(w_gene)) if api is not None: ind = api.query_index_by_arch(w_gene) info = api.query_meta_info_by_index(ind) metrics = info.get_metrics('cifar10', 'ori-test') acc = metrics['accuracy'] else: acc = 0.0 logger.log( 'full valid : val_acc = {:.2f} test_acc = {:.2f} pval_acc = {:.2f}' .format(full_valid_acc.avg, acc, prev_acc)) logger.log('\n' + '-' * 100) # check the performance from the architecture dataset logger.log( 'GDAS : run {:} epochs, cost {:.1f} s, last-geno is {:}.'.format( total_epoch, search_time.sum, genotypes[total_epoch - 1])) if api is not None: logger.log('{:}'.format(api.query_by_arch(genotypes[total_epoch - 1]))) logger.close()
class MacroGraph(NodeOpGraph): def __init__(self, config, primitives, ops_dict, *args, **kwargs): self.config = config self.primitives = primitives self.ops_dict = ops_dict self.nasbench_api = API('/home/siemsj/nasbench_201.pth') super(MacroGraph, self).__init__(*args, **kwargs) def _build_graph(self): num_cells_per_stack = self.config['num_cells_per_stack'] C = self.config['init_channels'] layer_channels = [C] * num_cells_per_stack + [C * 2] + [C * 2] * num_cells_per_stack + [C * 4] + [ C * 4] * num_cells_per_stack layer_reductions = [False] * num_cells_per_stack + [True] + [False] * num_cells_per_stack + [True] + [ False] * num_cells_per_stack stem = NASBENCH_201_Stem(C=C) self.add_node(0, type='input') self.add_node(1, op=stem, type='stem') C_prev = C self.cells = nn.ModuleList() for cell_num, (C_curr, reduction) in enumerate(zip(layer_channels, layer_reductions)): if reduction: cell = ResNetBasicblock(C_prev, C_curr, 2, True) self.add_node(cell_num + 2, op=cell, primitives=self.primitives, transform=lambda x: x[0]) else: cell = Cell(primitives=self.primitives, stride=1, C_prev=C_prev, C=C_curr, ops_dict=self.ops_dict, cell_type='normal') self.add_node(cell_num + 2, op=cell, primitives=self.primitives) C_prev = C_curr lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True)) pooling = nn.AdaptiveAvgPool2d(1) classifier = nn.Linear(C_prev, self.config['num_classes']) self.add_node(cell_num + 3, op=lastact, transform=lambda x: x[0], type='postprocessing_nb201') self.add_node(cell_num + 4, op=pooling, transform=lambda x: x[0], type='pooling') self.add_node(cell_num + 5, op=classifier, transform=lambda x: x[0].view(x[0].size(0), -1), type='output') # Edges for i in range(1, cell_num + 6): self.add_edge(i - 1, i, type='input', desc='previous') def sample(self, same_cell_struct=True, n_ops_per_edge=1, n_input_edges=None, dist=None, seed=1): """ same_cell_struct: True; if the sampled cell topology is the same or not n_ops_per_edge: 1; number of sampled operations per edge in cell n_input_edges: None; list equal with length with number of intermediate nodes. Determines the number of predecesor nodes for each of them dist: None; distribution to sample operations in edges from seed: 1; random seed """ # create a new graph that we will discretize new_graph = MacroGraph(self.config, self.primitives, self.ops_dict) np.random.seed(seed) seeds = {'normal': seed + 1, 'reduction': seed + 2} for node in new_graph: cell = new_graph.get_node_op(node) if not isinstance(cell, Cell): continue if same_cell_struct: np.random.seed(seeds[new_graph.get_node_type(node)]) for edge in cell.edges: op_choices = cell.get_edge_op_choices(*edge) sampled_op = np.random.choice(op_choices, n_ops_per_edge, False, p=dist) cell[edge[0]][edge[1]]['op_choices'] = [*sampled_op] if n_input_edges is not None: for inter_node, k in zip(cell.inter_nodes(), n_input_edges): # in case the start node index is not 0 node_idx = list(cell.nodes).index(inter_node) prev_node_choices = list(cell.nodes)[:node_idx] assert k <= len(prev_node_choices), 'cannot sample more' ' than number of predecesor nodes' sampled_input_edges = np.random.choice(prev_node_choices, k, False) for i in set(prev_node_choices) - set(sampled_input_edges): cell.remove_edge(i, inter_node) return new_graph @classmethod def from_config(cls, config=None, filename=None): with open(filename, 'r') as f: graph_dict = yaml.safe_load(f) if config is None: raise ('No configuration provided') graph = cls(config, []) graph_type = graph_dict['type'] edges = [(*eval(e), attr) for e, attr in graph_dict['edges'].items()] graph.clear() graph.add_edges_from(edges) C = config['init_channels'] C_curr = config['stem_multiplier'] * C stem = Stem(C_curr=C_curr) C_prev_prev, C_prev, C_curr = C_curr, C_curr, C for node, attr in graph_dict['nodes'].items(): node_type = attr['type'] if node_type == 'input': graph.add_node(node, type='input') elif node_type == 'stem': graph.add_node(node, op=stem, type='stem') elif node_type in ['normal', 'reduction']: assert attr['op']['type'] == 'Cell' graph.add_node(node, op=Cell.from_config(attr['op'], primitives=attr['op']['primitives'], C_prev_prev=C_prev_prev, C_prev=C_prev, C=C_curr, reduction_prev=graph_dict['nodes'][node - 1]['type'] == 'reduction', cell_type=node_type), type=node_type) C_prev_prev, C_prev = C_prev, config['channel_multiplier'] * C_curr elif node_type == 'pooling': pooling = nn.AdaptiveAvgPool2d(1) graph.add_node(node, op=pooling, transform=lambda x: x[0], type='pooling') elif node_type == 'output': classifier = nn.Linear(C_prev, config['num_classes']) graph.add_node(node, op=classifier, transform=lambda x: x[0].view(x[0].size(0), -1), type='output') return graph @staticmethod def export_nasbench_201_results_to_dict(information): results_dict = {} dataset_names = information.get_dataset_names() results_dict['arch'] = information.arch_str results_dict['datasets'] = dataset_names for ida, dataset in enumerate(dataset_names): dataset_results = {} dataset_results['dataset'] = dataset metric = information.get_compute_costs(dataset) flop, param, latency, training_time = metric['flops'], metric['params'], metric['latency'], metric[ 'T-train@total'] dataset_results['flop'] = flop dataset_results['params (MB)'] = param dataset_results['latency (ms)'] = latency * 1000 if latency is not None and latency > 0 else None dataset_results['training_time'] = training_time train_info = information.get_metrics(dataset, 'train') if dataset == 'cifar10-valid': valid_info = information.get_metrics(dataset, 'x-valid') dataset_results['train_loss'] = train_info['loss'] dataset_results['train_accuracy'] = train_info['accuracy'] dataset_results['valid_loss'] = valid_info['loss'] dataset_results['valid_accuracy'] = valid_info['accuracy'] elif dataset == 'cifar10': test__info = information.get_metrics(dataset, 'ori-test') dataset_results['train_loss'] = train_info['loss'] dataset_results['train_accuracy'] = train_info['accuracy'] dataset_results['test_loss'] = test__info['loss'] dataset_results['test_accuracy'] = test__info['accuracy'] else: valid_info = information.get_metrics(dataset, 'x-valid') test__info = information.get_metrics(dataset, 'x-test') dataset_results['train_loss'] = train_info['loss'] dataset_results['train_accuracy'] = train_info['accuracy'] dataset_results['valid_loss'] = valid_info['loss'] dataset_results['valid_accuracy'] = valid_info['accuracy'] dataset_results['test_loss'] = test__info['loss'] dataset_results['test_accuracy'] = test__info['accuracy'] results_dict[dataset] = dataset_results return results_dict def query_architecture(self, arch_weights): arch_weight_idx_to_parent = {0: 0, 1: 0, 2: 1, 3: 0, 4: 1, 5: 2} arch_strs = { 'cell_normal_from_0_to_1': '', 'cell_normal_from_0_to_2': '', 'cell_normal_from_1_to_2': '', 'cell_normal_from_0_to_3': '', 'cell_normal_from_1_to_3': '', 'cell_normal_from_2_to_3': '', } for arch_weight_idx, (edge_key, edge_weights) in enumerate(arch_weights.items()): edge_weights_norm = torch.softmax(edge_weights, dim=-1) selected_op_str = PRIMITIVES[edge_weights_norm.argmax()] arch_strs[edge_key] = '{}~{}'.format(selected_op_str, arch_weight_idx_to_parent[arch_weight_idx]) arch_str = '|{}|+|{}|{}|+|{}|{}|{}|'.format(*arch_strs.values()) if not hasattr(self, 'nasbench_api'): self.nasbench_api = API('/home/siemsj/nasbench_201.pth') index = self.nasbench_api.query_index_by_arch(arch_str) self.nasbench_api.show(index) info = self.nasbench_api.query_by_index(index) return self.export_nasbench_201_results_to_dict(info)
def make(ww="without"): from nas_201_api import NASBench201API as API api = API('/Users/madoibito80/NAS-Bench-201-v1_0-e61699.pth') macc = {} nparams = {} for mode in modes: for way in ways: if mode == "P" and way == "best": continue for trial in trials: fname = str(trial) + "_" + mode arcs = opener("./snapshot/" + ww + "/" + way + "/" + fname + ".txt", freeze=50, mode=mode) accs = [] for i in range(len(arcs)): if i % 100 == 0: print(mode, way, trial, i) index = api.query_index_by_arch(arcs[i]) flg = True try: info = api.query_meta_info_by_index(index) except: print("error: ", mode, way, trial, i) flg = False if flg: res = info.get_metrics('cifar10', 'ori-test', None, False) # cifar10 : training the model on the CIFAR-10 training + validation set. # ,criteria , , False=3average of NAS-Bench) acc = res['accuracy'] accs.append(float(acc)) else: accs.append(accs[-1]) if trial == 0: macc[mode + way] = np.zeros((len(trials), len(accs))) nparams[mode + way] = np.zeros((len(trials), 1)) macc[mode + way][trial] = np.array(accs) # get final performance for table try: print(dir(info)) except: print("no dir") metric = info.get_comput_costs('cifar10') flop, param, latency = metric['flops'], metric[ 'params'], metric['latency'] nparams[mode + way][trial] = float(param) f = open("./" + ww + ".pickle", "wb") pickle.dump(macc, f) pickle.dump(nparams, f) f.close() print(macc) print(nparams) return macc
class Nasbench201: def __init__(self, dataset, apiloc): self.dataset = dataset self.api = API(apiloc, verbose=False) self.epochs = '12' def get_network(self, uid): #config = self.api.get_net_config(uid, self.dataset) config = self.api.get_net_config(uid, 'cifar10-valid') config['num_classes'] = 1 network = get_cell_based_tiny_net(config) return network def __iter__(self): for uid in range(len(self)): network = self.get_network(uid) yield uid, network def __getitem__(self, index): return index def __len__(self): return 15625 def num_activations(self): network = self.get_network(0) return network.classifier.in_features #def get_12epoch_accuracy(self, uid, acc_type, trainval, traincifar10=False): # archinfo = self.api.query_meta_info_by_index(uid) # if (self.dataset == 'cifar10' or traincifar10) and trainval: # #return archinfo.get_metrics('cifar10-valid', acc_type, iepoch=12)['accuracy'] # return archinfo.get_metrics('cifar10-valid', 'x-valid', iepoch=12)['accuracy'] # elif traincifar10: # return archinfo.get_metrics('cifar10', acc_type, iepoch=12)['accuracy'] # else: # return archinfo.get_metrics(self.dataset, 'ori-test', iepoch=12)['accuracy'] def get_12epoch_accuracy(self, uid, acc_type, trainval, traincifar10=False): #archinfo = self.api.query_meta_info_by_index(uid) #if (self.dataset == 'cifar10' and trainval) or traincifar10: info = self.api.get_more_info(uid, 'cifar10-valid', iepoch=None, hp=self.epochs, is_random=True) #else: # info = self.api.get_more_info(uid, self.dataset, iepoch=None, hp=self.epochs, is_random=True) return info['valid-accuracy'] def get_final_accuracy(self, uid, acc_type, trainval): #archinfo = self.api.query_meta_info_by_index(uid) if self.dataset == 'cifar10' and trainval: info = self.api.query_meta_info_by_index( uid, hp='200').get_metrics('cifar10-valid', 'x-valid') #info = self.api.query_by_index(uid, 'cifar10-valid', hp='200') #info = self.api.get_more_info(uid, 'cifar10-valid', iepoch=None, hp='200', is_random=True) else: info = self.api.query_meta_info_by_index( uid, hp='200').get_metrics(self.dataset, acc_type) #info = self.api.query_by_index(uid, self.dataset, hp='200') #info = self.api.get_more_info(uid, self.dataset, iepoch=None, hp='200', is_random=True) return info['accuracy'] #return info['valid-accuracy'] #if self.dataset == 'cifar10' and trainval: # return archinfo.get_metrics('cifar10-valid', acc_type, iepoch=11)['accuracy'] #else: # #return archinfo.get_metrics(self.dataset, 'ori-test', iepoch=12)['accuracy'] # return archinfo.get_metrics(self.dataset, 'x-test', iepoch=11)['accuracy'] ##dataset = self.dataset ##if self.dataset == 'cifar10' and trainval: ## dataset = 'cifar10-valid' ##archinfo = self.api.get_more_info(uid, dataset, iepoch=None, use_12epochs_result=True, is_random=True) ##return archinfo['valid-accuracy'] def get_accuracy(self, uid, acc_type, trainval=True): archinfo = self.api.query_meta_info_by_index(uid) if self.dataset == 'cifar10' and trainval: return archinfo.get_metrics('cifar10-valid', acc_type)['accuracy'] else: return archinfo.get_metrics(self.dataset, acc_type)['accuracy'] def get_accuracy_for_all_datasets(self, uid): archinfo = self.api.query_meta_info_by_index(uid, hp='200') c10 = archinfo.get_metrics('cifar10', 'ori-test')['accuracy'] c10_val = archinfo.get_metrics('cifar10-valid', 'x-valid')['accuracy'] c100 = archinfo.get_metrics('cifar100', 'x-test')['accuracy'] c100_val = archinfo.get_metrics('cifar100', 'x-valid')['accuracy'] imagenet = archinfo.get_metrics('ImageNet16-120', 'x-test')['accuracy'] imagenet_val = archinfo.get_metrics('ImageNet16-120', 'x-valid')['accuracy'] return c10, c10_val, c100, c100_val, imagenet, imagenet_val #def train_and_eval(self, arch, dataname, acc_type, trainval=True): # unique_hash = self.__getitem__(arch) # time = self.get_training_time(unique_hash) # acc12 = self.get_12epoch_accuracy(unique_hash, acc_type, trainval) # acc = self.get_final_accuracy(unique_hash, acc_type, trainval) # return acc12, acc, time def train_and_eval(self, arch, dataname, acc_type, trainval=True, traincifar10=False): unique_hash = self.__getitem__(arch) time = self.get_training_time(unique_hash) acc12 = self.get_12epoch_accuracy(unique_hash, acc_type, trainval, traincifar10) acc = self.get_final_accuracy(unique_hash, acc_type, trainval) return acc12, acc, time def random_arch(self): return random.randint(0, len(self) - 1) def get_training_time(self, unique_hash): #info = self.api.get_more_info(unique_hash, 'cifar10-valid' if self.dataset == 'cifar10' else self.dataset, iepoch=None, use_12epochs_result=True, is_random=True) #info = self.api.get_more_info(unique_hash, 'cifar10-valid', iepoch=None, use_12epochs_result=True, is_random=True) info = self.api.get_more_info(unique_hash, 'cifar10-valid', iepoch=None, hp='12', is_random=True) return info['train-all-time'] + info['valid-per-time'] #if self.dataset == 'cifar10' and trainval: # info = self.api.get_more_info(unique_hash, 'cifar10-valid', iepoch=None, hp=self.epochs, is_random=True) #else: # info = self.api.get_more_info(unique_hash, self.dataset, iepoch=None, hp=self.epochs, is_random=True) ##info = self.api.get_more_info(unique_hash, 'cifar10-valid', iepoch=None, use_12epochs_result=True, is_random=True) #return info['train-all-time'] + info['valid-per-time'] def mutate_arch(self, arch): op_names = get_search_spaces('cell', 'nas-bench-201') #config = self.api.get_net_config(arch, self.dataset) config = self.api.get_net_config(arch, 'cifar10-valid') parent_arch = Structure(self.api.str2lists(config['arch_str'])) child_arch = deepcopy(parent_arch) node_id = random.randint(0, len(child_arch.nodes) - 1) node_info = list(child_arch.nodes[node_id]) snode_id = random.randint(0, len(node_info) - 1) xop = random.choice(op_names) while xop == node_info[snode_id][0]: xop = random.choice(op_names) node_info[snode_id] = (xop, node_info[snode_id][1]) child_arch.nodes[node_id] = tuple(node_info) arch_index = self.api.query_index_by_arch(child_arch) return arch_index
class NAS201(ObjectiveFunction): def __init__(self, data_dir, task='cifar10-valid', log_scale=True, negative=True, use_12_epochs_result=False, seed=None): """ data_dir: data directory that contains NAS-Bench-201-v1_0-e61699.pth file task: the target image tasks. Options: cifar10-valid, cifar100, ImageNet16-120 log_scale: whether output the objective in log scale negative: whether output the objective in negative form use_12_epochs_result: whether use the statistics at the end of training of the 12th epoch instead of all the way till the end. seed: set the random seed to access trained model performance: Options: 0, 1, 2 seed=None will select the seed randomly """ self.api = API(os.path.join(data_dir, 'NAS-Bench-201-v1_1-096897.pth')) if isinstance(task, list): task = task[0] self.task = task self.use_12_epochs_result = use_12_epochs_result if task == 'cifar10-valid': best_val_arch_index = 6111 best_val_acc = 91.60666665039064 / 100 best_test_arch_index = 1459 best_test_acc = 91.52333333333333 / 100 elif task == 'cifar100': best_val_arch_index = 9930 best_val_acc = 73.49333323567708 / 100 best_test_arch_index = 9930 best_test_acc = 73.51333326009114 / 100 elif task == 'ImageNet16-120': best_val_arch_index = 10676 best_val_acc = 46.766666727701825 / 100 best_test_arch_index = 857 best_test_acc = 47.311111097547744 / 100 else: raise NotImplementedError("task" + str(task) + " is not implemented in the dataset.") if log_scale: best_val_acc = np.log(best_val_acc) best_val_err = 1. - best_val_acc best_test_err = 1. - best_test_acc if log_scale: best_val_err = np.log(best_val_err) best_test_err = np.log(best_val_err) if negative: best_val_err = -best_val_err best_test_err = -best_test_err self.best_val_err = best_val_err self.best_test_err = best_test_err self.best_val_acc = best_val_acc self.best_test_acc = best_test_acc super(NAS201, self).__init__(dim=None, optimum_location=best_test_arch_index, optimal_val=best_test_err, bounds=None) self.log_scale = log_scale self.seed = seed self.X = [] self.y_valid_acc = [] self.y_test_acc = [] self.costs = [] self.negative = negative # self.optimal_val = # lowest mean validation error # self.y_star_test = # lowest mean test error def _retrieve(self, G, budget, which='eval'): # set random seed for evaluation if which == 'test': seed = 3 else: seed_list = [777, 888, 999] if self.seed is None: seed = random.choice(seed_list) elif self.seed >= 3: seed = self.seed else: seed = seed_list[self.seed] # find architecture index arch_str = G.name # print(arch_str) try: arch_index = self.api.query_index_by_arch(arch_str) acc_results = self.api.query_by_index(arch_index, self.task, use_12epochs_result=self.use_12_epochs_result,) if seed is not None and 3 <= seed < 777: # some architectures only contain 1 seed result acc_results = self.api.get_more_info(arch_index, self.task, None, use_12epochs_result=self.use_12_epochs_result, is_random=False) val_acc = acc_results['valid-accuracy'] / 100 test_acc = acc_results['test-accuracy'] / 100 else: try: acc_results = self.api.get_more_info(arch_index, self.task, None, use_12epochs_result=self.use_12_epochs_result, is_random=seed) val_acc = acc_results['valid-accuracy'] / 100 test_acc = acc_results['test-accuracy'] / 100 # val_acc = acc_results[seed].get_eval('x-valid')['accuracy'] / 100 # if self.task == 'cifar10-valid': # test_acc = acc_results[seed].get_eval('ori-test')['accuracy'] / 100 # else: # test_acc = acc_results[seed].get_eval('x-test')['accuracy'] / 100 except: # some architectures only contain 1 seed result acc_results = self.api.get_more_info(arch_index, self.task, None, use_12epochs_result=self.use_12_epochs_result, is_random=False) val_acc = acc_results['valid-accuracy'] / 100 test_acc = acc_results['test-accuracy'] / 100 auxiliary_info = self.api.query_meta_info_by_index(arch_index, use_12epochs_result=self.use_12_epochs_result) cost_info = auxiliary_info.get_compute_costs(self.task) # auxiliary cost results such as number of flops and number of parameters cost_results = {'flops': cost_info['flops'], 'params': cost_info['params'], 'latency': cost_info['latency']} except FileNotFoundError: val_acc = 0.01 test_acc = 0.01 print('missing arch info') cost_results = {'flops': None, 'params': None, 'latency': None} # store val and test performance + auxiliary cost information self.X.append(arch_str) self.y_valid_acc.append(val_acc) self.y_test_acc.append(test_acc) self.costs.append(cost_results) if which == 'eval': err = 1 - val_acc elif which == 'test': err = 1 - test_acc else: raise ValueError("Unknown query parameter: which = " + str(which)) if self.log_scale: y = np.log(err) else: y = err if self.negative: y = -y return y def eval(self, G, budget=199, n_repeat=1): # input is a list of graphs [G1,G2, ....] if n_repeat == 1: return self._retrieve(G, budget, 'eval'), [np.nan] return np.mean(np.array([self._retrieve(G, budget, 'eval') for _ in range(n_repeat)])), [np.nan] def test(self, G, budget=199, n_repeat=1): return np.mean(np.array([self._retrieve(G, budget, 'test') for _ in range(n_repeat)])) def get_results(self, ignore_invalid_configs=False): regret_validation = [] regret_test = [] costs = [] model_graph_specs = [] inc_valid = 0 inc_test = 0 for i in range(len(self.X)): if inc_valid < self.y_valid_acc[i]: inc_valid = self.y_valid_acc[i] inc_test = self.y_test_acc[i] regret_validation.append(float(self.best_val_acc - inc_valid)) regret_test.append(float(self.best_test_acc - inc_test)) model_graph_specs.append(self.X[i]) costs.append(self.costs[i]) res = dict() res['regret_validation'] = regret_validation res['regret_test'] = regret_test res['costs'] = costs res['model_graph_specs'] = model_graph_specs return res @staticmethod def get_configuration_space(): # for unpruned graph cs = ConfigSpace.ConfigurationSpace() ops_choices = ['nor_conv_3x3', 'nor_conv_1x1', 'avg_pool_3x3', 'skip_connect', 'none'] for i in range(6): cs.add_hyperparameter(ConfigSpace.CategoricalHyperparameter("edge_%d" % i, ops_choices)) return cs