def evaluate_one_shot(model, xloader, api, cal_mode, seed=111): print( "This is an old version of codes to use NAS-Bench-API, and should be modified to align with the new version. Please contact me for more details if you use this function." ) weights = deepcopy(model.state_dict()) model.train(cal_mode) with torch.no_grad(): logits = nn.functional.log_softmax(model.arch_parameters, dim=-1) archs = CellStructure.gen_all(model.op_names, model.max_nodes, False) probs, accuracies, gt_accs_10_valid, gt_accs_10_test = [], [], [], [] loader_iter = iter(xloader) random.seed(seed) random.shuffle(archs) for idx, arch in enumerate(archs): arch_index = api.query_index_by_arch(arch) metrics = api.get_more_info(arch_index, "cifar10-valid", None, False, False) gt_accs_10_valid.append(metrics["valid-accuracy"]) metrics = api.get_more_info(arch_index, "cifar10", None, False, False) gt_accs_10_test.append(metrics["test-accuracy"]) select_logits = [] for i, node_info in enumerate(arch.nodes): for op, xin in node_info: node_str = "{:}<-{:}".format(i + 1, xin) op_index = model.op_names.index(op) select_logits.append(logits[model.edge2index[node_str], op_index]) cur_prob = sum(select_logits).item() probs.append(cur_prob) cor_prob_valid = np.corrcoef(probs, gt_accs_10_valid)[0, 1] cor_prob_test = np.corrcoef(probs, gt_accs_10_test)[0, 1] print( "{:} correlation for probabilities : {:.6f} on CIFAR-10 validation and {:.6f} on CIFAR-10 test" .format(time_string(), cor_prob_valid, cor_prob_test)) for idx, arch in enumerate(archs): model.set_cal_mode("dynamic", arch) try: inputs, targets = next(loader_iter) except: loader_iter = iter(xloader) inputs, targets = next(loader_iter) _, logits = model(inputs.cuda()) _, preds = torch.max(logits, dim=-1) correct = (preds == targets.cuda()).float() accuracies.append(correct.mean().item()) if idx != 0 and (idx % 500 == 0 or idx + 1 == len(archs)): cor_accs_valid = np.corrcoef(accuracies, gt_accs_10_valid[:idx + 1])[0, 1] cor_accs_test = np.corrcoef(accuracies, gt_accs_10_test[:idx + 1])[0, 1] print( "{:} {:05d}/{:05d} mode={:5s}, correlation : accs={:.5f} for CIFAR-10 valid, {:.5f} for CIFAR-10 test." .format(time_string(), idx, len(archs), "Train" if cal_mode else "Eval", cor_accs_valid, cor_accs_test)) model.load_state_dict(weights) return archs, probs, accuracies
def generate_meta_info(save_dir, max_node, divide=40): aa_nas_bench_ss = get_search_spaces('cell', 'nas-bench-201') archs = CellStructure.gen_all(aa_nas_bench_ss, max_node, False) print ('There are {:} archs vs {:}.'.format(len(archs), len(aa_nas_bench_ss) ** ((max_node-1)*max_node/2))) random.seed( 88 ) # please do not change this line for reproducibility random.shuffle( archs ) # to test fixed-random shuffle #print ('arch [0] : {:}\n---->>>> {:}'.format( archs[0], archs[0].tostr() )) #print ('arch [9] : {:}\n---->>>> {:}'.format( archs[9], archs[9].tostr() )) assert archs[0 ].tostr() == '|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|', 'please check the 0-th architecture : {:}'.format(archs[0]) assert archs[9 ].tostr() == '|avg_pool_3x3~0|+|none~0|none~1|+|skip_connect~0|none~1|nor_conv_3x3~2|', 'please check the 9-th architecture : {:}'.format(archs[9]) assert archs[123].tostr() == '|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|none~0|avg_pool_3x3~1|nor_conv_3x3~2|', 'please check the 123-th architecture : {:}'.format(archs[123]) total_arch = len(archs) num = 50000 indexes_5W = list(range(num)) random.seed( 1021 ) random.shuffle( indexes_5W ) train_split = sorted( list(set(indexes_5W[:num//2])) ) valid_split = sorted( list(set(indexes_5W[num//2:])) ) assert len(train_split) + len(valid_split) == num assert train_split[0] == 0 and train_split[10] == 26 and train_split[111] == 203 and valid_split[0] == 1 and valid_split[10] == 18 and valid_split[111] == 242, '{:} {:} {:} - {:} {:} {:}'.format(train_split[0], train_split[10], train_split[111], valid_split[0], valid_split[10], valid_split[111]) splits = {num: {'train': train_split, 'valid': valid_split} } info = {'archs' : [x.tostr() for x in archs], 'total' : total_arch, 'max_node' : max_node, 'splits': splits} save_dir = Path(save_dir) save_dir.mkdir(parents=True, exist_ok=True) save_name = save_dir / 'meta-node-{:}.pth'.format(max_node) assert not save_name.exists(), '{:} already exist'.format(save_name) torch.save(info, save_name) print ('save the meta file into {:}'.format(save_name)) script_name_full = save_dir / 'BENCH-201-N{:}.opt-full.script'.format(max_node) script_name_less = save_dir / 'BENCH-201-N{:}.opt-less.script'.format(max_node) full_file = open(str(script_name_full), 'w') less_file = open(str(script_name_less), 'w') gaps = total_arch // divide for start in range(0, total_arch, gaps): xend = min(start+gaps, total_arch) full_file.write('bash ./scripts-search/NAS-Bench-201/train-models.sh 0 {:5d} {:5d} -1 \'777 888 999\'\n'.format(start, xend-1)) less_file.write('bash ./scripts-search/NAS-Bench-201/train-models.sh 1 {:5d} {:5d} -1 \'777 888 999\'\n'.format(start, xend-1)) print ('save the training script into {:} and {:}'.format(script_name_full, script_name_less)) full_file.close() less_file.close() script_name = save_dir / 'meta-node-{:}.cal-script.txt'.format(max_node) macro = 'OMP_NUM_THREADS=6 CUDA_VISIBLE_DEVICES=0' with open(str(script_name), 'w') as cfile: for start in range(0, total_arch, gaps): xend = min(start+gaps, total_arch) cfile.write('{:} python exps/NAS-Bench-201/statistics.py --mode cal --target_dir {:06d}-{:06d}-C16-N5\n'.format(macro, start, xend-1)) print ('save the post-processing script into {:}'.format(script_name))
def evaluate_one_shot(model, xloader, api, cal_mode, seed=111): weights = deepcopy(model.state_dict()) model.train(cal_mode) with torch.no_grad(): logits = nn.functional.log_softmax(model.arch_parameters, dim=-1) archs = CellStructure.gen_all(model.op_names, model.max_nodes, False) probs, accuracies, gt_accs_10_valid, gt_accs_10_test = [], [], [], [] loader_iter = iter(xloader) random.seed(seed) random.shuffle(archs) for idx, arch in enumerate(archs): arch_index = api.query_index_by_arch(arch) metrics = api.get_more_info(arch_index, 'cifar10-valid', None, False, False) gt_accs_10_valid.append(metrics['valid-accuracy']) metrics = api.get_more_info(arch_index, 'cifar10', None, False, False) gt_accs_10_test.append(metrics['test-accuracy']) select_logits = [] for i, node_info in enumerate(arch.nodes): for op, xin in node_info: node_str = '{:}<-{:}'.format(i + 1, xin) op_index = model.op_names.index(op) select_logits.append(logits[model.edge2index[node_str], op_index]) cur_prob = sum(select_logits).item() probs.append(cur_prob) cor_prob_valid = np.corrcoef(probs, gt_accs_10_valid)[0, 1] cor_prob_test = np.corrcoef(probs, gt_accs_10_test)[0, 1] print( '{:} correlation for probabilities : {:.6f} on CIFAR-10 validation and {:.6f} on CIFAR-10 test' .format(time_string(), cor_prob_valid, cor_prob_test)) for idx, arch in enumerate(archs): model.set_cal_mode('dynamic', arch) try: inputs, targets = next(loader_iter) except: loader_iter = iter(xloader) inputs, targets = next(loader_iter) _, logits = model(inputs.cuda()) _, preds = torch.max(logits, dim=-1) correct = (preds == targets.cuda()).float() accuracies.append(correct.mean().item()) if idx != 0 and (idx % 500 == 0 or idx + 1 == len(archs)): cor_accs_valid = np.corrcoef(accuracies, gt_accs_10_valid[:idx + 1])[0, 1] cor_accs_test = np.corrcoef(accuracies, gt_accs_10_test[:idx + 1])[0, 1] print( '{:} {:05d}/{:05d} mode={:5s}, correlation : accs={:.5f} for CIFAR-10 valid, {:.5f} for CIFAR-10 test.' .format(time_string(), idx, len(archs), 'Train' if cal_mode else 'Eval', cor_accs_valid, cor_accs_test)) model.load_state_dict(weights) return archs, probs, accuracies
def traverse_net(max_node): aa_nas_bench_ss = get_search_spaces('cell', 'nats-bench') archs = CellStructure.gen_all(aa_nas_bench_ss, max_node, False) print ('There are {:} archs vs {:}.'.format(len(archs), len(aa_nas_bench_ss) ** ((max_node-1)*max_node/2))) random.seed( 88 ) # please do not change this line for reproducibility random.shuffle( archs ) assert archs[0 ].tostr() == '|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|', 'please check the 0-th architecture : {:}'.format(archs[0]) assert archs[9 ].tostr() == '|avg_pool_3x3~0|+|none~0|none~1|+|skip_connect~0|none~1|nor_conv_3x3~2|', 'please check the 9-th architecture : {:}'.format(archs[9]) assert archs[123].tostr() == '|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|none~0|avg_pool_3x3~1|nor_conv_3x3~2|', 'please check the 123-th architecture : {:}'.format(archs[123]) return [x.tostr() for x in archs]
def generate_meta_info(save_dir, max_node, divide=40): aa_nas_bench_ss = get_search_spaces('cell', 'nas-bench-201') archs = CellStructure.gen_all(aa_nas_bench_ss, max_node, False) print('There are {:} archs vs {:}.'.format( len(archs), len(aa_nas_bench_ss)**((max_node - 1) * max_node / 2))) random.seed(88) # please do not change this line for reproducibility random.shuffle(archs) # to test fixed-random shuffle #print ('arch [0] : {:}\n---->>>> {:}'.format( archs[0], archs[0].tostr() )) #print ('arch [9] : {:}\n---->>>> {:}'.format( archs[9], archs[9].tostr() )) assert archs[0].tostr( ) == '|avg_pool_3x3~0|+|nor_conv_1x1~0|skip_connect~1|+|nor_conv_1x1~0|skip_connect~1|skip_connect~2|', 'please check the 0-th architecture : {:}'.format( archs[0]) assert archs[9].tostr( ) == '|avg_pool_3x3~0|+|none~0|none~1|+|skip_connect~0|none~1|nor_conv_3x3~2|', 'please check the 9-th architecture : {:}'.format( archs[9]) assert archs[123].tostr( ) == '|avg_pool_3x3~0|+|avg_pool_3x3~0|nor_conv_1x1~1|+|none~0|avg_pool_3x3~1|nor_conv_3x3~2|', 'please check the 123-th architecture : {:}'.format( archs[123]) total_arch = len(archs) num = 50000 indexes_5W = list(range(num)) random.seed(1021) random.shuffle(indexes_5W) train_split = sorted(list(set(indexes_5W[:num // 2]))) valid_split = sorted(list(set(indexes_5W[num // 2:]))) assert len(train_split) + len(valid_split) == num assert train_split[0] == 0 and train_split[10] == 26 and train_split[ 111] == 203 and valid_split[0] == 1 and valid_split[ 10] == 18 and valid_split[ 111] == 242, '{:} {:} {:} - {:} {:} {:}'.format( train_split[0], train_split[10], train_split[111], valid_split[0], valid_split[10], valid_split[111]) splits = {num: {'train': train_split, 'valid': valid_split}} info = { 'archs': [x.tostr() for x in archs], 'total': total_arch, 'max_node': max_node, 'splits': splits } save_dir = Path(save_dir) save_dir.mkdir(parents=True, exist_ok=True) save_name = save_dir / 'meta-node-{:}.pth'.format(max_node) assert not save_name.exists(), '{:} already exist'.format(save_name) torch.save(info, save_name) print('save the meta file into {:}'.format(save_name))