def main(args): file_name = 'log_%s_%d' % ('gpus', args.gpu) logger = setup_logger(file_name, args.save_dir, args.gpu, log_level='DEBUG', filename='%s.txt' % file_name) logger.info(args) if args.search_space == 'darts': with open(args.darts_file_path, 'rb') as f: if args.darts_training_nums: all_data = pickle.load(f)[:args.darts_training_nums] else: all_data = pickle.load(f) else: nasbench_datas = data.build_datasets(args) all_data = data.dataset_all(args, nasbench_datas) for predictor in args.predictor_list: logger.info( f'================== predictor type: {predictor} ======================' ) predictor_unsupervised(args, predictor, all_data, train_epochs=args.epochs, logger=logger)
def generate_nasbench_101_all_datas(nasbench_data, args): all_data = data.dataset_all(args, nasbench_data) save_path = os.path.join(nas_bench_101_base_path, 'all_data_new.pkl') with open(save_path, 'wb') as fb: pickle.dump(all_data, fb)
parser.add_argument( '--dataname', type=str, default='cifar10-valid', choices=['cifar10-valid', 'cifar10', 'cifar100', 'ImageNet16-120'], help='The evaluation of dataset of NASBench-201.') args = parser.parse_args() if args.search_space == 'nasbench_101': args.seq_len = 120 with open(nas_bench_101_all_data, 'rb') as fpkl: all_data = pickle.load(fpkl) elif args.search_space == 'nasbench_201': args.seq_len = 96 # 5461 nasbench_datas = data.build_datasets(args) all_data = data.dataset_all(args, nasbench_datas) else: raise NotImplementedError( 'This search space does not support at present!') path_based_encoding = [ tuple(map(int, d['pe_path_enc_vec'].tolist())) for d in all_data ] path_based_position_aware_encoding = [ tuple(map(int, d['pe_path_enc_aware_vec'].tolist())) for d in all_data ] print('path_based encoding', len(path_based_encoding), len(set(path_based_encoding))) print('position_aware_path_based_encoding', len(path_based_position_aware_encoding), len(set(path_based_position_aware_encoding)))
def main(args): file_name = 'log_%s_%d' % ('gpus', args.gpu) logger = setup_logger(file_name, args.save_dir, args.gpu, log_level='DEBUG', filename='%s.txt' % file_name) logger.info(args) if args.search_space == 'nasbench_101': with open(nas_bench_101_all_data, 'rb') as fpkl: all_data = pickle.load(fpkl) elif args.search_space == 'nasbench_201': nasbench_datas = data.build_datasets(args) all_data = data.dataset_all(args, nasbench_datas) elif args.search_space == 'darts': with open(darts_converted_with_label, 'rb') as fb: all_data = pickle.load(fb) else: raise NotImplementedError( f'The search space {args.search_space} does not support now!') for k in range(args.trails): seed = random_id_int(4) set_random_seed(seed) s_results_dict = defaultdict(list) k_results_dict = defaultdict(list) duration_dict = defaultdict(list) logger.info( f'====================== Trails {k} Begin Setting Seed to {seed} ===========================' ) for budget in args.search_budget: train_data, test_data = data.dataset_split_idx_predictive_comparison( all_data, budget) print( f'budget: {budget}, train data size: {len(train_data)}, test data size: {len(test_data)}' ) if args.compare_supervised == 'T': logger.info( f'==== predictor type: SUPERVISED, load pretrain model False, ' f'search budget is {budget}. Training epoch is {args.epochs} ====' ) spearman_corr, kendalltau_corr, duration = predictor_comparision( args, 'SS_RL', train_data, test_data, flag=False, train_epochs=args.epochs, logger=logger) if math.isnan(spearman_corr): spearman_corr = 0 if math.isnan(kendalltau_corr): kendalltau_corr = 0 s_results_dict[f'supervised#{budget}#{args.epochs}'].append( spearman_corr) k_results_dict[f'supervised#{budget}#{args.epochs}'].append( kendalltau_corr) duration_dict[f'supervised#{budget}#{args.epochs}'].append( duration) for predictor_type, dir in zip(args.predictor_list, args.load_dir): logger.info( f'==== predictor type: {predictor_type}, load pretrain model True. ' f'Search budget is {budget}. Training epoch is {args.epochs}. ' f'The model save dir is {dir.split("/")[-1][:-3]} ====') spearman_corr, kendalltau_corr, duration = predictor_comparision( args, predictor_type, train_data, test_data, flag=True, load_dir=dir, train_epochs=args.epochs, logger=logger) if math.isnan(spearman_corr): spearman_corr = 0 if math.isnan(kendalltau_corr): kendalltau_corr = 0 s_results_dict[predictor_type + '#' + str(budget) + '#' + str(args.epochs)].append(spearman_corr) k_results_dict[predictor_type + '#' + str(budget) + '#' + str(args.epochs)].append(kendalltau_corr) duration_dict[predictor_type + '#' + str(budget) + '#' + str(args.epochs)].append(duration) file_id = random_id(6) save_path = os.path.join( args.save_dir, f'{file_id}_{args.predictor_list[0]}_{args.search_space.split("_")[-1]}_{args.gpu}_{k}.pkl' ) with open(save_path, 'wb') as fp: pickle.dump(s_results_dict, fp) pickle.dump(k_results_dict, fp) pickle.dump(duration_dict, fp)