def main(args):
file_name = 'log_%s_%d' % ('gpus', args.gpu)
logger = setup_logger(file_name,
args.save_dir,
args.gpu,
log_level='DEBUG',
filename='%s.txt' % file_name)
logger.info(args)
if args.search_space == 'darts':
with open(args.darts_file_path, 'rb') as f:
if args.darts_training_nums:
all_data = pickle.load(f)[:args.darts_training_nums]
else:
all_data = pickle.load(f)
else:
nasbench_datas = data.build_datasets(args)
all_data = data.dataset_all(args, nasbench_datas)
for predictor in args.predictor_list:
logger.info(
f'================== predictor type: {predictor} ======================'
)
predictor_unsupervised(args,
predictor,
all_data,
train_epochs=args.epochs,
logger=logger)
def generate_nasbench_101_all_datas(nasbench_data, args):
all_data = data.dataset_all(args, nasbench_data)
save_path = os.path.join(nas_bench_101_base_path, 'all_data_new.pkl')
with open(save_path, 'wb') as fb:
pickle.dump(all_data, fb)
parser.add_argument(
'--dataname',
type=str,
default='cifar10-valid',
choices=['cifar10-valid', 'cifar10', 'cifar100', 'ImageNet16-120'],
help='The evaluation of dataset of NASBench-201.')
args = parser.parse_args()
if args.search_space == 'nasbench_101':
args.seq_len = 120
with open(nas_bench_101_all_data, 'rb') as fpkl:
all_data = pickle.load(fpkl)
elif args.search_space == 'nasbench_201':
args.seq_len = 96 # 5461
nasbench_datas = data.build_datasets(args)
all_data = data.dataset_all(args, nasbench_datas)
else:
raise NotImplementedError(
'This search space does not support at present!')
path_based_encoding = [
tuple(map(int, d['pe_path_enc_vec'].tolist())) for d in all_data
]
path_based_position_aware_encoding = [
tuple(map(int, d['pe_path_enc_aware_vec'].tolist())) for d in all_data
]
print('path_based encoding', len(path_based_encoding),
len(set(path_based_encoding)))
print('position_aware_path_based_encoding',
len(path_based_position_aware_encoding),
len(set(path_based_position_aware_encoding)))
def main(args):
file_name = 'log_%s_%d' % ('gpus', args.gpu)
logger = setup_logger(file_name,
args.save_dir,
args.gpu,
log_level='DEBUG',
filename='%s.txt' % file_name)
logger.info(args)
if args.search_space == 'nasbench_101':
with open(nas_bench_101_all_data, 'rb') as fpkl:
all_data = pickle.load(fpkl)
elif args.search_space == 'nasbench_201':
nasbench_datas = data.build_datasets(args)
all_data = data.dataset_all(args, nasbench_datas)
elif args.search_space == 'darts':
with open(darts_converted_with_label, 'rb') as fb:
all_data = pickle.load(fb)
else:
raise NotImplementedError(
f'The search space {args.search_space} does not support now!')
for k in range(args.trails):
seed = random_id_int(4)
set_random_seed(seed)
s_results_dict = defaultdict(list)
k_results_dict = defaultdict(list)
duration_dict = defaultdict(list)
logger.info(
f'====================== Trails {k} Begin Setting Seed to {seed} ==========================='
)
for budget in args.search_budget:
train_data, test_data = data.dataset_split_idx_predictive_comparison(
all_data, budget)
print(
f'budget: {budget}, train data size: {len(train_data)}, test data size: {len(test_data)}'
)
if args.compare_supervised == 'T':
logger.info(
f'==== predictor type: SUPERVISED, load pretrain model False, '
f'search budget is {budget}. Training epoch is {args.epochs} ===='
)
spearman_corr, kendalltau_corr, duration = predictor_comparision(
args,
'SS_RL',
train_data,
test_data,
flag=False,
train_epochs=args.epochs,
logger=logger)
if math.isnan(spearman_corr):
spearman_corr = 0
if math.isnan(kendalltau_corr):
kendalltau_corr = 0
s_results_dict[f'supervised#{budget}#{args.epochs}'].append(
spearman_corr)
k_results_dict[f'supervised#{budget}#{args.epochs}'].append(
kendalltau_corr)
duration_dict[f'supervised#{budget}#{args.epochs}'].append(
duration)
for predictor_type, dir in zip(args.predictor_list, args.load_dir):
logger.info(
f'==== predictor type: {predictor_type}, load pretrain model True. '
f'Search budget is {budget}. Training epoch is {args.epochs}. '
f'The model save dir is {dir.split("/")[-1][:-3]} ====')
spearman_corr, kendalltau_corr, duration = predictor_comparision(
args,
predictor_type,
train_data,
test_data,
flag=True,
load_dir=dir,
train_epochs=args.epochs,
logger=logger)
if math.isnan(spearman_corr):
spearman_corr = 0
if math.isnan(kendalltau_corr):
kendalltau_corr = 0
s_results_dict[predictor_type + '#' + str(budget) + '#' +
str(args.epochs)].append(spearman_corr)
k_results_dict[predictor_type + '#' + str(budget) + '#' +
str(args.epochs)].append(kendalltau_corr)
duration_dict[predictor_type + '#' + str(budget) + '#' +
str(args.epochs)].append(duration)
file_id = random_id(6)
save_path = os.path.join(
args.save_dir,
f'{file_id}_{args.predictor_list[0]}_{args.search_space.split("_")[-1]}_{args.gpu}_{k}.pkl'
)
with open(save_path, 'wb') as fp:
pickle.dump(s_results_dict, fp)
pickle.dump(k_results_dict, fp)
pickle.dump(duration_dict, fp)