def correlation_between_one_shot_nb(model_path, config, epoch):
if config['search_space'] == '1':
search_space = SearchSpace1()
elif config['search_space'] == '2':
search_space = SearchSpace2()
elif config['search_space'] == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
model = DartsWrapper(
save_path=model_path,
seed=0,
batch_size=128,
grad_clip=5,
epochs=200,
num_intermediate_nodes=search_space.num_intermediate_nodes,
search_space=search_space,
cutout=False)
if 'random_ws' in model_path:
discrete = True
normalize = False
else:
discrete = False
normalize = True
model.load(epoch=epoch)
nb_test_errors = []
nb_valid_errors = []
one_shot_test_errors = []
for adjacency_matrix, ops, model_spec in search_space.generate_search_space_without_loose_ends(
):
if str(config['search_space']) == '1' or str(
config['search_space']) == '2':
adjacency_matrix_ss = np.delete(np.delete(adjacency_matrix, -2, 0),
-2, 0)
# Remove input, output and 5th node
ops_ss = ops[1:-2]
elif str(config['search_space']) == '3':
adjacency_matrix_ss = adjacency_matrix
# Remove input and output node
ops_ss = ops[1:-1]
else:
raise ValueError('Unknown search space')
one_shot_test_error = model.evaluate_test(
(adjacency_matrix_ss, ops_ss),
split='test',
discrete=discrete,
normalize=normalize)
one_shot_test_errors.extend(np.repeat(one_shot_test_error, 3))
# Query NASBench
data = nasbench.query(model_spec)
nb_test_errors.extend([1 - item['test_accuracy'] for item in data])
nb_valid_errors.extend(
[1 - item['validation_accuracy'] for item in data])
print('NB', nb_test_errors[-1], 'OS', one_shot_test_errors[-1],
'weights', model.model.arch_parameters())
correlation = np.corrcoef(one_shot_test_errors, nb_test_errors)[0, -1]
return correlation, nb_test_errors, nb_valid_errors, one_shot_test_errors
def correlation_between_one_shot_nb(model_path, config, epoch):
if config['search_space'] == '1':
search_space = SearchSpace1()
elif config['search_space'] == '2':
search_space = SearchSpace2()
elif config['search_space'] == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
model = DartsWrapper(save_path=model_path, seed=0, batch_size=128, grad_clip=5, epochs=200,
num_intermediate_nodes=search_space.num_intermediate_nodes, search_space=search_space,
cutout=False)
discrete = True
normalize = False
model.load(epoch=epoch)
controller = torch.load(os.path.join(model_path, 'controller_epoch_{}.pt'.format(epoch)))
nb_test_errors = {'4': [], '12': [], '36': [], '108': []}
nb_valid_errors = {'4': [], '12': [], '36': [], '108': []}
one_shot_test_errors = []
for idx in range(100):
(adjacency_matrix_ss, ops_ss), _, _ = controller()
print(adjacency_matrix_ss, ops_ss)
one_shot_test_error = model.evaluate_test((adjacency_matrix_ss, ops_ss), split='test', discrete=discrete,
normalize=normalize)
one_shot_test_errors.extend(np.repeat(one_shot_test_error, 3))
# # Query NASBench
# Create nested list from numpy matrix
if str(config['search_space']) == '1' or str(config['search_space']) == '2':
adjacency_matrix_ss = upscale_to_nasbench_format(adjacency_matrix_ss)
# Remove input, output and 5th node
ops_ss.append(CONV1X1)
nasbench_adjacency_matrix = adjacency_matrix_ss.astype(np.int).tolist()
ops_ss.insert(0, INPUT)
ops_ss.append(OUTPUT)
# Assemble the model spec
model_spec = api.ModelSpec(
# Adjacency matrix of the module
matrix=nasbench_adjacency_matrix,
# Operations at the vertices of the module, matches order of matrix
ops=ops_ss)
for nb_epoch_budget in [4, 12, 36, 108]:
data = nasbench.query(model_spec=model_spec, epochs=nb_epoch_budget)
nb_test_errors[str(nb_epoch_budget)].extend([1 - item['test_accuracy'] for item in data])
nb_valid_errors[str(nb_epoch_budget)].extend([1 - item['validation_accuracy'] for item in data])
# print('NB', nb_test_errors[-1], 'OS', one_shot_test_errors[-1], 'weights', model.model.arch_parameters())
# correlation = np.corrcoef(one_shot_test_errors, nb_test_errors)[0, -1]
return None, nb_test_errors, nb_valid_errors, one_shot_test_errors