def eval_random_ws_model(config, model):
model_list = pickle.load(open(model, 'rb'))
accs = []
for model in model_list:
adjacency_matrix, node_list = model[0]
if int(config['search_space']) == int('1'):
adjacency_matrix = upscale_to_nasbench_format(adjacency_matrix)
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif int(config['search_space']) == int('2'):
adjacency_matrix = upscale_to_nasbench_format(adjacency_matrix)
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif int(config['search_space']) == int('3'):
node_list = [INPUT, *node_list, OUTPUT]
else:
raise ValueError('Unknown search space')
# Convert the adjacency matrix in format for nasbench
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
# Query nasbench
data = nasbench.query(model_spec)
valid_acc, test_acc = [], []
for item in data:
test_acc.append(item['test_accuracy'])
valid_acc.append(item['validation_accuracy'])
accs.append([np.mean(valid_acc), np.mean(test_acc)])
return accs
def correlation_between_one_shot_nb(model_path, config, epoch):
if config['search_space'] == '1':
search_space = SearchSpace1()
elif config['search_space'] == '2':
search_space = SearchSpace2()
elif config['search_space'] == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
model = DartsWrapper(save_path=model_path, seed=0, batch_size=128, grad_clip=5, epochs=200,
num_intermediate_nodes=search_space.num_intermediate_nodes, search_space=search_space,
cutout=False)
discrete = True
normalize = False
model.load(epoch=epoch)
controller = torch.load(os.path.join(model_path, 'controller_epoch_{}.pt'.format(epoch)))
nb_test_errors = {'4': [], '12': [], '36': [], '108': []}
nb_valid_errors = {'4': [], '12': [], '36': [], '108': []}
one_shot_test_errors = []
for idx in range(100):
(adjacency_matrix_ss, ops_ss), _, _ = controller()
print(adjacency_matrix_ss, ops_ss)
one_shot_test_error = model.evaluate_test((adjacency_matrix_ss, ops_ss), split='test', discrete=discrete,
normalize=normalize)
one_shot_test_errors.extend(np.repeat(one_shot_test_error, 3))
# # Query NASBench
# Create nested list from numpy matrix
if str(config['search_space']) == '1' or str(config['search_space']) == '2':
adjacency_matrix_ss = upscale_to_nasbench_format(adjacency_matrix_ss)
# Remove input, output and 5th node
ops_ss.append(CONV1X1)
nasbench_adjacency_matrix = adjacency_matrix_ss.astype(np.int).tolist()
ops_ss.insert(0, INPUT)
ops_ss.append(OUTPUT)
# Assemble the model spec
model_spec = api.ModelSpec(
# Adjacency matrix of the module
matrix=nasbench_adjacency_matrix,
# Operations at the vertices of the module, matches order of matrix
ops=ops_ss)
for nb_epoch_budget in [4, 12, 36, 108]:
data = nasbench.query(model_spec=model_spec, epochs=nb_epoch_budget)
nb_test_errors[str(nb_epoch_budget)].extend([1 - item['test_accuracy'] for item in data])
nb_valid_errors[str(nb_epoch_budget)].extend([1 - item['validation_accuracy'] for item in data])
# print('NB', nb_test_errors[-1], 'OS', one_shot_test_errors[-1], 'weights', model.model.arch_parameters())
# correlation = np.corrcoef(one_shot_test_errors, nb_test_errors)[0, -1]
return None, nb_test_errors, nb_valid_errors, one_shot_test_errors
def sample(self, with_loose_ends, upscale=True):
if with_loose_ends:
adjacency_matrix_sample = self._sample_adjacency_matrix_with_loose_ends()
else:
adjacency_matrix_sample = self._sample_adjacency_matrix_without_loose_ends(
adjacency_matrix=np.zeros([self.num_intermediate_nodes + 2, self.num_intermediate_nodes + 2]),
node=self.num_intermediate_nodes + 1)
assert self._check_validity_of_adjacency_matrix(adjacency_matrix_sample), 'Incorrect graph'
if upscale and self.search_space_number in [1, 2]:
adjacency_matrix_sample = upscale_to_nasbench_format(adjacency_matrix_sample)
return adjacency_matrix_sample, random.choices(PRIMITIVES, k=self.num_intermediate_nodes)
def train_and_eval(config):
adjacency_matrix, node_list = config.adjacency_matrix, config.node_list
if type(search_space) == SearchSpace1 or type(
search_space) == SearchSpace2:
# Fill up adjacency matrix and node list with entries for unused nodes
adjacency_matrix = upscale_to_nasbench_format(adjacency_matrix)
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
else:
node_list = [INPUT, *node_list, OUTPUT]
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
nasbench_data = nasbench.query(model_spec)
return nasbench_data['validation_accuracy'], nasbench_data['training_time']
def generate_adjacency_matrix_without_loose_ends(self):
for adjacency_matrix in self._generate_adjacency_matrix(
adjacency_matrix=np.zeros([6, 6]), node=OUTPUT_NODE - 1):
yield upscale_to_nasbench_format(adjacency_matrix)
def create_nasbench_adjacency_matrix_with_loose_ends(self, parents):
return upscale_to_nasbench_format(
self._create_adjacency_matrix_with_loose_ends(parents))
def create_nasbench_adjacency_matrix(self, parents, **kwargs):
adjacency_matrix = self._create_adjacency_matrix(
parents, adjacency_matrix=np.zeros([6, 6]), node=OUTPUT_NODE - 1)
# Create nasbench compatible adjacency matrix
return upscale_to_nasbench_format(adjacency_matrix)
