def add_dummy_node(matrix_in, ops_in):
# {1, 2, 3, 4, 5, 6, 7}
matrix = np.zeros((NUM_VERTICES, NUM_VERTICES), dtype=np.int8)
for i in range(matrix_in.shape[0]):
idxs = np.where(matrix_in[i] == 1)
for id in idxs[0]:
if id == matrix_in.shape[0] - 1:
matrix[i, NUM_VERTICES-1] = 1
else:
matrix[i, id] = 1
ops = ops_in[:(matrix_in.shape[0] - 1)] + ['isolate'] * (NUM_VERTICES - matrix_in.shape[0]) + ops_in[-1:]
find_isolate_node(matrix)
return matrix, ops
def matrix_dummy_nodes(self, matrix_in, ops_in):
# {2, 3, 4, 5, 6, 7}
matrix = np.zeros((NUM_VERTICES, NUM_VERTICES))
for i in range(matrix_in.shape[0]):
idxs = np.where(matrix_in[i] == 1)
for id in idxs[0]:
if id == matrix_in.shape[0] - 1:
matrix[i, 6] = 1
else:
matrix[i, id] = 1
ops = ops_in[:(matrix_in.shape[0]-1)] + ['isolate'] * (7-matrix_in.shape[0]) + ops_in[-1:]
find_isolate_node(matrix)
return matrix, ops
def mutate_gvae(self, nasbench, mutation_rate=1.0):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
while True:
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
edge_mutation_prob = mutation_rate / NUM_VERTICES
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
op_mutation_prob = mutation_rate / OP_SPOTS
for ind in range(1, OP_SPOTS + 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
isolate_nodes = find_isolate_node(new_matrix)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
return {
'matrix': new_matrix,
'ops': new_ops,
'isolate_node_idxs': isolate_nodes
}
def get_clean_dummy_arch(self):
total_arch = {}
total_keys = [k for k in self.nasbench.computed_statistics]
best_key = None
best_val = 0
for k in total_keys:
val_acc = []
test_acc = []
arch_matrix = self.nasbench.fixed_statistics[k]['module_adjacency']
arch_ops = self.nasbench.fixed_statistics[k]['module_operations']
if arch_matrix.shape[0] < 7:
matrix, ops = self.matrix_dummy_nodes(arch_matrix, arch_ops)
else:
matrix = arch_matrix
ops = arch_ops
spec = api.ModelSpec(matrix=arch_matrix, ops=arch_ops)
if arch_matrix.shape[0] == 7:
isolate_list = find_isolate_node(arch_matrix)
if len(isolate_list) >= 1:
print(arch_matrix)
print(isolate_list)
if not self.nasbench.is_valid(spec):
continue
for i in range(3):
val_acc.append(self.nasbench.computed_statistics[k][108][i]
['final_validation_accuracy'])
test_acc.append(self.nasbench.computed_statistics[k][108][i]
['final_test_accuracy'])
val_mean = float(np.mean(val_acc))
test_mean = float(np.mean(test_acc))
if best_val < val_mean:
best_val = val_mean
best_key = k
total_arch[k] = {
# 'o_matrix': arch_matrix.astype(np.float32),
'o_matrix': arch_matrix,
'o_ops': arch_ops,
# 'matrix': matrix.astype(np.float32),
'matrix': matrix,
'ops': ops,
'val': val_mean,
'test': test_mean,
'key': k
}
best_arch = total_arch[best_key]
print(best_arch['val'], best_arch['test'])
return total_arch, total_keys
def get_clean_dummy_arch(self):
total_arch = {}
total_keys = [k for k in self.nasbench.computed_statistics]
for k in total_keys:
val_acc = []
test_acc = []
arch_matrix = self.nasbench.fixed_statistics[k]['module_adjacency']
arch_ops = self.nasbench.fixed_statistics[k]['module_operations']
if arch_matrix.shape[0] < 7:
matrix, ops = self.matrix_dummy_nodes(arch_matrix, arch_ops)
else:
matrix = arch_matrix.astype(np.int16)
ops = arch_ops
spec = api.ModelSpec(matrix=arch_matrix, ops=arch_ops)
if arch_matrix.shape[0] == 7:
isolate_list = find_isolate_node(arch_matrix)
if len(isolate_list) >= 1:
print(arch_matrix)
print(isolate_list)
if not self.nasbench.is_valid(spec):
continue
for i in range(3):
val_acc.append(self.nasbench.computed_statistics[k][108][i]
['final_validation_accuracy'])
test_acc.append(self.nasbench.computed_statistics[k][108][i]
['final_test_accuracy'])
val_mean = float(np.mean(val_acc))
test_mean = float(np.mean(test_acc))
path_indices = Cell(matrix=matrix,
ops=ops).encode_paths_seq_aware(length=120)
total_arch[k] = {
# 'o_matrix': arch_matrix.astype(np.float32),
'o_matrix': arch_matrix,
'o_ops': arch_ops,
# 'matrix': matrix.astype(np.float32),
'matrix': matrix,
'ops': ops,
'val': val_mean,
'test': test_mean,
'key': k,
'path_indices': path_indices
}
return total_arch, total_keys
def random_cell_gnn(cls, nasbench):
"""
From the NASBench repository
https://github.com/google-research/nasbench
"""
while True:
matrix = np.random.choice(
[0, 1], size=(NUM_VERTICES, NUM_VERTICES))
matrix = np.triu(matrix, 1)
isolate_nodes = find_isolate_node(matrix)
ops = np.random.choice(OPS, size=NUM_VERTICES).tolist()
ops[0] = INPUT
ops[-1] = OUTPUT
spec = api.ModelSpec(matrix=matrix, ops=ops)
if nasbench.is_valid(spec):
return {
'matrix': matrix,
'ops': ops,
'isolate_node_idxs': isolate_nodes
}
def mutate_edit_distance(self, nasbench, edit_dist, candidate_num, data):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
arch_list = []
new_ops = copy.deepcopy(self.ops)
edges = [(src, dst) for src in range(0, NUM_VERTICES - 1) for dst in range(src+1, NUM_VERTICES)]
op_available_tuple = []
for ind in range(1, OP_SPOTS + 1):
available = [o for o in OPS if o != new_ops[ind]]
for o in available:
op_available_tuple.append((ind, o))
idx_list = self.generate_edit_compose(edges, op_available_tuple, edit_dist)
random.shuffle(idx_list)
for edit_idx in idx_list:
if edit_dist > 1 and len(arch_list) >= candidate_num:
break
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
for j in edit_idx:
if j >= len(edges):
nest_idx = op_available_tuple[j - len(edges)]
new_ops[nest_idx[0]] = nest_idx[1]
else:
edge_conn = edges[j]
new_matrix[edge_conn[0], edge_conn[1]] = 1 - new_matrix[edge_conn[0], edge_conn[1]]
isolate_nodes = find_isolate_node(new_matrix)
new_spec = api.ModelSpec(new_matrix, new_ops)
flag = self.verify_correctness((new_matrix, new_ops), data, edit_dist)
if nasbench.is_valid(new_spec) and flag:
arch_list.append({
'matrix': new_matrix,
'ops': new_ops,
'isolate_node_idxs': isolate_nodes
})
return arch_list
def get_all_path_encooding(self):
total_arch_bananas_dict = {}
total_keys = [k for k in self.nasbench.computed_statistics]
for k in total_keys:
arch_matrix = self.nasbench.fixed_statistics[k]['module_adjacency']
arch_ops = self.nasbench.fixed_statistics[k]['module_operations']
if arch_matrix.shape[0] < 7:
matrix, ops = self.matrix_dummy_nodes(arch_matrix, arch_ops)
else:
matrix = arch_matrix
ops = arch_ops
spec = api.ModelSpec(matrix=arch_matrix, ops=arch_ops)
if arch_matrix.shape[0] == 7:
isolate_list = find_isolate_node(arch_matrix)
if len(isolate_list) >= 1:
print(arch_matrix)
print(isolate_list)
if not self.nasbench.is_valid(spec):
continue
arch = {'matrix': matrix, 'ops': ops}
encoding = Cell(**arch).encode_paths()
total_arch_bananas_dict[k] = encoding
return total_arch_bananas_dict
def get_candidates_gin(self,
data,
num=100,
acq_opt_type='mutation',
encode_paths=True,
allow_isomorphisms=False,
patience_factor=5,
deterministic_loss=True,
num_best_arches=10):
"""
Creates a set of candidate architectures with mutated and/or random architectures
"""
# test for isomorphisms using a hash map of path indices
candidates = []
dic = {}
for d in data:
arch = d[0]
path_indices = self.get_path_indices(arch)
dic[path_indices] = 1
if acq_opt_type in ['mutation', 'mutation_random']:
# mutate architectures with the lowest validation error
best_arches = [
arch[0]
for arch in sorted(data, key=lambda i: i[2])[:num_best_arches *
patience_factor]
]
# stop when candidates is size num
# use patience_factor instead of a while loop to avoid long or infinite runtime
for arch in best_arches:
if len(candidates) >= num:
break
for i in range(num):
mutated = self.mutate_arch(arch)
mutated_matrix, mutated_ops = mutated['matrix'], mutated[
'ops']
isolate_nodes = find_isolate_node(mutated_matrix)
mutated['matrix'] = mutated_matrix
mutated['isolate_node_idxs'] = isolate_nodes
archtuple = self.query_arch_gin(
mutated,
train=False,
deterministic=deterministic_loss,
encode_paths=encode_paths)
path_indices = self.get_path_indices(mutated)
if allow_isomorphisms or path_indices not in dic:
dic[path_indices] = 1
candidates.append(archtuple)
if acq_opt_type in ['random', 'mutation_random']:
for _ in range(num * patience_factor):
if len(candidates) >= 2 * num:
break
archtuple = self.query_arch(train=False,
deterministic=deterministic_loss,
encode_paths=encode_paths)
path_indices = self.get_path_indices(archtuple[0])
if allow_isomorphisms or path_indices not in dic:
dic[path_indices] = 1
candidates.append(archtuple)
return candidates[:num]
