def add_dummy_node(matrix_in, ops_in): # {1, 2, 3, 4, 5, 6, 7} matrix = np.zeros((NUM_VERTICES, NUM_VERTICES), dtype=np.int8) for i in range(matrix_in.shape[0]): idxs = np.where(matrix_in[i] == 1) for id in idxs[0]: if id == matrix_in.shape[0] - 1: matrix[i, NUM_VERTICES-1] = 1 else: matrix[i, id] = 1 ops = ops_in[:(matrix_in.shape[0] - 1)] + ['isolate'] * (NUM_VERTICES - matrix_in.shape[0]) + ops_in[-1:] find_isolate_node(matrix) return matrix, ops
def matrix_dummy_nodes(self, matrix_in, ops_in): # {2, 3, 4, 5, 6, 7} matrix = np.zeros((NUM_VERTICES, NUM_VERTICES)) for i in range(matrix_in.shape[0]): idxs = np.where(matrix_in[i] == 1) for id in idxs[0]: if id == matrix_in.shape[0] - 1: matrix[i, 6] = 1 else: matrix[i, id] = 1 ops = ops_in[:(matrix_in.shape[0]-1)] + ['isolate'] * (7-matrix_in.shape[0]) + ops_in[-1:] find_isolate_node(matrix) return matrix, ops
def mutate_gvae(self, nasbench, mutation_rate=1.0): """ similar to perturb. A stochastic approach to perturbing the cell inspird by https://github.com/google-research/nasbench """ while True: new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) edge_mutation_prob = mutation_rate / NUM_VERTICES for src in range(0, NUM_VERTICES - 1): for dst in range(src + 1, NUM_VERTICES): if random.random() < edge_mutation_prob: new_matrix[src, dst] = 1 - new_matrix[src, dst] op_mutation_prob = mutation_rate / OP_SPOTS for ind in range(1, OP_SPOTS + 1): if random.random() < op_mutation_prob: available = [o for o in OPS if o != new_ops[ind]] new_ops[ind] = random.choice(available) isolate_nodes = find_isolate_node(new_matrix) new_spec = api.ModelSpec(new_matrix, new_ops) if nasbench.is_valid(new_spec): return { 'matrix': new_matrix, 'ops': new_ops, 'isolate_node_idxs': isolate_nodes }
def get_clean_dummy_arch(self): total_arch = {} total_keys = [k for k in self.nasbench.computed_statistics] best_key = None best_val = 0 for k in total_keys: val_acc = [] test_acc = [] arch_matrix = self.nasbench.fixed_statistics[k]['module_adjacency'] arch_ops = self.nasbench.fixed_statistics[k]['module_operations'] if arch_matrix.shape[0] < 7: matrix, ops = self.matrix_dummy_nodes(arch_matrix, arch_ops) else: matrix = arch_matrix ops = arch_ops spec = api.ModelSpec(matrix=arch_matrix, ops=arch_ops) if arch_matrix.shape[0] == 7: isolate_list = find_isolate_node(arch_matrix) if len(isolate_list) >= 1: print(arch_matrix) print(isolate_list) if not self.nasbench.is_valid(spec): continue for i in range(3): val_acc.append(self.nasbench.computed_statistics[k][108][i] ['final_validation_accuracy']) test_acc.append(self.nasbench.computed_statistics[k][108][i] ['final_test_accuracy']) val_mean = float(np.mean(val_acc)) test_mean = float(np.mean(test_acc)) if best_val < val_mean: best_val = val_mean best_key = k total_arch[k] = { # 'o_matrix': arch_matrix.astype(np.float32), 'o_matrix': arch_matrix, 'o_ops': arch_ops, # 'matrix': matrix.astype(np.float32), 'matrix': matrix, 'ops': ops, 'val': val_mean, 'test': test_mean, 'key': k } best_arch = total_arch[best_key] print(best_arch['val'], best_arch['test']) return total_arch, total_keys
def get_clean_dummy_arch(self): total_arch = {} total_keys = [k for k in self.nasbench.computed_statistics] for k in total_keys: val_acc = [] test_acc = [] arch_matrix = self.nasbench.fixed_statistics[k]['module_adjacency'] arch_ops = self.nasbench.fixed_statistics[k]['module_operations'] if arch_matrix.shape[0] < 7: matrix, ops = self.matrix_dummy_nodes(arch_matrix, arch_ops) else: matrix = arch_matrix.astype(np.int16) ops = arch_ops spec = api.ModelSpec(matrix=arch_matrix, ops=arch_ops) if arch_matrix.shape[0] == 7: isolate_list = find_isolate_node(arch_matrix) if len(isolate_list) >= 1: print(arch_matrix) print(isolate_list) if not self.nasbench.is_valid(spec): continue for i in range(3): val_acc.append(self.nasbench.computed_statistics[k][108][i] ['final_validation_accuracy']) test_acc.append(self.nasbench.computed_statistics[k][108][i] ['final_test_accuracy']) val_mean = float(np.mean(val_acc)) test_mean = float(np.mean(test_acc)) path_indices = Cell(matrix=matrix, ops=ops).encode_paths_seq_aware(length=120) total_arch[k] = { # 'o_matrix': arch_matrix.astype(np.float32), 'o_matrix': arch_matrix, 'o_ops': arch_ops, # 'matrix': matrix.astype(np.float32), 'matrix': matrix, 'ops': ops, 'val': val_mean, 'test': test_mean, 'key': k, 'path_indices': path_indices } return total_arch, total_keys
def random_cell_gnn(cls, nasbench): """ From the NASBench repository https://github.com/google-research/nasbench """ while True: matrix = np.random.choice( [0, 1], size=(NUM_VERTICES, NUM_VERTICES)) matrix = np.triu(matrix, 1) isolate_nodes = find_isolate_node(matrix) ops = np.random.choice(OPS, size=NUM_VERTICES).tolist() ops[0] = INPUT ops[-1] = OUTPUT spec = api.ModelSpec(matrix=matrix, ops=ops) if nasbench.is_valid(spec): return { 'matrix': matrix, 'ops': ops, 'isolate_node_idxs': isolate_nodes }
def mutate_edit_distance(self, nasbench, edit_dist, candidate_num, data): """ similar to perturb. A stochastic approach to perturbing the cell inspird by https://github.com/google-research/nasbench """ arch_list = [] new_ops = copy.deepcopy(self.ops) edges = [(src, dst) for src in range(0, NUM_VERTICES - 1) for dst in range(src+1, NUM_VERTICES)] op_available_tuple = [] for ind in range(1, OP_SPOTS + 1): available = [o for o in OPS if o != new_ops[ind]] for o in available: op_available_tuple.append((ind, o)) idx_list = self.generate_edit_compose(edges, op_available_tuple, edit_dist) random.shuffle(idx_list) for edit_idx in idx_list: if edit_dist > 1 and len(arch_list) >= candidate_num: break new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) for j in edit_idx: if j >= len(edges): nest_idx = op_available_tuple[j - len(edges)] new_ops[nest_idx[0]] = nest_idx[1] else: edge_conn = edges[j] new_matrix[edge_conn[0], edge_conn[1]] = 1 - new_matrix[edge_conn[0], edge_conn[1]] isolate_nodes = find_isolate_node(new_matrix) new_spec = api.ModelSpec(new_matrix, new_ops) flag = self.verify_correctness((new_matrix, new_ops), data, edit_dist) if nasbench.is_valid(new_spec) and flag: arch_list.append({ 'matrix': new_matrix, 'ops': new_ops, 'isolate_node_idxs': isolate_nodes }) return arch_list
def get_all_path_encooding(self): total_arch_bananas_dict = {} total_keys = [k for k in self.nasbench.computed_statistics] for k in total_keys: arch_matrix = self.nasbench.fixed_statistics[k]['module_adjacency'] arch_ops = self.nasbench.fixed_statistics[k]['module_operations'] if arch_matrix.shape[0] < 7: matrix, ops = self.matrix_dummy_nodes(arch_matrix, arch_ops) else: matrix = arch_matrix ops = arch_ops spec = api.ModelSpec(matrix=arch_matrix, ops=arch_ops) if arch_matrix.shape[0] == 7: isolate_list = find_isolate_node(arch_matrix) if len(isolate_list) >= 1: print(arch_matrix) print(isolate_list) if not self.nasbench.is_valid(spec): continue arch = {'matrix': matrix, 'ops': ops} encoding = Cell(**arch).encode_paths() total_arch_bananas_dict[k] = encoding return total_arch_bananas_dict
def get_candidates_gin(self, data, num=100, acq_opt_type='mutation', encode_paths=True, allow_isomorphisms=False, patience_factor=5, deterministic_loss=True, num_best_arches=10): """ Creates a set of candidate architectures with mutated and/or random architectures """ # test for isomorphisms using a hash map of path indices candidates = [] dic = {} for d in data: arch = d[0] path_indices = self.get_path_indices(arch) dic[path_indices] = 1 if acq_opt_type in ['mutation', 'mutation_random']: # mutate architectures with the lowest validation error best_arches = [ arch[0] for arch in sorted(data, key=lambda i: i[2])[:num_best_arches * patience_factor] ] # stop when candidates is size num # use patience_factor instead of a while loop to avoid long or infinite runtime for arch in best_arches: if len(candidates) >= num: break for i in range(num): mutated = self.mutate_arch(arch) mutated_matrix, mutated_ops = mutated['matrix'], mutated[ 'ops'] isolate_nodes = find_isolate_node(mutated_matrix) mutated['matrix'] = mutated_matrix mutated['isolate_node_idxs'] = isolate_nodes archtuple = self.query_arch_gin( mutated, train=False, deterministic=deterministic_loss, encode_paths=encode_paths) path_indices = self.get_path_indices(mutated) if allow_isomorphisms or path_indices not in dic: dic[path_indices] = 1 candidates.append(archtuple) if acq_opt_type in ['random', 'mutation_random']: for _ in range(num * patience_factor): if len(candidates) >= 2 * num: break archtuple = self.query_arch(train=False, deterministic=deterministic_loss, encode_paths=encode_paths) path_indices = self.get_path_indices(archtuple[0]) if allow_isomorphisms or path_indices not in dic: dic[path_indices] = 1 candidates.append(archtuple) return candidates[:num]