def get_val_loss_nn_pred(self, nasbench, deterministic=1, patience=50): accs = [] test_accs = [] while len(accs) < 3 and patience > 0: patience -= 1 acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['validation_accuracy'] test_acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['test_accuracy'] # if acc not in accs: accs.append(acc) # if test_acc not in test_accs: test_accs.append(test_acc) return 100*(np.mean(np.array(accs))), 100*(np.mean(np.array(test_accs)))
def get_val_loss2(self, nasbench, deterministic=1, patience=50): if not deterministic: # output one of the three validation accuracies at random return 100*(nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['validation_accuracy']) else: # query the api until we see all three accuracies, then average them # a few architectures only have two accuracies, so we use patience to avoid an infinite loop accs = [] while len(accs) < 3 and patience > 0: patience -= 1 acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['validation_accuracy'] if acc not in accs: accs.append(acc) return round(100*(np.mean(accs)), 3)
def perturb(self, nasbench, edits=1): """ create new perturbed cell inspird by https://github.com/google-research/nasbench """ new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) for _ in range(edits): while True: if np.random.random() < 0.5: for src in range(0, NUM_VERTICES - 1): for dst in range(src + 1, NUM_VERTICES): new_matrix[src][dst] = 1 - new_matrix[src][dst] else: for ind in range(1, NUM_VERTICES - 1): available = [op for op in OPS if op != new_ops[ind]] new_ops[ind] = np.random.choice(available) new_spec = api.ModelSpec(new_matrix, new_ops) if nasbench.is_valid(new_spec): break return { 'matrix': new_matrix, 'ops': new_ops }
def mutate_rates(self, nasbench, edge_rate, node_rate): """ similar to perturb. A stochastic approach to perturbing the cell inspird by https://github.com/google-research/nasbench """ while True: new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) h, w = new_matrix.shape edge_mutation_prob = edge_rate for src in range(0, h - 1): for dst in range(src + 1, h): if random.random() < edge_mutation_prob: new_matrix[src, dst] = 1 - new_matrix[src, dst] op_mutation_prob = node_rate for ind in range(1, OP_SPOTS + 1): if random.random() < op_mutation_prob: available = [o for o in OPS if o != new_ops[ind]] new_ops[ind] = random.choice(available) new_spec = api.ModelSpec(new_matrix, new_ops) if nasbench.is_valid(new_spec): return { 'matrix': new_matrix, 'ops': new_ops }
def mutate_gvae(self, nasbench, mutation_rate=1.0): """ similar to perturb. A stochastic approach to perturbing the cell inspird by https://github.com/google-research/nasbench """ while True: new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) edge_mutation_prob = mutation_rate / NUM_VERTICES for src in range(0, NUM_VERTICES - 1): for dst in range(src + 1, NUM_VERTICES): if random.random() < edge_mutation_prob: new_matrix[src, dst] = 1 - new_matrix[src, dst] op_mutation_prob = mutation_rate / OP_SPOTS for ind in range(1, OP_SPOTS + 1): if random.random() < op_mutation_prob: available = [o for o in OPS if o != new_ops[ind]] new_ops[ind] = random.choice(available) isolate_nodes = find_isolate_node(new_matrix) new_spec = api.ModelSpec(new_matrix, new_ops) if nasbench.is_valid(new_spec): return { 'matrix': new_matrix, 'ops': new_ops, 'isolate_node_idxs': isolate_nodes }
def get_test_loss2(self, nasbench, patience=50): """ query the api until we see all three accuracies, then average them a few architectures only have two accuracies, so we use patience to avoid an infinite loop """ accs = [] while len(accs) < 3 and patience > 0: patience -= 1 acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['test_accuracy'] if acc not in accs: accs.append(acc) return 100*round((np.mean(accs)), 3)
def random_cell(cls, nasbench): """ From the NASBench repository https://github.com/google-research/nasbench """ while True: matrix = np.random.choice([0, 1], size=(NUM_VERTICES, NUM_VERTICES)) matrix = np.triu(matrix, 1) ops = np.random.choice(OPS, size=NUM_VERTICES).tolist() ops[0] = INPUT ops[-1] = OUTPUT spec = api.ModelSpec(matrix=matrix, ops=ops) if nasbench.is_valid(spec): return {'matrix': matrix, 'ops': ops}
def mutate2(self, nasbench, mutation_rate=1.0): """ similar to perturb. A stochastic approach to perturbing the cell inspird by https://github.com/google-research/nasbench """ iteration = 0 while True: new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) vertices = self.matrix.shape[0] op_spots = vertices - 2 edge_mutation_prob = mutation_rate / vertices for src in range(0, vertices - 1): for dst in range(src + 1, vertices): if random.random() < edge_mutation_prob: new_matrix[src, dst] = 1 - new_matrix[src, dst] if op_spots != 0: op_mutation_prob = mutation_rate / op_spots for ind in range(1, op_spots + 1): if random.random() < op_mutation_prob: available = [o for o in OPS if o != new_ops[ind]] new_ops[ind] = random.choice(available) new_spec = api.ModelSpec(new_matrix, new_ops) ops_idx = [-1] + [OPS.index(new_ops[idx]) for idx in range(1, len(new_ops)-1)] + [-2] iteration += 1 if iteration == 500: ops_idx = [-1] + [OPS.index(self.ops[idx]) for idx in range(1, len(self.ops) - 1)] + [-2] return { 'matrix': copy.deepcopy(self.matrix), 'ops': copy.deepcopy(self.ops), 'ops_idx': ops_idx } if nasbench.is_valid(new_spec): return { 'matrix': new_matrix, 'ops': new_ops, 'ops_idx': ops_idx }
def mutate_edit_distance(self, nasbench, edit_dist, candidate_num, data): """ similar to perturb. A stochastic approach to perturbing the cell inspird by https://github.com/google-research/nasbench """ arch_list = [] new_ops = copy.deepcopy(self.ops) edges = [(src, dst) for src in range(0, NUM_VERTICES - 1) for dst in range(src+1, NUM_VERTICES)] op_available_tuple = [] for ind in range(1, OP_SPOTS + 1): available = [o for o in OPS if o != new_ops[ind]] for o in available: op_available_tuple.append((ind, o)) idx_list = self.generate_edit_compose(edges, op_available_tuple, edit_dist) random.shuffle(idx_list) for edit_idx in idx_list: if edit_dist > 1 and len(arch_list) >= candidate_num: break new_matrix = copy.deepcopy(self.matrix) new_ops = copy.deepcopy(self.ops) for j in edit_idx: if j >= len(edges): nest_idx = op_available_tuple[j - len(edges)] new_ops[nest_idx[0]] = nest_idx[1] else: edge_conn = edges[j] new_matrix[edge_conn[0], edge_conn[1]] = 1 - new_matrix[edge_conn[0], edge_conn[1]] isolate_nodes = find_isolate_node(new_matrix) new_spec = api.ModelSpec(new_matrix, new_ops) flag = self.verify_correctness((new_matrix, new_ops), data, edit_dist) if nasbench.is_valid(new_spec) and flag: arch_list.append({ 'matrix': new_matrix, 'ops': new_ops, 'isolate_node_idxs': isolate_nodes }) return arch_list
def random_cell_both(cls, nasbench): """ From the NASBench repository https://github.com/google-research/nasbench """ while True: matrix = np.random.choice( [0, 1], size=(NUM_VERTICES, NUM_VERTICES)) matrix = np.triu(matrix, 1) matrix_orig = matrix.copy() isolate_nodes = find_isolate_node(matrix) ops = np.random.choice(OPS, size=NUM_VERTICES).tolist() ops[0] = INPUT ops[-1] = OUTPUT spec = api.ModelSpec(matrix=matrix, ops=ops) if nasbench.is_valid(spec): return { 'matrix': matrix, 'matrix_orig': matrix_orig, 'ops': ops, 'isolate_node_idxs': isolate_nodes }
def modelspec(self): return api.ModelSpec(matrix=self.matrix, ops=self.ops)