def get_val_loss_nn_pred(self, nasbench, deterministic=1, patience=50):
accs = []
test_accs = []
while len(accs) < 3 and patience > 0:
patience -= 1
acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['validation_accuracy']
test_acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['test_accuracy']
# if acc not in accs:
accs.append(acc)
# if test_acc not in test_accs:
test_accs.append(test_acc)
return 100*(np.mean(np.array(accs))), 100*(np.mean(np.array(test_accs)))
def get_val_loss2(self, nasbench, deterministic=1, patience=50):
if not deterministic:
# output one of the three validation accuracies at random
return 100*(nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['validation_accuracy'])
else:
# query the api until we see all three accuracies, then average them
# a few architectures only have two accuracies, so we use patience to avoid an infinite loop
accs = []
while len(accs) < 3 and patience > 0:
patience -= 1
acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['validation_accuracy']
if acc not in accs:
accs.append(acc)
return round(100*(np.mean(accs)), 3)
def perturb(self, nasbench, edits=1):
"""
create new perturbed cell
inspird by https://github.com/google-research/nasbench
"""
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
for _ in range(edits):
while True:
if np.random.random() < 0.5:
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
new_matrix[src][dst] = 1 - new_matrix[src][dst]
else:
for ind in range(1, NUM_VERTICES - 1):
available = [op for op in OPS if op != new_ops[ind]]
new_ops[ind] = np.random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
break
return {
'matrix': new_matrix,
'ops': new_ops
}
def mutate_rates(self, nasbench, edge_rate, node_rate):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
while True:
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
h, w = new_matrix.shape
edge_mutation_prob = edge_rate
for src in range(0, h - 1):
for dst in range(src + 1, h):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
op_mutation_prob = node_rate
for ind in range(1, OP_SPOTS + 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
return {
'matrix': new_matrix,
'ops': new_ops
}
def mutate_gvae(self, nasbench, mutation_rate=1.0):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
while True:
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
edge_mutation_prob = mutation_rate / NUM_VERTICES
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
op_mutation_prob = mutation_rate / OP_SPOTS
for ind in range(1, OP_SPOTS + 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
isolate_nodes = find_isolate_node(new_matrix)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
return {
'matrix': new_matrix,
'ops': new_ops,
'isolate_node_idxs': isolate_nodes
}
def get_test_loss2(self, nasbench, patience=50):
"""
query the api until we see all three accuracies, then average them
a few architectures only have two accuracies, so we use patience to avoid an infinite loop
"""
accs = []
while len(accs) < 3 and patience > 0:
patience -= 1
acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['test_accuracy']
if acc not in accs:
accs.append(acc)
return 100*round((np.mean(accs)), 3)
def random_cell(cls, nasbench):
"""
From the NASBench repository
https://github.com/google-research/nasbench
"""
while True:
matrix = np.random.choice([0, 1],
size=(NUM_VERTICES, NUM_VERTICES))
matrix = np.triu(matrix, 1)
ops = np.random.choice(OPS, size=NUM_VERTICES).tolist()
ops[0] = INPUT
ops[-1] = OUTPUT
spec = api.ModelSpec(matrix=matrix, ops=ops)
if nasbench.is_valid(spec):
return {'matrix': matrix, 'ops': ops}
def mutate2(self, nasbench, mutation_rate=1.0):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
iteration = 0
while True:
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
vertices = self.matrix.shape[0]
op_spots = vertices - 2
edge_mutation_prob = mutation_rate / vertices
for src in range(0, vertices - 1):
for dst in range(src + 1, vertices):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
if op_spots != 0:
op_mutation_prob = mutation_rate / op_spots
for ind in range(1, op_spots + 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
ops_idx = [-1] + [OPS.index(new_ops[idx]) for idx in range(1, len(new_ops)-1)] + [-2]
iteration += 1
if iteration == 500:
ops_idx = [-1] + [OPS.index(self.ops[idx]) for idx in range(1, len(self.ops) - 1)] + [-2]
return {
'matrix': copy.deepcopy(self.matrix),
'ops': copy.deepcopy(self.ops),
'ops_idx': ops_idx
}
if nasbench.is_valid(new_spec):
return {
'matrix': new_matrix,
'ops': new_ops,
'ops_idx': ops_idx
}
def mutate_edit_distance(self, nasbench, edit_dist, candidate_num, data):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
arch_list = []
new_ops = copy.deepcopy(self.ops)
edges = [(src, dst) for src in range(0, NUM_VERTICES - 1) for dst in range(src+1, NUM_VERTICES)]
op_available_tuple = []
for ind in range(1, OP_SPOTS + 1):
available = [o for o in OPS if o != new_ops[ind]]
for o in available:
op_available_tuple.append((ind, o))
idx_list = self.generate_edit_compose(edges, op_available_tuple, edit_dist)
random.shuffle(idx_list)
for edit_idx in idx_list:
if edit_dist > 1 and len(arch_list) >= candidate_num:
break
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
for j in edit_idx:
if j >= len(edges):
nest_idx = op_available_tuple[j - len(edges)]
new_ops[nest_idx[0]] = nest_idx[1]
else:
edge_conn = edges[j]
new_matrix[edge_conn[0], edge_conn[1]] = 1 - new_matrix[edge_conn[0], edge_conn[1]]
isolate_nodes = find_isolate_node(new_matrix)
new_spec = api.ModelSpec(new_matrix, new_ops)
flag = self.verify_correctness((new_matrix, new_ops), data, edit_dist)
if nasbench.is_valid(new_spec) and flag:
arch_list.append({
'matrix': new_matrix,
'ops': new_ops,
'isolate_node_idxs': isolate_nodes
})
return arch_list
def random_cell_both(cls, nasbench):
"""
From the NASBench repository
https://github.com/google-research/nasbench
"""
while True:
matrix = np.random.choice(
[0, 1], size=(NUM_VERTICES, NUM_VERTICES))
matrix = np.triu(matrix, 1)
matrix_orig = matrix.copy()
isolate_nodes = find_isolate_node(matrix)
ops = np.random.choice(OPS, size=NUM_VERTICES).tolist()
ops[0] = INPUT
ops[-1] = OUTPUT
spec = api.ModelSpec(matrix=matrix, ops=ops)
if nasbench.is_valid(spec):
return {
'matrix': matrix,
'matrix_orig': matrix_orig,
'ops': ops,
'isolate_node_idxs': isolate_nodes
}
def modelspec(self):
return api.ModelSpec(matrix=self.matrix, ops=self.ops)
