def single_lookup(seed_hash, dataset, output_path):
logging.info(f"START: seed_hash: {seed_hash}")
seed_arch = dataset.get_modelspec_by_hash(seed_hash)
seed_matrix, seed_ops = seed_arch.matrix, seed_arch.ops
seed_test_acc = dataset.query(
api101.ModelSpec(matrix=seed_matrix, ops=seed_ops), 'test', epochs=108)
random.seed(seed_hash)
row_dict = dict()
hash_iterator = list(dataset.hash_iterator())
random.shuffle(hash_iterator)
for _, arch_hash in enumerate(hash_iterator):
arch = dataset.get_modelspec_by_hash(arch_hash)
matrix, ops = arch.matrix, arch.ops
arch_test_acc = dataset.query(
api101.ModelSpec(matrix=matrix, ops=ops), 'test', epochs=108)
true_distance = get_edit_distance(
seed_matrix, seed_ops, matrix, ops, max_edit_distance)
row_dict[true_distance] = ((true_distance, seed_hash, arch_hash,
seed_test_acc - arch_test_acc, seed_test_acc, arch_test_acc))
# row_dict.keys() : 0, 1, ..., max_edit_distance
if len(row_dict) == max_edit_distance + 1:
break
with open(f'{output_path}/{seed_hash}.csv', 'w') as f:
writer = csv.writer(f)
writer.writerows(list(row_dict.values()))
def adj_neighborhood(self, nasbench, shuffle=True):
nbhd = []
# add op neighbors
for vertex in range(1, OP_SPOTS + 1):
if self.is_valid_vertex(vertex):
available = [op for op in OPS if op != self.ops[vertex]]
for op in available:
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
new_ops[vertex] = op
new_arch = {'matrix': new_matrix, 'ops': new_ops}
nbhd.append(new_arch)
# add edge neighbors
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
new_matrix[src][dst] = 1 - new_matrix[src][dst]
new_arch = {'matrix': new_matrix, 'ops': new_ops}
if self.matrix[src][dst] and self.is_valid_edge((src, dst)):
spec = api.ModelSpec(matrix=new_matrix, ops=new_ops)
if nasbench.is_valid(spec):
nbhd.append(new_arch)
if not self.matrix[src][dst] and Cell101(
**new_arch).is_valid_edge((src, dst)):
spec = api.ModelSpec(matrix=new_matrix, ops=new_ops)
if nasbench.is_valid(spec):
nbhd.append(new_arch)
if shuffle:
random.shuffle(nbhd)
return nbhd
def get_val_loss(self,
nasbench,
deterministic=True,
patience=50,
epochs=None,
dataset=None):
if not deterministic:
# output one of the three validation accuracies at random
acc = 0
if epochs:
acc = (100 * (1 - nasbench.query(
api.ModelSpec(matrix=self.matrix, ops=self.ops),
epochs=epochs)['validation_accuracy']))
else:
acc = (100 * (1 - nasbench.query(
api.ModelSpec(matrix=self.matrix,
ops=self.ops))['validation_accuracy']))
return np.round(acc, 4)
else:
# query the api until we see all three accuracies, then average them
# a few architectures only have two accuracies, so we use patience to avoid an infinite loop
accs = []
while len(accs) < 3 and patience > 0:
patience -= 1
if epochs:
acc = nasbench.query(api.ModelSpec(matrix=self.matrix,
ops=self.ops),
epochs=epochs)['validation_accuracy']
else:
acc = nasbench.query(
api.ModelSpec(matrix=self.matrix,
ops=self.ops))['validation_accuracy']
if acc not in accs:
accs.append(acc)
return round(100 * (1 - np.mean(accs)), 4)
def test_equal(nasbench_search_space):
import numpy as np
from nasbench import api
ss = nasbench_search_space
arch_1 = (np.array(
[[0, 0, 1, 0, 1, 1, 0], [0, 0, 1, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0, 1],
[0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0]],
dtype=np.int8), [1, 2, 1, 1, 0])
arch_2 = (np.array(
[[0, 0, 1, 0, 1, 1, 0], [0, 0, 1, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0, 1],
[0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0]],
dtype=np.int8), [1, 2, 1, 1, 0])
spec_1 = api.ModelSpec(
arch_1[0],
["input"] + [ss.ops_choices[ind] for ind in arch_1[1]] + ["output"])
spec_2 = api.ModelSpec(
arch_2[0],
["input"] + [ss.ops_choices[ind] for ind in arch_2[1]] + ["output"])
assert not spec_1 == spec_2
r_1 = ss.rollout_from_genotype(spec_1)
r_2 = ss.rollout_from_genotype(spec_2)
assert r_1.genotype.hash_spec(ss.ops_choices) == r_2.genotype.hash_spec(
ss.ops_choices)
assert r_1 == r_2
ss.compare_reduced = False
assert r_1 != r_2
def get_test_loss(self, nasbench, patience=50, epochs=None, dataset=None):
"""
query the api until we see all three accuracies, then average them
a few architectures only have two accuracies, so we use patience to avoid an infinite loop
"""
accs = []
while len(accs) < 3 and patience > 0:
patience -= 1
if epochs:
acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops), epochs=epochs)['test_accuracy']
else:
acc = nasbench.query(api.ModelSpec(matrix=self.matrix, ops=self.ops))['test_accuracy']
if acc not in accs:
accs.append(acc)
return round(100*(1-np.mean(accs)), 4)
def mutate(self,
nasbench,
mutation_rate=1.0,
patience=5000):
"""
A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
p = 0
while p < patience:
p += 1
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
edge_mutation_prob = mutation_rate / (NUM_VERTICES * (NUM_VERTICES - 1) / 2)
# flip each edge w.p. so expected flips is 1. same for ops
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
op_mutation_prob = mutation_rate / OP_SPOTS
for ind in range(1, OP_SPOTS + 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
return {
'matrix': new_matrix,
'ops': new_ops
}
return self.mutate(nasbench, mutation_rate+1)
def mutate(self, nasbench, mutation_rate=1.0):
"""
similar to perturb. A stochastic approach to perturbing the cell
inspird by https://github.com/google-research/nasbench
"""
while True:
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
edge_mutation_prob = mutation_rate / NUM_VERTICES
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
op_mutation_prob = mutation_rate / OP_SPOTS
for ind in range(1, OP_SPOTS + 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
return {
'matrix': new_matrix,
'ops': new_ops
}
def config2data_A(config, b):
VERTICES = 7
MAX_EDGES = 9
budget = 108
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = config["edge_%d" % i]
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
return None, None
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = b.dataset.query(model_spec, epochs=budget)
msp = b.dataset.get_metrics_from_spec(model_spec)
except api.OutOfDomainError:
return None, None
test_acc = [msp[1][108][k]['final_test_accuracy'] for k in range(3)]
return data, np.mean(test_acc)
def perturb(self, nasbench, edits=1):
"""
create new perturbed cell
inspird by https://github.com/google-research/nasbench
"""
new_matrix = copy.deepcopy(self.matrix)
new_ops = copy.deepcopy(self.ops)
for _ in range(edits):
while True:
if np.random.random() < 0.5:
for src in range(0, NUM_VERTICES - 1):
for dst in range(src+1, NUM_VERTICES):
new_matrix[src][dst] = 1 - new_matrix[src][dst]
else:
for ind in range(1, NUM_VERTICES - 1):
available = [op for op in OPS if op != new_ops[ind]]
new_ops[ind] = np.random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
break
return {
'matrix': new_matrix,
'ops': new_ops
}
def model_to_metric(_ops, _matrix):
model_spec = api.ModelSpec(matrix=_matrix,
ops=[ID_TO_NODE[t] for t in _ops])
computed_metrics = nasbench.get_metrics_from_spec(model_spec)[1]
return np.mean([d["final_test_accuracy"] for d in
computed_metrics[max_epochs]])\
.astype(np.float32).reshape([1])
def mutate_spec(nasbench, old_spec, mutation_rate=1.0):
"""Computes a valid mutated spec from the old_spec."""
while True:
new_matrix = copy.deepcopy(old_spec.original_matrix)
new_ops = copy.deepcopy(old_spec.original_ops)
# In expectation, V edges flipped (note that most end up being pruned).
edge_mutation_prob = mutation_rate / NUM_VERTICES
for src in range(0, NUM_VERTICES - 1):
for dst in range(src + 1, NUM_VERTICES):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
# In expectation, one op is resampled.
op_mutation_prob = mutation_rate / OP_SPOTS
for ind in range(1, NUM_VERTICES - 1):
if random.random() < op_mutation_prob:
available = [
o for o in nasbench.config['available_ops']
if o != new_ops[ind]
]
new_ops[ind] = random.choice(available)
new_spec = api.ModelSpec(new_matrix, new_ops)
if nasbench.is_valid(new_spec):
return new_spec
def objective_function(self, config, budget=108):
bitlist = [0] * (VERTICES * (VERTICES - 1) // 2)
for i in range(MAX_EDGES):
bitlist[config["edge_%d" % i]] = 1
out = 0
for bit in bitlist:
out = (out << 1) | bit
matrix = np.fromfunction(graph_util.gen_is_edge_fn(out),
(VERTICES, VERTICES),
dtype=np.int8)
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(config, model_spec, budget)
return 1 - data["validation_accuracy"], data["training_time"]
def objective_function(self, config, budget=108):
# bit = 0
# for i in range(VERTICES * (VERTICES - 1) // 2):
# bit += config["edge_%d" % i] * 2 ** i
# matrix = np.fromfunction(graph_util.gen_is_edge_fn(bit),
# (VERTICES, VERTICES),
# dtype=np.int8)
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = config["edge_%d" % i]
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(config, model_spec, budget)
return 1 - data["validation_accuracy"], data["training_time"]
def random_cell_gnn(self, nasbench):
while True:
key = random.sample(self.total_keys, 1)[0]
architecture = self.total_archs[key]
spec = api.ModelSpec(matrix=architecture['o_matrix'],
ops=architecture['o_ops'])
if nasbench.is_valid(spec):
key = spec.hash_spec(self.ops_t)
o_matrix, o_ops = spec.matrix, spec.ops
if key in self.total_keys:
matrix = self.total_archs[key]['matrix']
ops = self.total_archs[key]['ops']
o_matrix = self.total_archs[key]['o_matrix']
val_loss = 100 * (1 - self.total_archs[key]['val'])
test_loss = 100 * (1 - self.total_archs[key]['test'])
else:
matrix, ops = self.matrix_dummy_nodes(o_matrix, o_ops)
arch = {'matrix': o_matrix, 'ops': o_ops}
val_loss = Cell(**arch).get_val_loss(self.nasbench, True)
test_loss = Cell(**arch).get_test_loss(self.nasbench)
return {
'matrix': matrix,
'matrix_orig': o_matrix,
'ops': ops,
'isolate_node_idxs': [],
'val_loss': val_loss,
'test_loss': test_loss
}
def generate_search_space_without_loose_ends(self):
# Create all possible connectivity patterns
for iter, adjacency_matrix in enumerate(
self.generate_adjacency_matrix_without_loose_ends()):
print(iter)
# Print graph
# Evaluate every possible combination of node ops.
n_repeats = int(np.sum(np.sum(adjacency_matrix, axis=1)[1:-1] > 0))
for combination in itertools.product(
[CONV1X1, CONV3X3, MAXPOOL3X3], repeat=n_repeats):
# Create node labels
# Add some op as node 6 which isn't used, here conv1x1
ops = [INPUT]
combination = list(combination)
for i in range(5):
if np.sum(adjacency_matrix, axis=1)[i + 1] > 0:
ops.append(combination.pop())
else:
ops.append(CONV1X1)
assert (len(combination) == 0, 'Something is wrong')
ops.append(OUTPUT)
# Create nested list from numpy matrix
nasbench_adjacency_matrix = adjacency_matrix.astype(
np.int).tolist()
# Assemble the model spec
model_spec = api.ModelSpec(
# Adjacency matrix of the module
matrix=nasbench_adjacency_matrix,
# Operations at the vertices of the module, matches order of matrix
ops=ops)
yield adjacency_matrix, ops, model_spec
def query_arch(self,
matrix,
ops,
encode_paths=True,
deterministic=True,
require_distance=False):
matrix = matrix.astype(np.int8)
model_spec = api.ModelSpec(matrix=matrix, ops=ops)
key = model_spec.hash_spec(self.ops_t)
if not model_spec.valid_spec:
return None
o_matrix = model_spec.matrix
o_ops = model_spec.ops
if key in self.total_keys:
matrix = self.total_archs[key]['matrix']
ops = self.total_archs[key]['ops']
val_loss = 100 * (1 - self.total_archs[key]['val'])
test_loss = 100 * (1 - self.total_archs[key]['test'])
arch = {'matrix': matrix, 'ops': ops}
else:
matrix, ops = self.matrix_dummy_nodes(o_matrix, o_ops)
arch = {'matrix': matrix, 'ops': ops}
val_loss = Cell(**arch).get_val_loss(self.nasbench, deterministic)
test_loss = Cell(**arch).get_test_loss(self.nasbench)
if encode_paths:
encoding = Cell(**arch).encode_paths()
else:
encoding = Cell(**arch).encode_cell2()
return [(matrix, ops, []), o_matrix, o_ops, encoding, val_loss,
test_loss, key]
def random_cell_cont_adj(nasbench):
"""
continuous adjacency matrix
draw num_paths randomly
draw continuous [0,1] for each edge, then threshold
"""
while True:
values = np.random.random(size=(NUM_VERTICES, NUM_VERTICES))
values = np.triu(values, 1)
n = np.random.randint(8) + 1
flat = values.flatten()
threshold = flat[np.argsort(flat)[-1 * n]]
# now convert it to a model spec
matrix = np.random.choice([0], size=(NUM_VERTICES, NUM_VERTICES))
for i in range(NUM_VERTICES):
for j in range(NUM_VERTICES):
if values[i][j] >= threshold:
matrix[i][j] = 1
ops = np.random.choice(OPS, size=NUM_VERTICES).tolist()
ops[0] = INPUT
ops[-1] = OUTPUT
spec = api.ModelSpec(matrix=matrix, ops=ops)
if nasbench.is_valid(spec):
return {'matrix': matrix, 'ops': ops}
def random_models(argv):
del argv # Unused
LIMIT = 100
OUTPUT = './experiments/random_sampling.json'
# Load the data from file (this will take some time)
nasbench = api.NASBench(NASBENCH_TFRECORD)
npEnc = api._NumpyEncoder()
for index, unique_hash in enumerate(nasbench.hash_iterator()):
if index >= LIMIT:
break
fixed_metrics, computed_metrics = nasbench.get_metrics_from_hash(
unique_hash)
model_spec = api.ModelSpec(matrix=fixed_metrics['module_adjacency'],
ops=fixed_metrics['module_operations'])
random_estimate = nasbench.random_estimate(model_spec, MODEL_DIR)
data = nasbench.query(model_spec)
merge = {**fixed_metrics, **computed_metrics}
merge['module_adjacency'] = npEnc.default(
fixed_metrics['module_adjacency'])
merge['random_sampling_time'] = random_estimate['prediction_time']
merge['random_samples'] = random_estimate['evaluation_results']
merge['train_accuracy'] = data['train_accuracy']
merge['test_accuracy'] = data['test_accuracy']
merge['validation_accuracy'] = data['validation_accuracy']
print(index, merge)
with open(OUTPUT, 'a') as f:
f.write(json.dumps(merge) + '\n')
def single_model(argv):
del argv # Unused
# Load the data from file (this will take some time)
nasbench = api.NASBench(NASBENCH_TFRECORD)
# Create an Inception-like module (5x5 convolution replaced with two 3x3
# convolutions).
model_spec = api.ModelSpec(
# Adjacency matrix of the module
matrix=[
[0, 1, 1, 1, 0, 1, 0], # input layer
[0, 0, 0, 0, 0, 0, 1], # 1x1 conv
[0, 0, 0, 0, 0, 0, 1], # 3x3 conv
[0, 0, 0, 0, 1, 0, 0], # 5x5 conv (replaced by two 3x3's)
[0, 0, 0, 0, 0, 0, 1], # 5x5 conv (replaced by two 3x3's)
[0, 0, 0, 0, 0, 0, 1], # 3x3 max-pool
[0, 0, 0, 0, 0, 0, 0]
], # output layer
# Operations at the vertices of the module, matches order of matrix
ops=[INPUT, CONV1X1, CONV3X3, CONV3X3, CONV3X3, MAXPOOL3X3, OUTPUT])
random_estimate = nasbench.random_estimate(model_spec, MODEL_DIR)
print(random_estimate)
# Query this model from dataset, returns a dictionary containing the metrics
# associated with this model.
# print('Querying an Inception-like model.')
data = nasbench.query(model_spec)
print(data)
def EvaluateNasbench(theta, search_space, logger, NASbenchName):
# get result log
stdout_backup = sys.stdout
result_path = os.path.join(cfg.OUT_DIR, "result.log")
log_file = open(result_path, "w")
sys.stdout = log_file
if NASbenchName == "nasbench201":
geotype = search_space.genotype(theta)
index = api_nasben201.query_index_by_arch(geotype)
api_nasben201.show(index)
else:
current_best = np.argmax(theta, axis=1)
config = ConfigSpace.Configuration(
search_space.search_space.get_configuration_space(),
vector=current_best)
adjacency_matrix, node_list = search_space.search_space.convert_config_to_nasbench_format(
config)
node_list = [
INPUT, *node_list, OUTPUT
] if search_space.search_space.search_space_number == 3 else [
INPUT, *node_list, CONV1X1, OUTPUT
]
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
nasbench_data = nasbench.query(model_spec, epochs=108)
print("the test accuracy in {}".format(NASbenchName))
print(nasbench_data['test_accuracy'])
log_file.close()
sys.stdout = stdout_backup
def eval_random_ws_model(config, model):
model_list = pickle.load(open(model, 'rb'))
accs = []
for model in model_list:
adjacency_matrix, node_list = model[0]
if int(config['search_space']) == int('1'):
adjacency_matrix = upscale_to_nasbench_format(adjacency_matrix)
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif int(config['search_space']) == int('2'):
adjacency_matrix = upscale_to_nasbench_format(adjacency_matrix)
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
elif int(config['search_space']) == int('3'):
node_list = [INPUT, *node_list, OUTPUT]
else:
raise ValueError('Unknown search space')
# Convert the adjacency matrix in format for nasbench
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
# Query nasbench
data = nasbench.query(model_spec)
valid_acc, test_acc = [], []
for item in data:
test_acc.append(item['test_accuracy'])
valid_acc.append(item['validation_accuracy'])
accs.append([np.mean(valid_acc), np.mean(test_acc)])
return accs
def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict:
"""
Copy of the 'objective_function' from nas_cifar10.py
We adapted the file in such a way, that the complete result is returned. The original implementation returns
only the validation error. Now, it can also return the test loss for a given configuration.
Parameters
----------
config : Dict
budget : int
The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned.
Returns
-------
Dict
"""
# Unify the return value to a dictionary.
failure = {
"test_accuracy": 0,
"validation_accuracy": 0,
"training_time": 0,
"info": "failure"
}
if self.benchmark.multi_fidelity is False:
assert budget == 108
edge_prob = []
for i in range(VERTICES * (VERTICES - 1) // 2):
edge_prob.append(config["edge_%d" % i])
idx = np.argsort(edge_prob)[::-1][:config["num_edges"]]
binay_encoding = np.zeros(len(edge_prob))
binay_encoding[idx] = 1
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = binay_encoding[i]
if graph_util.num_edges(matrix) > MAX_EDGES:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.benchmark.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
self.benchmark.record_valid(config, data, model_spec)
# We dont need this field.
data.pop('module_adjacency')
return data
def objective_function(self, config, budget=108):
edge_prob = []
for i in range(VERTICES * (VERTICES - 1) // 2):
edge_prob.append(config["edge_%d" % i])
idx = np.argsort(edge_prob)[::-1][:config["num_edges"]]
binay_encoding = np.zeros(len(edge_prob))
binay_encoding[idx] = 1
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = binay_encoding[i]
if graph_util.num_edges(matrix) > MAX_EDGES:
self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(config, model_spec, budget)
return 1 - data["validation_accuracy"], data["training_time"]
def is_valid_NAS(g, nasbench, START_TYPE=1, END_TYPE=0):
L = {
1: 'input',
0: 'output',
2: 'conv1x1-bn-relu',
3: 'conv3x3-bn-relu',
4: 'maxpool3x3'
}
# first need to be a valid DAG computation graph
res = is_valid_DAG(g, START_TYPE, END_TYPE)
node_atts = nx.get_node_attributes(g, 'Label')
labels = [node_atts[i] for i in range(len(node_atts))]
node_list = [L[x] for x in np.array(labels)]
adjacency_matrix = nx.to_numpy_array(g).astype(int)
# Check for nasbench validity
try:
model_spec = api.ModelSpec(matrix=adjacency_matrix, ops=node_list)
# data = nasbench.query(model_spec)
if nasbench.is_valid(model_spec):
return res
else:
return False
except:
return False
return res
def _query_benchmark(self, config: Dict, fidelity: Dict,
run_index: int) -> Dict:
adjacency_matrix, node_list = self.search_space.convert_config_to_nasbench_format(
config)
node_list = [INPUT, *node_list, CONV1X1, OUTPUT]
adjacency_list = adjacency_matrix.astype(np.int).tolist()
model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
try:
nasbench_data = self._query_api(model_spec=model_spec,
run_index=run_index,
epochs=int(fidelity['budget']))
except api.OutOfDomainError:
return {
"trainable_parameters": 0,
"training_time": 0,
"train_accuracy": 0,
"validation_accuracy": 0,
"test_accuracy": 0,
"module_operations": 0,
"info": "failure"
}
nasbench_data.pop('module_adjacency')
return nasbench_data
def _convert_cell_to_nb101_spec(cell):
matrix = np.triu(np.ones((7,7)), 1)
ops_to_nb101 = {
'MaxPool1x1': 'maxpool3x3',
'ReLUConvBN1x1': 'conv1x1-bn-relu',
'ReLUConvBN3x3': 'conv3x3-bn-relu',
}
ops_to_nb101_edges = {
'Identity': 1,
'Zero': 0,
}
num_vertices = 7
ops = ['input'] * num_vertices
ops[-1] = 'output'
for i in range(1, 6):
ops[i] = ops_to_nb101[cell.nodes[i+1]['subgraph'].edges[1, 2]['op'].get_op_name]
for i, j in cell.edges:
matrix[i-1][j-1] = ops_to_nb101_edges[cell.edges[i, j]['op'].get_op_name]
spec = api.ModelSpec(matrix=matrix, ops=ops)
return spec
def config2data_B(config, b):
VERTICES = 7
MAX_EDGES = 9
budget = 108
bitlist = [0] * (VERTICES * (VERTICES - 1) // 2)
for i in range(MAX_EDGES):
bitlist[config["edge_%d" % i]] = 1
out = 0
for bit in bitlist:
out = (out << 1) | bit
matrix = np.fromfunction(graph_util.gen_is_edge_fn(out),
(VERTICES, VERTICES),
dtype=np.int8)
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
return None, None
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = b.dataset.query(model_spec, epochs=budget)
msp = b.dataset.get_metrics_from_spec(model_spec)
except api.OutOfDomainError:
return None, None
test_acc = [msp[1][108][k]['final_test_accuracy'] for k in range(3)]
return data, test_acc
def get_test_mean_acc(self, chromosome):
cell = api.ModelSpec(matrix=chromosome.matrix,
ops=chromosome.operations)
fixed_stats, computed_stats = self.api.get_metrics_from_spec(cell)
stats = computed_stats[108]
mean_score = [stats[i]['final_test_accuracy'] for i in range(3)]
return np.mean(mean_score)
def generate_arch(n, nasbench, need_perf=False):
count = 0
archs = []
seqs = []
valid_accs = []
all_keys = list(nasbench.hash_iterator())
np.random.shuffle(all_keys)
for key in all_keys:
count += 1
if n is not None and count > n:
break
fixed_stat, computed_stat = nasbench.get_metrics_from_hash(key)
arch = api.ModelSpec(
matrix=fixed_stat['module_adjacency'],
ops=fixed_stat['module_operations'],
)
if need_perf:
data = nasbench.query(arch)
if data['validation_accuracy'] < 0.85:
continue
valid_accs.append(data['validation_accuracy'])
archs.append(arch)
seqs.append(convert_arch_to_seq(arch.matrix, arch.ops))
if need_perf:
return archs, seqs, valid_accs
return archs, seqs
def gen_data_point(nasbench):
i = 0
epoch = 108
padding = [0, 0, 0, 0, 0, 0, 0]
best_val_acc = 0
best_test_acc = 0
tf_data = {}
for unique_hash in nasbench.hash_iterator():
fixed_metrics, computed_metrics = nasbench.get_metrics_from_hash(
unique_hash)
ops_array = transform_operations(fixed_metrics['module_operations'])
ops_list = convert_tf_format(ops_array)
adj_array = fixed_metrics['module_adjacency'].tolist()
print('\nIterating over {} / {} unique models in the dataset.'.format(
i, 423623))
test_acc_avg = 0.0
val_acc_avg = 0.0
training_time = 0.0
for repeat_index in range(len(computed_metrics[epoch])):
assert len(computed_metrics[epoch]
) == 3, 'len(computed_metrics[epoch]) should be 3'
data_point = computed_metrics[epoch][repeat_index]
val_acc_avg += data_point['final_validation_accuracy']
test_acc_avg += data_point['final_test_accuracy']
training_time += data_point['final_training_time']
val_acc_avg = val_acc_avg / 3.0
test_acc_avg = test_acc_avg / 3.0
tf_data[unique_hash] = (adj_array, ops_list, val_acc_avg)
training_time_avg = training_time / 3.0
adj_array = fixed_metrics['module_adjacency'].tolist()
params = fixed_metrics['trainable_parameters']
print('parameters size: {}'.format(params))
model_spec = api.ModelSpec(fixed_metrics['module_adjacency'],
fixed_metrics['module_operations'])
data = nasbench.query(model_spec, epochs=108)
print('api training time: {}'.format(data['training_time']))
print('real training time: {}'.format(training_time_avg))
if len(adj_array) <= 6:
for row in range(len(adj_array)):
for _ in range(7 - len(adj_array)):
adj_array[row].append(0)
for _ in range(7 - len(adj_array)):
adj_array.append(padding)
if val_acc_avg > best_val_acc:
best_val_acc = val_acc_avg
if test_acc_avg > best_test_acc:
best_test_acc = test_acc_avg
print('best val. acc: {:.4f}, best test acc {:.4f}'.format(
best_val_acc, best_test_acc))
i += 1
return tf_data
