def objective_function(self, config, budget=108):
edge_prob = []
for i in range(VERTICES * (VERTICES - 1) // 2):
edge_prob.append(config["edge_%d" % i])
idx = np.argsort(edge_prob)[::-1][:config["num_edges"]]
binay_encoding = np.zeros(len(edge_prob))
binay_encoding[idx] = 1
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = binay_encoding[i]
if graph_util.num_edges(matrix) > MAX_EDGES:
self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(config, model_spec, budget)
return 1 - data["validation_accuracy"], data["training_time"]
def objective_function(self, config, budget=108):
# bit = 0
# for i in range(VERTICES * (VERTICES - 1) // 2):
# bit += config["edge_%d" % i] * 2 ** i
# matrix = np.fromfunction(graph_util.gen_is_edge_fn(bit),
# (VERTICES, VERTICES),
# dtype=np.int8)
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = config["edge_%d" % i]
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(config, model_spec, budget)
return 1 - data["validation_accuracy"], data["training_time"]
def config2data_A(config, b):
VERTICES = 7
MAX_EDGES = 9
budget = 108
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = config["edge_%d" % i]
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
return None, None
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = b.dataset.query(model_spec, epochs=budget)
msp = b.dataset.get_metrics_from_spec(model_spec)
except api.OutOfDomainError:
return None, None
test_acc = [msp[1][108][k]['final_test_accuracy'] for k in range(3)]
return data, np.mean(test_acc)
def objective_function(self, config, budget=108):
bitlist = [0] * (VERTICES * (VERTICES - 1) // 2)
for i in range(MAX_EDGES):
bitlist[config["edge_%d" % i]] = 1
out = 0
for bit in bitlist:
out = (out << 1) | bit
matrix = np.fromfunction(graph_util.gen_is_edge_fn(out),
(VERTICES, VERTICES),
dtype=np.int8)
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(config, model_spec, budget)
return 1 - data["validation_accuracy"], data["training_time"]
def generate_buckets(vertices, bits, max_edges, num_ops, verify_isomorphism):
buckets = {}
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > max_edges):
return
# Iterate through all possible labelings
for labeling in itertools.product(
*[range(num_ops) for _ in range(vertices - 2)]):
labeling = [-1] + list(labeling) + [-2]
fingerprint = graph_util.hash_module(matrix, labeling)
if fingerprint not in buckets:
buckets[fingerprint] = (matrix.tolist(), labeling)
# This catches the "false positive" case of two models which are not
# isomorphic hashing to the same bucket.
elif verify_isomorphism:
canonical_graph = buckets[fingerprint]
if not graph_util.is_isomorphic(
(matrix.tolist(), labeling), canonical_graph):
logging.fatal(
'Matrix:\n%s\nLabel: %s\nis not isomorphic to'
' canonical matrix:\n%s\nLabel: %s', str(matrix),
str(labeling), str(canonical_graph[0]),
str(canonical_graph[1]))
sys.exit()
return buckets
def config2data_B(config, b):
VERTICES = 7
MAX_EDGES = 9
budget = 108
bitlist = [0] * (VERTICES * (VERTICES - 1) // 2)
for i in range(MAX_EDGES):
bitlist[config["edge_%d" % i]] = 1
out = 0
for bit in bitlist:
out = (out << 1) | bit
matrix = np.fromfunction(graph_util.gen_is_edge_fn(out),
(VERTICES, VERTICES),
dtype=np.int8)
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
return None, None
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = b.dataset.query(model_spec, epochs=budget)
msp = b.dataset.get_metrics_from_spec(model_spec)
except api.OutOfDomainError:
return None, None
test_acc = [msp[1][108][k]['final_test_accuracy'] for k in range(3)]
return data, test_acc
def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict:
"""
Copy of the 'objective_function' from nas_cifar10.py
We adapted the file in such a way, that the complete result is returned. The original implementation returns
only the validation error. Now, it can also return the test loss for a given configuration.
Parameters
----------
config : Dict
budget : int
The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned.
Returns
-------
Dict
"""
# Unify the return value to a dictionary.
failure = {
"test_accuracy": 0,
"validation_accuracy": 0,
"training_time": 0,
"info": "failure"
}
if self.benchmark.multi_fidelity is False:
assert budget == 108
edge_prob = []
for i in range(VERTICES * (VERTICES - 1) // 2):
edge_prob.append(config["edge_%d" % i])
idx = np.argsort(edge_prob)[::-1][:config["num_edges"]]
binay_encoding = np.zeros(len(edge_prob))
binay_encoding[idx] = 1
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = binay_encoding[i]
if graph_util.num_edges(matrix) > MAX_EDGES:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.benchmark.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
self.benchmark.record_valid(config, data, model_spec)
# We dont need this field.
data.pop('module_adjacency')
return data
def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict:
"""
Copy of the 'objective_function' from nas_cifar10.py
We adapted the file in such a way, that the complete result is returned. The original implementation returns
only the validation error. Now, it can also return the test loss for a given configuration.
Parameters
----------
config : Dict
budget : int
The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned.
Returns
-------
Dict
"""
failure = {
"test_accuracy": 0,
"validation_accuracy": 0,
"training_time": 0,
"info": "failure"
}
if self.benchmark.multi_fidelity is False:
assert budget == 108
bitlist = [0] * (VERTICES * (VERTICES - 1) // 2)
for i in range(MAX_EDGES):
bitlist[config["edge_%d" % i]] = 1
out = 0
for bit in bitlist:
out = (out << 1) | bit
matrix = np.fromfunction(graph_util.gen_is_edge_fn(out),
(VERTICES, VERTICES),
dtype=np.int8)
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.benchmark.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
self.benchmark.record_valid(config, data, model_spec)
# We dont need this field.
data.pop('module_adjacency')
return data
def generate_graph(max_vertices, max_edges, num_ops, verify_isomorphism,
output_file):
FLAGS = Namespace(max_vertices=max_vertices,
num_ops=num_ops,
max_edges=max_edges,
verify_isomorphism=verify_isomorphism,
output_file=output_file)
total_graphs = 0 # Total number of graphs (including isomorphisms)
# hash --> (matrix, label) for the canonical graph associated with each hash
buckets = {}
logging.info('Using %d vertices, %d op labels, max %d edges',
FLAGS.max_vertices, FLAGS.num_ops, FLAGS.max_edges)
for vertices in range(2, FLAGS.max_vertices + 1):
for bits in range(2**(vertices * (vertices - 1) // 2)):
# Construct adj matrix from bit string
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
# Discard any graphs which can be pruned or exceed constraints
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > FLAGS.max_edges):
continue
# Iterate through all possible labelings
for labeling in itertools.product(
*[range(FLAGS.num_ops) for _ in range(vertices - 2)]):
total_graphs += 1
labeling = [-1] + list(labeling) + [-2]
fingerprint = graph_util.hash_module(matrix, labeling)
if fingerprint not in buckets:
buckets[fingerprint] = (matrix.tolist(), labeling)
# This catches the "false positive" case of two models which are not
# isomorphic hashing to the same bucket.
elif FLAGS.verify_isomorphism:
canonical_graph = buckets[fingerprint]
if not graph_util.is_isomorphic(
(matrix.tolist(), labeling), canonical_graph):
logging.fatal(
'Matrix:\n%s\nLabel: %s\nis not isomorphic to'
' canonical matrix:\n%s\nLabel: %s', str(matrix),
str(labeling), str(canonical_graph[0]),
str(canonical_graph[1]))
sys.exit()
logging.info('Up to %d vertices: %d graphs (%d without hashing)',
vertices, len(buckets), total_graphs)
with open(FLAGS.output_file, 'w') as f:
json.dump(buckets, f, sort_keys=True)
def _query_benchmark(self, config: Dict, run_index: int, budget: int = 108) -> Dict:
"""
Copied from the 'objective_function' from nas_cifar10.py
We adapted the file in such a way, that the complete result is returned. The original implementation returns
only the validation error. Now, it can also return the test loss for a given configuration.
Parameters
----------
config : Dict
run_index : int
Specifies the seed to use. Can be one of 0, 1, 2.
budget : int
The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned.
Returns
-------
Dict
"""
failure = {"test_accuracy": 0, "train_accuracy": 0, "validation_accuracy": 0, "training_time": 0,
"info": "failure", "trainable_parameters": 0, "module_operations": 0}
if self.benchmark.multi_fidelity is False:
assert budget == 108
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for i in range(VERTICES * (VERTICES - 1) // 2):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = config["edge_%d" % i]
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
labeling = [config["op_node_%d" % i] for i in range(5)]
labeling = ['input'] + list(labeling) + ['output']
model_spec = api.ModelSpec(matrix, labeling)
try:
data = modified_query(self.benchmark, run_index=run_index, model_spec=model_spec, epochs=budget)
except api.OutOfDomainError:
self.benchmark.record_invalid(config, 1, 1, 0)
return failure
self.benchmark.record_valid(config, data, model_spec)
# We dont need this field.
data.pop('module_adjacency')
return data
def sample(self, n, batch_size=None):
if self.mode == "eval":
# return the current rollout
return [NasBench101Rollout(
*self.cur_solution, search_space=self.search_space)] * n
assert batch_size is None
rollouts = []
cur_matrix, cur_ops = self.cur_solution
ss = self.search_space
for n_r in range(n):
if np.random.rand() < self.mutation_edges_prob:
while 1:
edge_ind = np.random.randint(0, ss.num_possible_edges, size=1)
while graph_util.num_edges(cur_matrix) == ss.max_edges and \
cur_matrix[ss.idx[0][edge_ind], ss.idx[1][edge_ind]] == 0:
edge_ind = np.random.randint(0, ss.num_possible_edges, size=1)
new_matrix = cur_matrix.copy()
new_matrix[ss.idx[0][edge_ind], ss.idx[1][edge_ind]] \
= 1 - cur_matrix[ss.idx[0][edge_ind], ss.idx[1][edge_ind]]
new_rollout = NasBench101Rollout(new_matrix, cur_ops,
search_space=self.search_space)
try:
ss.nasbench._check_spec(new_rollout.genotype)
except api.OutOfDomainError:
# ignore out-of-domain archs (disconnected)
continue
else:
cur_matrix = new_matrix
break
else:
ops_ind = np.random.randint(0, ss.num_ops, size=1)[0]
new_ops = np.random.randint(0, ss.num_op_choices - 1, size=1)[0]
while new_ops == cur_ops[ops_ind]:
new_ops = np.random.randint(0, ss.num_op_choices - 1, size=1)[0]
cur_ops[ops_ind] = new_ops
rollouts.append(NasBench101Rollout(
cur_matrix,
cur_ops,
search_space=self.search_space
))
return rollouts
def objective_function_from_matrix(self, matrix, budget=108):
if self.multi_fidelity is False:
assert budget == 108
# if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES:
if graph_util.num_edges(matrix) > MAX_EDGES:
# self.record_invalid(config, 1, 1, 0)
return 1, 0
labeling = [
'input', 'conv1x1-bn-relu', 'conv3x3-bn-relu', 'conv3x3-bn-relu',
'conv3x3-bn-relu', 'maxpool3x3', 'output'
]
model_spec = api.ModelSpec(matrix, labeling)
try:
data = self.dataset.query(model_spec, epochs=budget)
except api.OutOfDomainError:
# self.record_invalid(config, 1, 1, 0)
return 1, 0
self.record_valid(matrix, data, model_spec)
return 1 - data["validation_accuracy"], data["training_time"]
def make_graphs(vertices, bits):
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
if graph_util.num_edges(matrix) > max_edges:
return []
if not graph_util.is_full_dag(matrix):
return []
out = []
for labeling in itertools.product(
*[range(num_ops) for _ in range(vertices - 2)]):
labeling = [-1] + list(labeling) + [-2]
out.append({
"hash": graph_util.hash_module(matrix, labeling),
"adj": matrix.tolist(),
"labeling": labeling,
})
return out
def evaluate(self, c):
c = self._check_shape(c)
evals = []
test_error = []
training_time = []
matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8)
idx = np.triu_indices(matrix.shape[0], k=1)
for _c in c:
for i in range(MATRIX_ELEMENTS):
row = idx[0][i]
col = idx[1][i]
matrix[row, col] = _c[i]
if graph_util.num_edges(matrix) > MAX_EDGES:
evals.append(1)
test_error.append(1)
training_time.append(0)
else:
ops = [OPS[i] for i in _c[MATRIX_ELEMENTS:]]
ops = ["input"] + list(ops) + ["output"]
model_spec = api.ModelSpec(matrix=matrix, ops=ops)
try:
data = self.nasbench.query(model_spec, epochs=108)
evals.append(1 - data["validation_accuracy"])
test_error.append(1 - data["test_accuracy"])
training_time.append(data["training_time"])
self.estimated_wall_clock_time += data["training_time"]
except api.OutOfDomainError:
evals.append(1)
test_error.append(1)
training_time.append(0)
evals = np.array(evals)
evals = evals if self.minimize else -evals
test_error = np.array(test_error)
training_time = np.array(training_time)
info = {"training_time": training_time, "test_error": test_error}
return evals, info
def generate_graphs(nasbench):
"""A function that generates all possible graphs that could have been
processed via NAS Bench, and yields tuples of x values and y values,
where y is zero when x is not contained in NASBench-101
Arguments:
nasbench: NASBench
an instantiation of the NASBench class provided in the official
release of nas bench source code
Returns:
generator: Iterator
a generator tha yields tuples of x values and y values, where
y is zero when x is not contained in NASBench-101
"""
# these settings were used in the NASBench-101 paper
max_vertices = 7
max_edges = 9
max_epochs = 108
max_adjacency_size = max_vertices * (max_vertices - 1) // 2
# a helper function that maps a model architecture to a metric
def model_to_metric(_ops, _matrix):
model_spec = api.ModelSpec(matrix=_matrix,
ops=[ID_TO_NODE[t] for t in _ops])
computed_metrics = nasbench.get_metrics_from_spec(model_spec)[1]
return np.mean([d["final_test_accuracy"] for d in
computed_metrics[max_epochs]])\
.astype(np.float32).reshape([1])
# generate all possible graphs and labellings
for vertices in range(2, max_vertices + 1):
for bits in range(2**(vertices * (vertices - 1) // 2)):
# generate an adjacency matrix for the graph
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
# discard graphs which can be pruned or exceed constraints
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > max_edges):
continue
# convert the binary adjacency matrix to a vector
vector = matrix[np.triu_indices(matrix.shape[0], k=1)]
# Iterate through all possible labellings
for labelling in itertools.product(
*[[CONV1X1, CONV3X3, MAXPOOL3X3]
for _ in range(vertices - 2)]):
# convert the graph and labelling to numpy arrays
ops = [INPUT] + list(labelling) + [OUTPUT]
ops = np.array([NODE_TO_ID[t] for t in ops]).astype(np.int32)
# yield samples encoded in a standard sequence format
yield np.concatenate(
[[NODE_TO_ID[START]], ops, [NODE_TO_ID[SEPARATOR]],
vector + NODE_TO_ID[ADJACENCY_ZERO], [NODE_TO_ID[STOP]],
[NODE_TO_ID[PAD]] * (max_vertices - ops.size +
max_adjacency_size - vector.size)],
axis=0), model_to_metric(ops, matrix)
def main(_):
total_graphs = 0 # Total number of graphs (including isomorphisms)
total_unlabeled_graphs = 0 # Total number of unlabeled graphs
# hash --> (matrix, label) for the canonical graph associated with each hash
buckets = {}
logging.info('Using %d vertices, %d op labels, min %d max %d edges',
FLAGS.max_vertices, FLAGS.num_ops, FLAGS.min_edges,
FLAGS.max_edges)
for vertices in range(FLAGS.min_vertices, FLAGS.max_vertices + 1):
for bits in range(2**(vertices * (vertices - 1) // 2)):
if bits % 100000 == 0:
print('bits:', bits)
# Construct adj matrix from bit string
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
# Discard any graphs which can be pruned or exceed constraints
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > FLAGS.max_edges
or graph_util.num_edges(matrix) < FLAGS.min_edges):
continue
# this step should be redundant with is_full_dag()
if graph_util.hanging_edge(matrix):
print(np.array(matrix))
continue
print('found valid ulabeled graph')
print(matrix)
total_unlabeled_graphs += 1
# Iterate through all possible labelings
for labeling in itertools.product(
*[range(FLAGS.num_ops) for _ in range(vertices - 2)]):
total_graphs += 1
labeling = [-1] + list(labeling) + [-2]
fingerprint = graph_util.hash_module(matrix, labeling)
# todo: check if hash is in nasbench
if fingerprint not in buckets:
buckets[fingerprint] = (matrix.tolist(), labeling)
# This catches the "false positive" case of two models which are not
# isomorphic hashing to the same bucket.
elif FLAGS.verify_isomorphism:
canonical_graph = buckets[fingerprint]
if not graph_util.is_isomorphic(
(matrix.tolist(), labeling), canonical_graph):
logging.fatal(
'Matrix:\n%s\nLabel: %s\nis not isomorphic to'
' canonical matrix:\n%s\nLabel: %s', str(matrix),
str(labeling), str(canonical_graph[0]),
str(canonical_graph[1]))
sys.exit()
logging.info('Up to %d vertices: %d graphs (%d without hashing)',
vertices, len(buckets), total_graphs)
logging.info('%d unlabeled graphs', total_unlabeled_graphs)
print('finished')
with tf.io.gfile.GFile(FLAGS.output_file, 'w') as f:
print('outputting now to ', FLAGS.output_file)
json.dump(buckets, f, sort_keys=True)
