def test_counterexample_bipartite(self):
# This is a counter example that shows that the hashing algorithm is not
# perfectly identifiable (i.e. there are non-isomorphic graphs with the same
# hash). If this tests fails, it means the algorithm must have been changed
# in some way that allows it to identify these graphs as non-isomoprhic.
matrix1 = np.array([[0, 1, 1, 1, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
matrix2 = np.array([[0, 1, 1, 1, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 1, 1, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 0],
[0, 0, 0, 0, 0, 1, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
labels = [-1, 1, 1, 1, 1, 2, 2, 2, 2, -2]
# This takes far too long to run so commenting it out. The graphs are
# non-isomorphic fairly obviously from visual inspection.
# self.assertFalse(graph_util.is_isomorphic((matrix1, labels),
# (matrix2, labels)))
self.assertEqual(graph_util.hash_module(matrix1, labels),
graph_util.hash_module(matrix2, labels))
def generate_buckets(vertices, bits, max_edges, num_ops, verify_isomorphism):
buckets = {}
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > max_edges):
return
# Iterate through all possible labelings
for labeling in itertools.product(
*[range(num_ops) for _ in range(vertices - 2)]):
labeling = [-1] + list(labeling) + [-2]
fingerprint = graph_util.hash_module(matrix, labeling)
if fingerprint not in buckets:
buckets[fingerprint] = (matrix.tolist(), labeling)
# This catches the "false positive" case of two models which are not
# isomorphic hashing to the same bucket.
elif verify_isomorphism:
canonical_graph = buckets[fingerprint]
if not graph_util.is_isomorphic(
(matrix.tolist(), labeling), canonical_graph):
logging.fatal(
'Matrix:\n%s\nLabel: %s\nis not isomorphic to'
' canonical matrix:\n%s\nLabel: %s', str(matrix),
str(labeling), str(canonical_graph[0]),
str(canonical_graph[1]))
sys.exit()
return buckets
def test_random_isomorphism_hashing(self):
# Tests that hash_module always provides the same hash for randomly
# generated isomorphic graphs.
for _ in range(1000):
# Generate random graph. Note: the algorithm works (i.e. same hash ==
# isomorphic graphs) for all directed graphs with coloring and does not
# require the graph to be a DAG.
size = random.randint(3, 20)
matrix = np.random.randint(0, 2, [size, size])
labels = [random.randint(0, 10) for _ in range(size)]
# Generate permutation of matrix and labels.
perm = np.random.permutation(size).tolist()
pmatrix, plabels = graph_util.permute_graph(matrix, labels, perm)
# Hashes should be identical.
hash1 = graph_util.hash_module(matrix, labels)
hash2 = graph_util.hash_module(pmatrix, plabels)
self.assertEqual(hash1, hash2)
def generate_graph(max_vertices, max_edges, num_ops, verify_isomorphism,
output_file):
FLAGS = Namespace(max_vertices=max_vertices,
num_ops=num_ops,
max_edges=max_edges,
verify_isomorphism=verify_isomorphism,
output_file=output_file)
total_graphs = 0 # Total number of graphs (including isomorphisms)
# hash --> (matrix, label) for the canonical graph associated with each hash
buckets = {}
logging.info('Using %d vertices, %d op labels, max %d edges',
FLAGS.max_vertices, FLAGS.num_ops, FLAGS.max_edges)
for vertices in range(2, FLAGS.max_vertices + 1):
for bits in range(2**(vertices * (vertices - 1) // 2)):
# Construct adj matrix from bit string
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
# Discard any graphs which can be pruned or exceed constraints
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > FLAGS.max_edges):
continue
# Iterate through all possible labelings
for labeling in itertools.product(
*[range(FLAGS.num_ops) for _ in range(vertices - 2)]):
total_graphs += 1
labeling = [-1] + list(labeling) + [-2]
fingerprint = graph_util.hash_module(matrix, labeling)
if fingerprint not in buckets:
buckets[fingerprint] = (matrix.tolist(), labeling)
# This catches the "false positive" case of two models which are not
# isomorphic hashing to the same bucket.
elif FLAGS.verify_isomorphism:
canonical_graph = buckets[fingerprint]
if not graph_util.is_isomorphic(
(matrix.tolist(), labeling), canonical_graph):
logging.fatal(
'Matrix:\n%s\nLabel: %s\nis not isomorphic to'
' canonical matrix:\n%s\nLabel: %s', str(matrix),
str(labeling), str(canonical_graph[0]),
str(canonical_graph[1]))
sys.exit()
logging.info('Up to %d vertices: %d graphs (%d without hashing)',
vertices, len(buckets), total_graphs)
with open(FLAGS.output_file, 'w') as f:
json.dump(buckets, f, sort_keys=True)
def hash_spec(self, canonical_ops):
"""Computes the isomorphism-invariant graph hash of this spec.
Args:
canonical_ops: list of operations in the canonical ordering which they
were assigned (i.e. the order provided in the config['available_ops']).
Returns:
MD5 hash of this spec which can be used to query the dataset.
"""
# Invert the operations back to integer label indices used in graph gen.
labeling = [-1] + [canonical_ops.index(op) for op in self.ops[1:-1]] + [-2]
return graph_util.hash_module(self.matrix, labeling)
def make_graphs(vertices, bits):
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
if graph_util.num_edges(matrix) > max_edges:
return []
if not graph_util.is_full_dag(matrix):
return []
out = []
for labeling in itertools.product(
*[range(num_ops) for _ in range(vertices - 2)]):
labeling = [-1] + list(labeling) + [-2]
out.append({
"hash": graph_util.hash_module(matrix, labeling),
"adj": matrix.tolist(),
"labeling": labeling,
})
return out
def test_hash_module(self):
# Diamond graph with label permutation
matrix1 = np.array(
[[0, 1, 1, 0,],
[0, 0, 0, 1],
[0, 0, 0, 1],
[0, 0, 0, 0]])
label1 = [-1, 1, 2, -2]
label2 = [-1, 2, 1, -2]
hash1 = graph_util.hash_module(matrix1, label1)
hash2 = graph_util.hash_module(matrix1, label2)
self.assertEqual(hash1, hash2)
# Simple graph with edge permutation
matrix1 = np.array(
[[0, 1, 1, 0, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0]])
label1 = [-1, 1, 2, 3, -2]
matrix2 = np.array(
[[0, 1, 0, 1, 0],
[0, 0, 1, 0, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0]])
label2 = [-1, 2, 3, 1, -2]
matrix3 = np.array(
[[0, 1, 1, 0, 0],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0]])
label3 = [-1, 2, 1, 3, -2]
hash1 = graph_util.hash_module(matrix1, label1)
hash2 = graph_util.hash_module(matrix2, label2)
hash3 = graph_util.hash_module(matrix3, label3)
self.assertEqual(hash1, hash2)
self.assertEqual(hash2, hash3)
hash4 = graph_util.hash_module(matrix1, label2)
self.assertNotEqual(hash4, hash1)
hash5 = graph_util.hash_module(matrix1, label3)
self.assertNotEqual(hash5, hash1)
# Connected non-isomorphic regular graphs on 6 interior vertices (8 total)
matrix1 = np.array(
[[0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 0, 0, 1, 0],
[0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0]])
matrix2 = np.array(
[[0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 1, 1, 0, 1, 0, 0],
[0, 0, 0, 0, 1, 0, 1, 0],
[0, 0, 0, 0, 1, 1, 0, 0],
[0, 0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0]])
label1 = [-1, 1, 1, 1, 1, 1, 1, -2]
hash1 = graph_util.hash_module(matrix1, label1)
hash2 = graph_util.hash_module(matrix2, label1)
self.assertNotEqual(hash1, hash2)
# Non-isomorphic tricky case (breaks if you don't include self)
hash1 = graph_util.hash_module(
np.array([[0, 1, 0, 0, 0],
[0, 0, 1, 0, 0],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0]]),
[-1, 1, 0, 0, -2])
hash2 = graph_util.hash_module(
np.array([[0, 1, 0, 0, 0],
[0, 0, 1, 0, 0],
[0, 0, 0, 1, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 0]]),
[-1, 0, 0, 1, -2])
self.assertNotEqual(hash1, hash2)
# Non-isomorphic tricky case (breaks if you don't use directed edges)
hash1 = graph_util.hash_module(
np.array([[0, 1, 0, 1],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 0, 0, 0]]),
[-1, 1, 0, -2])
hash2 = graph_util.hash_module(
np.array([[0, 1, 0, 1],
[0, 0, 1, 0],
[0, 0, 0, 1],
[0, 0, 0, 0]]),
[-1, 0, 1, -2])
self.assertNotEqual(hash1, hash2)
# Non-isomorphic tricky case (breaks if you only use out-neighbors and self)
hash1 = graph_util.hash_module(np.array([[0, 1, 1, 1, 1, 0, 0],
[0, 0, 1, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0]]),
[-1, 1, 0, 0, 0, 0, -2])
hash2 = graph_util.hash_module(np.array([[0, 1, 1, 1, 1, 0, 0],
[0, 0, 1, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0]]),
[-1, 0, 0, 0, 1, 0, -2])
self.assertNotEqual(hash1, hash2)
def pretraining_gae(dataset, cfg):
""" implementation of VGAE pretraining on DARTS Search Space """
X_adj, X_ops, indices, X_adj_val, X_ops_val, indices_val = _build_dataset(dataset)
print('train set size: {}, validation set size: {}'.format(indices.shape[0], indices_val.shape[0]))
model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.dim,
num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda()
optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08)
epochs = args.epochs
bs = args.bs
loss_total = []
best_graph_acc = 0
for epoch in range(0, epochs):
chunks = X_adj.shape[0] // bs
if X_adj.shape[0] % bs > 0:
chunks += 1
X_adj_split = torch.split(X_adj, bs, dim=0)
X_ops_split = torch.split(X_ops, bs, dim=0)
indices_split = torch.split(indices, bs, dim=0)
loss_epoch = []
Z = []
for i, (adj, ops, ind) in enumerate(zip(X_adj_split, X_ops_split, indices_split)):
optimizer.zero_grad()
adj, ops = adj.cuda(), ops.cuda()
# preprocessing
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
# forward
ops_recon, adj_recon, mu, logvar = model(ops, adj)
Z.append(mu)
adj_recon, ops_recon = prep_reverse(adj_recon, ops_recon)
adj, ops = prep_reverse(adj, ops)
loss = VAEReconstructed_Loss(**cfg['loss'])((ops_recon, adj_recon), (ops, adj), mu, logvar)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 5)
optimizer.step()
loss_epoch.append(loss.item())
if i % 500 == 0:
print('epoch {}: batch {} / {}: loss: {:.5f}'.format(epoch, i, chunks, loss.item()))
Z = torch.cat(Z, dim=0)
z_mean, z_std = Z.mean(0), Z.std(0)
validity_counter = 0
buckets = {}
model.eval()
for _ in range(args.latent_points):
z = torch.randn(11, args.dim).cuda()
z = z * z_std + z_mean
op, ad = model.decoder(z.unsqueeze(0))
op = op.squeeze(0).cpu()
ad = ad.squeeze(0).cpu()
max_idx = torch.argmax(op, dim=-1)
one_hot = torch.zeros_like(op)
for i in range(one_hot.shape[0]):
one_hot[i][max_idx[i]] = 1
op_decode = to_ops_darts(max_idx)
ad_decode = (ad>0.5).int().triu(1).numpy()
ad_decode = np.ndarray.tolist(ad_decode)
if is_valid_darts(ad_decode, op_decode):
validity_counter += 1
fingerprint = graph_util.hash_module(np.array(ad_decode), one_hot.numpy().tolist())
if fingerprint not in buckets:
buckets[fingerprint] = (ad_decode, one_hot.numpy().astype('int8').tolist())
validity = validity_counter / args.latent_points
print('Ratio of valid decodings from the prior: {:.4f}'.format(validity))
print('Ratio of unique decodings from the prior: {:.4f}'.format(len(buckets) / (validity_counter+1e-8)))
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val = get_val_acc_vae(model,cfg,X_adj_val, X_ops_val,indices_val)
print('validation set: acc_ops:{0:.2f}, mean_corr_adj:{1:.2f}, mean_fal_pos_adj:{2:.2f}, acc_adj:{3:.2f}'.format(
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val))
#print("reconstructed adj matrix:", adj_recon[1])
#print("original adj matrix:", adj[1])
#print("reconstructed ops matrix:", ops_recon[1])
#print("original ops matrix:", ops[1])
print('epoch {}: average loss {:.5f}'.format(epoch, sum(loss_epoch)/len(loss_epoch)))
loss_total.append(sum(loss_epoch) / len(loss_epoch))
print('loss for epochs: \n', loss_total)
save_checkpoint_vae(model, optimizer, epoch, sum(loss_epoch) / len(loss_epoch), args.dim, args.name, args.dropout, args.seed)
print('loss for epochs: ', loss_total)
def pretraining_model(dataset, cfg, args):
nasbench = api.NASBench('data/nasbench_only108.tfrecord')
train_ind_list, val_ind_list = range(int(len(dataset)*0.9)), range(int(len(dataset)*0.9), len(dataset))
X_adj_train, X_ops_train, indices_train = _build_dataset(dataset, train_ind_list)
X_adj_val, X_ops_val, indices_val = _build_dataset(dataset, val_ind_list)
model = Model(input_dim=args.input_dim, hidden_dim=args.hidden_dim, latent_dim=args.dim,
num_hops=args.hops, num_mlp_layers=args.mlps, dropout=args.dropout, **cfg['GAE']).cuda()
optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), eps=1e-08)
epochs = args.epochs
bs = args.bs
loss_total = []
for epoch in range(0, epochs):
chunks = len(train_ind_list) // bs
if len(train_ind_list) % bs > 0:
chunks += 1
X_adj_split = torch.split(X_adj_train, bs, dim=0)
X_ops_split = torch.split(X_ops_train, bs, dim=0)
indices_split = torch.split(indices_train, bs, dim=0)
loss_epoch = []
Z = []
for i, (adj, ops, ind) in enumerate(zip(X_adj_split, X_ops_split, indices_split)):
optimizer.zero_grad()
adj, ops = adj.cuda(), ops.cuda()
# preprocessing
adj, ops, prep_reverse = preprocessing(adj, ops, **cfg['prep'])
# forward
ops_recon, adj_recon, mu, logvar = model(ops, adj.to(torch.long))
Z.append(mu)
adj_recon, ops_recon = prep_reverse(adj_recon, ops_recon)
adj, ops = prep_reverse(adj, ops)
loss = VAEReconstructed_Loss(**cfg['loss'])((ops_recon, adj_recon), (ops, adj), mu, logvar)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 5)
optimizer.step()
loss_epoch.append(loss.item())
if i%1000==0:
print('epoch {}: batch {} / {}: loss: {:.5f}'.format(epoch, i, chunks, loss.item()))
Z = torch.cat(Z, dim=0)
z_mean, z_std = Z.mean(0), Z.std(0)
validity_counter = 0
buckets = {}
model.eval()
for _ in range(args.latent_points):
z = torch.randn(7, args.dim).cuda()
z = z * z_std + z_mean
op, ad = model.decoder(z.unsqueeze(0))
op = op.squeeze(0).cpu()
ad = ad.squeeze(0).cpu()
max_idx = torch.argmax(op, dim=-1)
one_hot = torch.zeros_like(op)
for i in range(one_hot.shape[0]):
one_hot[i][max_idx[i]] = 1
op_decode = transform_operations(max_idx)
ad_decode = (ad>0.5).int().triu(1).numpy()
ad_decode = np.ndarray.tolist(ad_decode)
spec = api.ModelSpec(matrix=ad_decode, ops=op_decode)
if nasbench.is_valid(spec):
validity_counter += 1
fingerprint = graph_util.hash_module(np.array(ad_decode), one_hot.numpy().tolist())
if fingerprint not in buckets:
buckets[fingerprint] = (ad_decode, one_hot.numpy().astype('int8').tolist())
validity = validity_counter / args.latent_points
print('Ratio of valid decodings from the prior: {:.4f}'.format(validity))
print('Ratio of unique decodings from the prior: {:.4f}'.format(len(buckets) / (validity_counter+1e-8)))
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val = get_val_acc_vae(model, cfg, X_adj_val, X_ops_val, indices_val)
print('validation set: acc_ops:{0:.4f}, mean_corr_adj:{1:.4f}, mean_fal_pos_adj:{2:.4f}, acc_adj:{3:.4f}'.format(
acc_ops_val, mean_corr_adj_val, mean_fal_pos_adj_val, acc_adj_val))
print('epoch {}: average loss {:.5f}'.format(epoch, sum(loss_epoch)/len(loss_epoch)))
loss_total.append(sum(loss_epoch) / len(loss_epoch))
save_checkpoint_vae(model, optimizer, epoch, sum(loss_epoch) / len(loss_epoch), args.dim, args.name, args.dropout, args.seed)
print('loss for epochs: \n', loss_total)
def main(_):
total_graphs = 0 # Total number of graphs (including isomorphisms)
total_unlabeled_graphs = 0 # Total number of unlabeled graphs
# hash --> (matrix, label) for the canonical graph associated with each hash
buckets = {}
logging.info('Using %d vertices, %d op labels, min %d max %d edges',
FLAGS.max_vertices, FLAGS.num_ops, FLAGS.min_edges,
FLAGS.max_edges)
for vertices in range(FLAGS.min_vertices, FLAGS.max_vertices + 1):
for bits in range(2**(vertices * (vertices - 1) // 2)):
if bits % 100000 == 0:
print('bits:', bits)
# Construct adj matrix from bit string
matrix = np.fromfunction(graph_util.gen_is_edge_fn(bits),
(vertices, vertices),
dtype=np.int8)
# Discard any graphs which can be pruned or exceed constraints
if (not graph_util.is_full_dag(matrix)
or graph_util.num_edges(matrix) > FLAGS.max_edges
or graph_util.num_edges(matrix) < FLAGS.min_edges):
continue
# this step should be redundant with is_full_dag()
if graph_util.hanging_edge(matrix):
print(np.array(matrix))
continue
print('found valid ulabeled graph')
print(matrix)
total_unlabeled_graphs += 1
# Iterate through all possible labelings
for labeling in itertools.product(
*[range(FLAGS.num_ops) for _ in range(vertices - 2)]):
total_graphs += 1
labeling = [-1] + list(labeling) + [-2]
fingerprint = graph_util.hash_module(matrix, labeling)
# todo: check if hash is in nasbench
if fingerprint not in buckets:
buckets[fingerprint] = (matrix.tolist(), labeling)
# This catches the "false positive" case of two models which are not
# isomorphic hashing to the same bucket.
elif FLAGS.verify_isomorphism:
canonical_graph = buckets[fingerprint]
if not graph_util.is_isomorphic(
(matrix.tolist(), labeling), canonical_graph):
logging.fatal(
'Matrix:\n%s\nLabel: %s\nis not isomorphic to'
' canonical matrix:\n%s\nLabel: %s', str(matrix),
str(labeling), str(canonical_graph[0]),
str(canonical_graph[1]))
sys.exit()
logging.info('Up to %d vertices: %d graphs (%d without hashing)',
vertices, len(buckets), total_graphs)
logging.info('%d unlabeled graphs', total_unlabeled_graphs)
print('finished')
with tf.io.gfile.GFile(FLAGS.output_file, 'w') as f:
print('outputting now to ', FLAGS.output_file)
json.dump(buckets, f, sort_keys=True)
for line in sys.stdin:
arch = json.loads(line)
hash_ = arch['hash']
matrix = np.vstack(arch['adj'])
labeling = deepcopy(arch['labeling'])
# --
# Operator exchanges
for i in range(1, len(labeling) - 1):
for op in range(num_ops):
if labeling[i] == op:
continue
l = copy(labeling)
l[i] = op
print(hash_, graph_util.hash_module(matrix, l))
# --
# Edge exchanges
for i in range(matrix.shape[0]):
for j in range(i + 1, matrix.shape[1]):
matrix[i, j] = 1 - matrix[i, j]
print(hash_, graph_util.hash_module(matrix, labeling))
matrix[i, j] = 1 - matrix[i, j]
sys.stdout.flush()
