def test_inverses(self):
"""
Verify that destringize inverts stringize.
"""
identical = lambda x, y: x == y
copied = DECORATED.copy()
assert iso.is_isomorphic(copied, DECORATED, identical, identical)
pydevDAG.Rewriter.stringize(copied)
assert not iso.is_isomorphic(copied, DECORATED, identical, identical)
pydevDAG.Rewriter.destringize(copied)
assert iso.is_isomorphic(copied, DECORATED, identical, identical)
def _is_single_label_isomorphic(
graph: nx.Graph, other: nx.Graph,
graph_query_params: "GraphQueryParams") -> bool:
"""
Returns True if and only if graphs are isomorphic and specified label
in graph_query_params are identical.
"""
if graph_query_params.graph_component == 'node':
return iso.is_isomorphic(
graph, other, node_match=graph_query_params.get_match_function())
elif graph_query_params.graph_component == 'edge':
return iso.is_isomorphic(
graph, other, edge_match=graph_query_params.get_match_function())
def ___non_isomorphic_graphs_dict(nx_objects):
# nx_objects = [nx.relabel.convert_node_labels_to_integers(o, label_attribute='number') for o in objects]
# Add labels as attributes
for g in nx_objects:
node_number = len(g.nodes())
labels = random.sample(range(node_number), node_number)
i = 0
for n in g.nodes():
g.nodes[n]['number'] = labels[i]
i += 1
# Filter out isomorphic graphs
nm = iso.numerical_node_match('number', 0)
graphs = dict()
checked = []
for g in nx_objects:
if g not in checked:
isomorphic = [
x for x in nx_objects if iso.is_isomorphic(x, g, node_match=nm)
]
graphs[g] = len(isomorphic)
checked = checked + isomorphic
return graphs
def get_graph_entry(graph_params):
# select or insert row in graph
adjset0 = [f"{i[0]} {i[1]}" for i in graph_params["adjacency"]]
G0 = parse_edgelist(adjset0)
existing_graphs = graph_Graph.objects.filter(
total_vertices=graph_params["total_vertices"])
for egraph in existing_graphs:
adjset = [f"{edge[0]} {edge[1]}" for edge in egraph.adjacency]
Gi = parse_edgelist(adjset)
if is_isomorphic(G0, Gi):
print("Return exising graph")
return egraph
graph, _ = graph_Graph.objects.get_or_create(
tag=graph_params[
"tag"], # Tag for graph type (e.g. Hamming(n,m) or K(n,m))
total_vertices=graph_params[
"total_vertices"], # Total number of vertices in graph
total_edges=graph_params[
"total_edges"], # Total number of edges in graph
max_edges=graph_params[
"max_edges"], # Maximum number of edges per vertex
adjacency=graph_params[
"adjacency"], # Sorted adjacency matrix of dimension [N, 2]
adjacency_hash=graph_params[
"adjacency_hash"], # md5 hash of adjacency list used for unique constraint
)
print("Creating new graph")
return graph
def isomorphic(self, other, consider_observations=False):
"""Check if two games have isomorphic graphs with regards to nodes and edges
other -- the other game
consider_observations -- if true, the equivalence relations from the observations must be the same in both graphs as well"""
if len(self.states) != len(other.states):
return False
a = self.graph.copy()
b = other.graph.copy()
for g in ((a, self), (b, other)):
if consider_observations:
for player, partitioning in enumerate(g[1].partitionings):
for observation in partitioning:
if len(observation) > 1:
for start, end in permutations(observation, 2):
g[0].add_edge(start, end, player=str(player))
g[0].add_node("hidden")
g[0].add_edge("hidden", g[1].initial_state)
State.orderable = True
iso = is_isomorphic(a, b, edge_match=lambda x, y: x == y)
State.orderable = False
return iso
def test_graphml(self):
output = convert(
'graph', self.test_input['distances'], {'format': 'graphml'})
expected_edges = set(self.test_input['distances']['data'].edges(
data='distance'))
actual_edges = set()
self.assertIsInstance(output['data'], (str, unicode))
tree = etree.fromstring(output['data'])
self.assertEqual(len(tree), 2)
self.assertEqual(tree[0].tag, self.GRAPHML_NS + 'key')
self.assertEqual(tree[1].tag, self.GRAPHML_NS + 'graph')
for edge in tree[1].findall(self.GRAPHML_NS + 'edge'):
edge = (edge.attrib['source'],
edge.attrib['target'],
int(edge.find(self.GRAPHML_NS + 'data').text))
self.assertNotIn(edge, actual_edges)
actual_edges.add(edge)
self.assertEqual(expected_edges, actual_edges)
output = convert(
'graph', output, {'format': 'networkx'})
self.assertTrue(
is_isomorphic(output['data'],
self.test_input['distances']['data'],
edge_match=numerical_edge_match('distance', 1)))
def uniquelist(l1, l2, fva):
l = []
temp = []
# li=[]
nm = iso.categorical_node_match('elem', 'C')
em = iso.numerical_edge_match('weight', 1)
for i, k in zip(l1, fva):
flag = 1
# c=0
for j in l2:
# c+=1
# GM = nx.algorithms.isomorphism.GraphMatcher(i,j)
if iso.is_isomorphic(i, j, edge_match=em, node_match=nm):
# print "1"
flag = 0
# li.append(c)
break
if flag == 1:
l.append(i)
temp.append(k)
# print len(l)
# print len(li)
# print li
# c=0
# for i in range(0,len(l2)):
# c=0
# if i+1 not in li:
# l.append(l2[c])
# print "hola"
# else:
# c+=1
return l, temp
def test_adjacencylist(self):
output = convert(
'graph', self.test_input['distances'], {'format': 'adjacencylist'})
expected_edges = set(self.test_input['distances']['data'].edges())
actual_edges = set()
for line in output['data'].splitlines():
parts = line.split(' ', 1)
if len(parts) > 1:
source, targets = parts
for target in targets.split(' '):
edge = (source, target)
self.assertNotIn(edge, actual_edges)
actual_edges.add(edge)
self.assertEqual(expected_edges, actual_edges)
output = convert(
'graph', output, {'format': 'networkx'})
# Don't take edges into consideration, because they were lost in the
# original conversion
self.assertTrue(
is_isomorphic(output['data'],
self.test_input['distances']['data'],
edge_match=None))
def subgraphIsomorphismCheck(self, G1, G2):
"""
Checks whether G1 contains a subgraph isomorphic to G2 by using Whitney isomorphism theorem.
Parameters:
G1 (NetworkX graph): The bigger graph.
G2 (NetworkX graph): The smaller graph.
Returns:
bool: Is graph G2 isomorphic to a subgraph in G1.
isomorphism.GraphMatcher: Graph matcher object with parameters.
"""
# transform graphs into line graphs and check for subgraph isomorphism
# isomorphism.GraphMatcher tries to find an induced subgraph of G1, such that it is isomorphic to G2.
# Consequently, if G2 is non-induced subgraph of G1, the algorithm will return False
GM = isomorphism.GraphMatcher(nx.line_graph(G1), nx.line_graph(G2))
subgraph_is_iso = GM.subgraph_is_isomorphic()
# check for exceptions
# e.g. line graphs of K_3 triangle graph and K_1,3 claw graph are isomorphic, but the original graphs are not
if subgraph_is_iso:
edgeListG1 = []
edgeListG2 = []
for edgeMaping in GM.mapping.items():
edgeListG1.append(edgeMaping[0])
edgeListG2.append(edgeMaping[1])
# let's construct the graphs the algorithm thinks are isomorphic and check them for a quick isomorphism.is_isomorphic
testG1 = nx.Graph(edgeListG1)
testG2 = nx.Graph(edgeListG2)
subgraph_is_iso = isomorphism.is_isomorphic(testG1, testG2)
return subgraph_is_iso, GM
def test_graphml(self):
output = convert(
'graph', self.test_input['distances'], {'format': 'graphml'})
expected_edges = set(self.test_input['distances']['data'].edges(
data='distance'))
actual_edges = set()
self.assertIsInstance(output['data'], (str, unicode))
tree = etree.fromstring(output['data'])
self.assertEqual(len(tree), 2)
self.assertEqual(tree[0].tag, self.GRAPHML_NS + 'key')
self.assertEqual(tree[1].tag, self.GRAPHML_NS + 'graph')
for edge in tree[1].findall(self.GRAPHML_NS + 'edge'):
edge = (edge.attrib['source'],
edge.attrib['target'],
int(edge.find(self.GRAPHML_NS + 'data').text))
self.assertNotIn(edge, actual_edges)
actual_edges.add(edge)
self.assertEqual(expected_edges, actual_edges)
output = convert(
'graph', output, {'format': 'networkx'})
self.assertTrue(
is_isomorphic(output['data'],
self.test_input['distances']['data'],
edge_match=numerical_edge_match('distance', 1)))
def find_vertex_types(T: nx.DiGraph) -> Dict[int, List[int]]:
"""
Computes the vertex-types and the vertex-type of each vertex of a given tree. Input tree needs to be directed
because otherwise the dfs will also go back to the root of the tree, instead of just going downwards.
:param T: the input tree.
:return: a dictionary, where each key is a vertex-type and its value is a list containing all nodes of that type.
"""
assert nx.algorithms.tree.is_tree(T), 'Input graph should be a tree.'
types = dict()
for n in nx.nodes(T):
if len(types.keys()) == 0:
types[len(types.keys())] = [n]
continue
flag = True
for t in types.keys():
if is_isomorphic(dfs_tree(T, types[t][0]), dfs_tree(T, n)):
types[t].append(n)
flag = False
break
if flag:
types[len(types.keys())] = [n]
return types
def test_adjacencylist(self):
output = convert(
'graph', self.test_input['distances'], {'format': 'adjacencylist'})
expected_edges = set(self.test_input['distances']['data'].edges())
actual_edges = set()
for line in output['data'].splitlines():
parts = line.split(' ', 1)
if len(parts) > 1:
source, targets = parts
for target in targets.split(' '):
edge = (source, target)
self.assertNotIn(edge, actual_edges)
actual_edges.add(edge)
self.assertEqual(expected_edges, actual_edges)
output = convert(
'graph', output, {'format': 'networkx'})
# Don't take edges into consideration, because they were lost in the
# original conversion
self.assertTrue(
is_isomorphic(output['data'],
self.test_input['distances']['data'],
edge_match=None))
def combined_if_do_program_graph_test():
s = '''
active proctype p(){
int x, y, z;
if /* 0 */
:: do
:: x = 1; /* 2 */
y == 5 /* 1 */
:: z = 3; /* 3 */
skip /* 1 */
:: if
:: z = (3 - x) * y; /* 4 */
true; /* 5 */
y = 3 /* 1 */
:: true /* 1 */
fi
od
/* 1 */
:: true; /* 6 */
if
:: true /* 7 */
:: true -> /* 8 */
x = y /* 7 */
fi
fi
/* 7 */
}
'''
tree = parser.parse(s)
g = tree[0].to_pg()
dump(g)
h = nx.MultiDiGraph()
h.add_edges_from([
(0, 2),
(0, 3),
(0, 4),
(0, 1),
(2, 1),
(3, 1),
(5, 1),
(1, 2),
(1, 3),
(1, 4),
(1, 1),
(4, 5),
# false; if ...
(0, 6),
(6, 7),
(6, 8),
(8, 7)
])
dump(g, node_label=None)
assert iso.is_isomorphic(g, h)
def nx_is_equal(g, h, isomorphic=False):
from networkx.algorithms.isomorphism import is_isomorphic
if isomorphic:
return is_isomorphic(g, h, lambda ga, ha: ga == ha)
g_edges, h_edges = g.edges(), h.edges()
return all(h_edge in g_edges
for h_edge in h_edges) and all(g_edge in h_edges
for g_edge in g_edges)
def graphs_isomorphic(dfdag1, dfdag2):
"""
Graphs are isomorphic and the nodes are equal. We should test the tests...
"""
g1 = dfdag1.nx_representation()
g2 = dfdag2.nx_representation()
# TODO add a generic node match helper...
return iso.is_isomorphic(g1,g2)
def getValue(self, graph):
''' Lookup polynomial evaluation or calculate value if graph size is 1.'''
if graph.number_of_edges() == 0:
return 1
try:
for candidate, value in self.knownGraphs[(graph.number_of_edges(), graph.number_of_nodes())]:
if isomorphism.faster_could_be_isomorphic(candidate, graph):
if isomorphism.is_isomorphic(candidate, graph):
return value
except KeyError:
raise KeyError
def test_inverses(self):
"""
Test that writing the string and then reading it yields identical graph.
"""
val = pydevDAG.StringUtils.as_string(DECORATED, pydevDAG.Writer.write)
res = pydevDAG.StringUtils.from_string(val, pydevDAG.Reader.read)
assert iso.is_isomorphic(
DECORATED,
res,
lambda x, y: x == y,
lambda x, y: x == y
)
def is_isomorphic(self, g1, g2, approximate=False):
"""Check if two graphs are isomorphic.
To accelerate the process, we use an approximate algorithm,
whose False value means definitely not isomorphic while True
value does not guarantee isomorphic for 100%.
"""
if approximate:
return isomorphism.faster_could_be_isomorphic(g1, g2)
else:
if isomorphism.faster_could_be_isomorphic(g1, g2):
if isomorphism.is_isomorphic(g1, g2):
return True
return False
def filter_isomorphisms(self, graphs):
# This code is taken from stack overflow
# https://stackoverflow.com/questions/46999771/comparing-a-large-number-of-graphs-for-isomorphism
unique = [] # A list of unique graphs
for new in graphs:
for old in unique:
if iso.is_isomorphic(new, old):
break
else:
unique.append(new)
return unique
def is_isomorphic(self, g1, g2, approximate=False):
"""Check if two graphs are isomorphic.
To accelerate the process, we use an approximate algorithm,
whose False value means definitely not isomorphic while True
value does not guarantee isomorphic for 100%.
"""
if approximate:
return isomorphism.faster_could_be_isomorphic(g1, g2)
else:
if isomorphism.faster_could_be_isomorphic(g1, g2):
if isomorphism.is_isomorphic(g1, g2):
return True
return False
def _test_insert_edges(test: unittest.TestCase, cls: Type[Algorithm]):
g = nx.gnp_random_graph(20, 0.3, seed=42)
g_original = g.copy()
insert_order = np.random.RandomState(seed=42).permutation(g.edges)
g.remove_edges_from(g.edges)
algo = cls(g)
for e in insert_order:
algo.insert_edge(*e)
valid = algo.is_valid_mis()
test.assertTrue(valid)
test.assertTrue(iso.is_isomorphic(g, g_original))
def is_isomorph_nx(graph1, graph2):
"""
graph1, graph2: графы в формате networkx, изоморфность которых проверяется
return: True, если графы изоморфны, иначе False
"""
is_iso = nx.faster_could_be_isomorphic(graph1, graph2)
node_match = iso.categorical_node_match('label', 'C')
edge_match = iso.categorical_edge_match(['weight', 'label'], [1, '-'])
if is_iso:
return iso.is_isomorphic(graph1,
graph2,
node_match=node_match,
edge_match=edge_match)
return False
def _test_insert_nodes(test: unittest.TestCase, cls: Type[Algorithm]):
g = nx.gnp_random_graph(20, 0.3, seed=42)
g_original = g.copy()
insert_order = np.random.RandomState(seed=42).permutation(g.nodes)
edges = dict()
for v in g.nodes:
edges[v] = {(v, n) for n in g[v]}
g.clear()
algo = cls(g)
for v in insert_order:
algo.insert_node(v, edges[v])
valid = algo.is_valid_mis()
test.assertTrue(valid)
test.assertTrue(iso.is_isomorphic(g, g_original))
def isomorph(a, b):
f1 = open(a, 'r')
s1 = f1.read()
f2 = open(b, 'r')
s2 = f2.read()
output = graph_data(s1, s2)
refined_nodes1 = output[0]
refined_edges1 = output[1]
refined_nodes2 = output[2]
refined_edges2 = output[3]
G1 = nx.DiGraph()
G2 = nx.DiGraph()
G1.add_nodes_from(refined_nodes1)
G1.add_edges_from(refined_edges1)
G2.add_nodes_from(refined_nodes2)
G2.add_edges_from(refined_edges2)
return is_isomorphic(G1, G2)
def is_equivalent(cls, graph1, graph2, node_match, edge_match):
"""
Whether these graphs represent equivalent storage configurations.
:param graph1: a graph
:param graph2: a graph
:param node_match: a function that checks whether nodes are equal
:type node_match: node * node -> bool
:param edge_match: a function that checks whether edges are equal
:type edge_match: node * node -> bool
:returns: True if the graphs are equivalent, otherwise False
:rtype: bool
"""
return iso.is_isomorphic(
graph1,
graph2,
node_match,
edge_match
)
def is_label_isomorphic(
graph: nx.Graph, other: nx.Graph,
graph_query_paramses: Sequence["GraphQueryParams"]) -> bool:
"""
Returns True if and only if graphs are isomorphic and specified labels
in all of the graph_label_matchers are identical for the two graphs.
Wrapper around is_label_matched.
Parameters
----------
graph :
other :
graph_query_paramses :
List of GraphQueryParam's
"""
res = []
res.append(iso.is_isomorphic(graph, other))
for query_params in graph_query_paramses:
res.append(_is_single_label_isomorphic(graph, other, query_params))
return np.all(res)
def equal(self, ns):
"""Compares the NS with the one passed by argument
:ns: NS instance
:returns: True/False
"""
same = True
same = same and iso.is_isomorphic(
self.getChain(),
ns.getChain(),
node_match=lambda n1Res, n2Res: n1Res == n2Res,
edge_match=lambda ed1Res, ed2Res: ed1Res == ed2Res)
same = same and self.__branches == ns._NS__branches
same = same and self.__splits == ns._NS__splits
same = same and self.__maxSplitW == ns._NS__maxSplitW
same = same and self.__prevNeighsCache == ns._NS__prevNeighsCache
same = same and self.__nextNeighsCache == ns._NS__nextNeighsCache
same = same and self.__branchHeads == ns._NS__branchHeads
return same
def test_is_isomorphic(self):
assert_true(iso.is_isomorphic(self.G1,self.G2))
assert_false(iso.is_isomorphic(self.G1,self.G4))
def __eq__(self, other):
return is_isomorphic(self, other, node_match=self._node_matcher)
def __eq__(self, other: 'GraphPattern') -> bool:
return ism.is_isomorphic(self._graph, other.graph)
def createSortedGraph(g):
gToReturn = nx.Graph()
for u in sorted(g.nodes()): gToReturn.add_node(u, {'w': g.node[u]['w']})
for u, v in g.edges(): gToReturn.add_edge(u, v, {'w': g.edge[u][v]['w']})
assert iso.is_isomorphic(gToReturn,g, edge_match=lambda e1,e2: e1['w']==e2['w'], node_match=lambda u,v: u['w']==v['w'])
return gToReturn
def test_is_isomorphic(self):
assert iso.is_isomorphic(self.G1, self.G2)
assert not iso.is_isomorphic(self.G1, self.G4)
def test_is_isomorphic(self):
assert_true(iso.is_isomorphic(self.G1, self.G2))
assert_false(iso.is_isomorphic(self.G1, self.G4))
def is_isomorphic(G1, G2):
return iso.is_isomorphic(G1, G2)
def is_label_isomorphic(G1, G2):
return iso.is_isomorphic(G1, G2, edge_match=iso.categorical_multiedge_match("label", None))
def are_genome_graphs_isomorphic(left, right):
return iso.is_isomorphic(left, right, edge_match=lambda l, r: l == r)
def regularized_evolution(acquisition_func,
observed_archs,
observed_archs_unpruned=None,
benchmark='nasbench101',
pool_size=200,
cycles=40,
n_mutation=10,
batch_size=1,
mutate_unpruned_arch=True):
"""Algorithm for regularized evolution (i.e. aging evolution).
Follows "Algorithm 1" in Real et al. "Regularized Evolution for Image
Classifier Architecture Search".
"""
# Generate some random archs into the evaluation pool
if mutate_unpruned_arch and observed_archs_unpruned is None:
raise ValueError(
"When mutate_unpruned_arch option is toggled on, you need to supplied the list of unpruned "
"observed architectures.")
if observed_archs_unpruned is not None:
assert len(observed_archs_unpruned) == len(observed_archs), " unequal length between the pruned/unpruned " \
"architecture lists"
n_random_archs = pool_size - len(observed_archs)
if mutate_unpruned_arch:
(random_archs, _,
random_archs_unpruned) = random_sampling(pool_size=n_random_archs,
benchmark=benchmark,
return_unpruned_archs=True)
population_unpruned = observed_archs_unpruned + random_archs_unpruned
else:
(random_archs, _, _) = random_sampling(
pool_size=n_random_archs,
benchmark=benchmark,
)
population_unpruned = None
population = observed_archs + random_archs
# Fill the population with the observed archs (a list of labelled graphs) and validation error
population_performance = []
for i, archs in enumerate(population):
arch_acq = acquisition_func.eval(archs, asscalar=True)
population_performance.append(arch_acq)
# Carry out evolution in cycles. Each cycle produces a bat model and removes another.
k_cycle = 0
while k_cycle < cycles:
# Sample randomly chosen models from the current population based on the acquisition function values
pseudo_prob = np.array(population_performance) / (
np.sum(population_performance))
if mutate_unpruned_arch:
samples = random.choices(population_unpruned,
weights=pseudo_prob,
k=30)
sample_indices = [population_unpruned.index(s) for s in samples]
else:
samples = random.choices(population, weights=pseudo_prob, k=30)
sample_indices = [population.index(s) for s in samples]
sample_performance = [
population_performance[idx] for idx in sample_indices
]
# The parents is the best n_mutation model in the sample. skip 2-node archs
top_n_mutation_archs_indices = np.argsort(sample_performance)[
-n_mutation:] # argsort>ascending
parents_archs = [
samples[idx] for idx in top_n_mutation_archs_indices
if len(samples[idx].nodes) > 3
]
# Create the child model and store it.
for parent in parents_archs:
child, child_unpruned = mutate_arch(parent, benchmark)
# skip invalid architectures whose number of edges exceed the max limit of 9
if np.sum(nx.to_numpy_array(child)) > MAX_EDGES:
continue
if iso.is_isomorphic(child, parent):
continue
skip = False
for prev_edit in population:
if iso.is_isomorphic(
child,
prev_edit,
):
skip = True
break
if skip: continue
child_arch_acq = acquisition_func.eval(child, asscalar=True)
population.append(child)
if mutate_unpruned_arch:
population_unpruned.append(child_unpruned)
population_performance.append(child_arch_acq)
# Remove the worst performing model and move to next evolution cycle
worst_n_mutation_archs_indices = np.argsort(
population_performance)[:n_mutation]
for bad_idx in sorted(worst_n_mutation_archs_indices, reverse=True):
population.pop(bad_idx)
population_performance.pop(bad_idx)
if mutate_unpruned_arch:
population_unpruned.pop(bad_idx)
# print(f'len pop = {len(population)}')
k_cycle += 1
# choose batch_size archs with highest acquisition function values to be evaluated next
best_archs_indices = np.argsort(population_performance)[-batch_size:]
recommended_pool = [
population[best_idx] for best_idx in best_archs_indices
]
if mutate_unpruned_arch:
recommended_pool_unpruned = [
population_unpruned[best_idx] for best_idx in best_archs_indices
]
return (recommended_pool,
recommended_pool_unpruned), (population, population_unpruned,
population_performance)
return (recommended_pool, None), (population, None, population_performance)
def assert_same_graph(graph, ref_graph):
assert graph.graph == ref_graph.graph
assert list(graph.nodes) == list(ref_graph.nodes)
assert list(graph.edges) == list(ref_graph.edges)
assert nxiso.is_isomorphic(graph, ref_graph,
node_match=equal_attributes, edge_match=equal_attributes)
def top_compare():
""" Compare top motif and mesostructure
"""
datapath = "data/mesos0825_s0dot2"
movdata = "data/hcl_mesos0825_sample0.2"
bsmap = "data/hcl_mesos0825_bm"
ofname = os.path.join(datapath, "mesos0825_s0dot2_top")
mobgraphs = {}
for person in movement_reader(open(movdata), BaseStationMap(bsmap)):
if person.which_day() != "0825":
continue
nn = len(set(person.locations))
if nn > 20:
continue
if nn not in mobgraphs:
mobgraphs[nn] = {}
mobgraphs[nn][person.id] = person.convert2graph()
new_file = True
for C in range(2, 16):
for kn in range(1, 5):
print C, kn
# Read dist matrix for (group, cluster) users
fileklab = os.path.join(datapath, "mesos0825_s0dot2_c%d_kn%d" % (C, kn))
distmat = []
i = 0
for line in open(fileklab):
if i == 0:
uids = [int(i) for i in line.strip("\r\n").split(",")]
i == 1
continue
distmat.append([float(i) for i in line.strip("\r\n").split(",")])
distmat = np.array(distmat)
distvec = distmat.sum(1) / len(uids)
uids_sorted = [x for (y, x) in sorted(zip(distvec, uids))]
N = len(uids_sorted)
print ("Total users %d: " % N)
mgs = mobgraphs[C]
mesos = Mesos(mgs[uids_sorted[0]], mgs[uids_sorted[1]])
topmesos = mesos.mesos
topmesos_sim = 1 - mesos.struct_dist()
motifs = {}
for i in range(N - 1):
u1 = uids_sorted[i]
u2 = uids_sorted[i + 1]
g1 = mgs[u1]
g2 = mgs[u2]
mesos = Mesos(g1, g2).mesos
found = False
for key in motifs.keys():
if isomorphism.is_isomorphic(key, mesos):
motifs[key].append((mesos, i))
found = True
if found:
break
if not found:
motifs[mesos] = [(mesos, i)]
res = []
for key, value in motifs.items():
res.append((len(value), value[0][0]))
res = sorted(res, key=lambda x: x[0], reverse=True)
topmotif = res[0][1]
topmotif_supp = 1.0 * res[0][0] / N
if new_file:
mode = "wb"
new_file = False
else:
mode = "ab"
ofile = open(ofname, mode)
ofile.write("%d\t%d" % (C, kn))
ofile.write("\t%.3f\t%.3f" % (topmesos_sim, topmotif_supp))
ofile.write("\t%s" % dumps_mobgraph(topmesos))
ofile.write("\t%s" % dumps_mobgraph(topmotif))
ofile.write("\n")
ofile.close()
def mutation(observed_archs,
observed_errors,
n_best=10,
n_mutate=None,
pool_size=250,
allow_isomorphism=False,
patience=50,
benchmark='nasbench101',
observed_archs_unpruned=None):
"""
BANANAS-style mutation.
The main difference with the previously implemented evolutionary algorithms is that it allows unlimited number of
edits based on the parent architecture. For previous implementations, we only allow 1-edit distance mutation.
(although *in expectation*, there is only one edit.)
"""
if n_mutate is None:
n_mutate = int(0.5 * pool_size)
assert pool_size >= n_mutate, " pool_size must be larger or equal to n_mutate"
def _banana_mutate(arch, mutation_rate=1.0, benchmark='nasbench101'):
"""Bananas Style mutation of the cell"""
if benchmark == 'nasbench201':
parent_arch_list = arch.name.split("|")
op_label_rebuild = [
str_i[:-2] for str_i in parent_arch_list if "~" in str_i
]
mutation_prob = mutation_rate / len(OPS_201)
child = None
while True:
try:
for idx, parent_choice in enumerate(op_label_rebuild):
ops_choices = OPS_201[:]
ops_choices.remove(parent_choice)
# Flip parameter with probability of mutation prob
if random.random() < mutation_prob:
choice_idx = np.random.randint(len(ops_choices))
op_label_rebuild[idx] = ops_choices[choice_idx]
child = create_nasbench201_graph(
op_label_rebuild,
edge_attr=arch.graph_type == 'edge_attr')
break
except:
continue
child_unpruned = child
return child, child_unpruned
elif benchmark == 'nasbench101':
while True:
new_ops = list(
nx.get_node_attributes(arch, 'op_name').values())
new_matrix = np.array(nx.adjacency_matrix(arch).todense())
vertice = min(VERTICES, new_matrix.shape[0])
if vertice > 2:
edge_mutation_prob = mutation_rate / vertice
for src in range(0, vertice - 1):
for dst in range(src + 1, vertice):
if random.random() < edge_mutation_prob:
new_matrix[src, dst] = 1 - new_matrix[src, dst]
op_mutation_prob = mutation_rate / (vertice - 2)
for ind in range(1, vertice - 1):
if random.random() < op_mutation_prob:
available = [o for o in OPS if o != new_ops[ind]]
new_ops[ind] = random.choice(available)
try:
pruned_adjacency_matrix, pruned_node_labeling = prune(
new_matrix, new_ops)
child_arch = nx.from_numpy_array(pruned_adjacency_matrix,
create_using=nx.DiGraph)
child_arch_unpruned = nx.from_numpy_array(
new_matrix, create_using=nx.DiGraph)
for i, n in enumerate(pruned_node_labeling):
child_arch.nodes[i]['op_name'] = n
for i, n in enumerate(new_ops):
child_arch_unpruned.nodes[i]['op_name'] = n
return child_arch, child_arch_unpruned
except:
continue
else:
raise NotImplementedError("Search space " + str(benchmark) +
" not implemented!")
population = collections.deque()
# Fill the population with the observed archs (a list of labelled graphs) and validation error
for i, archs in enumerate(observed_archs):
model = Model()
model.arch = archs
model.unpruned_arch = observed_archs_unpruned[
i] if observed_archs_unpruned is not None else None
model.error = observed_errors[i]
population.append(model)
best_archs = [arch for arch in sorted(population, key=lambda i: -i.error)
][:n_best]
evaluation_pool, evaluation_pool_unpruned = [], []
per_arch = n_mutate // n_best
for arch in best_archs:
n_child = 0
patience_ = patience
while n_child < per_arch and patience_ > 0:
child = Model()
child.arch, child.unpruned_arch = _banana_mutate(
arch.unpruned_arch
if observed_archs_unpruned is not None else arch.arch,
benchmark=benchmark,
)
# skip 2-node archs
if benchmark == 'nasbench101':
if len(
child.arch.nodes
) <= 3: # Skip input-output 3-graphs in nasbench101 search space
patience_ -= 1
continue
if np.sum(np.array(nx.adjacency_matrix(
child.arch).todense())) > MAX_EDGES:
patience_ -= 1
continue
elif benchmark == 'nasbench201':
if len(child.arch.nodes
) == 0: # Skip empty graphs in nasbench201 search space
patience_ -= 1
continue
skip = False
if not allow_isomorphism:
# if disallow isomorphism, we enforce that each time, we mutate n distinct graphs. For now we do not
# check the isomorphism in all of the previous graphs though
if benchmark == 'nasbench201':
name = child.arch.name
if name == arch.arch.name:
patience_ -= 1
continue
prev_pool_names = (prev_edit.name
for prev_edit in evaluation_pool)
if name in prev_pool_names:
patience_ -= 1
continue
elif benchmark == 'nasbench101':
if iso.is_isomorphic(child.arch, arch.arch):
# and desirable to simply do Weisfiler-Lehman Isomorphism test, since we are based on
# Weisfeiler-Lehman graph kernel already and the isomorphism test is essentially free.
patience_ -= 1
continue
for prev_edit in evaluation_pool:
if iso.is_isomorphic(
child.arch,
prev_edit,
):
skip = True
break
if skip:
patience_ -= 1
continue
child.error = 0
evaluation_pool.append(child.arch)
evaluation_pool_unpruned.append(child.unpruned_arch)
n_child += 1
# Add some random archs into the evaluation pool, if either 1) patience is reached or 2)
nrandom_archs = max(pool_size - len(evaluation_pool), 0)
if nrandom_archs:
random_evaluation_pool, _, random_evaluation_pool_unpruned = random_sampling(
pool_size=nrandom_archs,
benchmark=benchmark,
return_unpruned_archs=True)
evaluation_pool += random_evaluation_pool
evaluation_pool_unpruned += random_evaluation_pool_unpruned
if observed_archs_unpruned is None:
return evaluation_pool, [None] * len(evaluation_pool)
return evaluation_pool, evaluation_pool_unpruned
def top_compare():
""" Compare top motif and mesostructure
"""
datapath = 'data/mesos0825_s0dot2'
movdata = 'data/hcl_mesos0825_sample0.2'
bsmap = 'data/hcl_mesos0825_bm'
ofname = os.path.join(datapath, 'mesos0825_s0dot2_top')
mobgraphs = {}
for person in movement_reader(open(movdata), BaseStationMap(bsmap)):
if person.which_day() != '0825':
continue
nn = len(set(person.locations))
if nn > 20:
continue
if nn not in mobgraphs:
mobgraphs[nn] = {}
mobgraphs[nn][person.id] = person.convert2graph()
new_file = True
for C in range(2, 16):
for kn in range(1, 5):
print C, kn
# Read dist matrix for (group, cluster) users
fileklab = os.path.join(datapath,
'mesos0825_s0dot2_c%d_kn%d' % (C, kn))
distmat = []
i = 0
for line in open(fileklab):
if i == 0:
uids = [int(i) for i in line.strip('\r\n').split(',')]
i == 1
continue
distmat.append(
[float(i) for i in line.strip('\r\n').split(',')])
distmat = np.array(distmat)
distvec = distmat.sum(1) / len(uids)
uids_sorted = [x for (y, x) in sorted(zip(distvec, uids))]
N = len(uids_sorted)
print('Total users %d: ' % N)
mgs = mobgraphs[C]
mesos = Mesos(mgs[uids_sorted[0]], mgs[uids_sorted[1]])
topmesos = mesos.mesos
topmesos_sim = 1 - mesos.struct_dist()
motifs = {}
for i in range(N - 1):
u1 = uids_sorted[i]
u2 = uids_sorted[i + 1]
g1 = mgs[u1]
g2 = mgs[u2]
mesos = Mesos(g1, g2).mesos
found = False
for key in motifs.keys():
if isomorphism.is_isomorphic(key, mesos):
motifs[key].append((mesos, i))
found = True
if found:
break
if not found:
motifs[mesos] = [(mesos, i)]
res = []
for key, value in motifs.items():
res.append((len(value), value[0][0]))
res = sorted(res, key=lambda x: x[0], reverse=True)
topmotif = res[0][1]
topmotif_supp = 1.0 * res[0][0] / N
if new_file:
mode = 'wb'
new_file = False
else:
mode = 'ab'
ofile = open(ofname, mode)
ofile.write('%d\t%d' % (C, kn))
ofile.write('\t%.3f\t%.3f' % (topmesos_sim, topmotif_supp))
ofile.write('\t%s' % dumps_mobgraph(topmesos))
ofile.write('\t%s' % dumps_mobgraph(topmotif))
ofile.write('\n')
ofile.close()
