def get_example_3():
"""get a binarized example, whose original graph is
more complicated than the above example
"""
g = DiGraph()
g.add_nodes_from(range(1, 10))
g.add_edges_from([(1, 2), (1, 3), (1, 7), (2, 4), (2, 5), (2, 6), (2, 7),
(3, 8), (3, 9)])
rewards = range(1, 10)
for r, n in zip(rewards, g.nodes()):
g.node[n]['r'] = r
# all edges have cost 2 except 1 -> 2 and 1 -> 3(cost 1)
for s, t in g.edges():
g[s][t]['c'] = 2
g[1][2]['c'] = 1
g[1][3]['c'] = 1
g = binarize_dag(g,
vertex_weight_key='r',
edge_weight_key='c',
dummy_node_name_prefix='d_')
# parameters and expected output
U = [0, 2, 3, 4, 100]
expected_edges_set = [
[],
[(1, 7)],
[(1, 'd_1'), ('d_1', 3), (3, 9)],
[(1, 'd_1'), ('d_1', 3), (3, 9), ('d_1', 2)],
# (1, 7) removed to make it a tree
list(set(g.edges()) - set([(1, 7)]))
]
return (g, U, expected_edges_set)
def ConstructShortestPathTree(G: DiGraph,
source,
target=None,
weight="weight"):
from collections import defaultdict
INF = float('inf')
dist = defaultdict(lambda: INF)
parent = defaultdict(lambda: None)
dist[source] = 0
q = [(0, source)]
visited = defaultdict(lambda: False)
while q:
s = heappop(q)
cost, _from = s
if visited[s]: continue
visited[s] = True
for to in G[_from]:
if dist[to] > G[_from][to][weight] + cost:
dist[to] = G[_from][to][weight] + cost
parent[to] = _from
heappush(q, (dist[to], to))
T = nx.DiGraph()
T.add_nodes_from([(node, {
"parent": parent[node],
"distance": dist[node]
}) for node in G.nodes().keys()])
edges = []
for to in parent:
_from = parent[to]
if _from is not None:
edges.append((_from, to, {f"{weight}": G[_from][to][weight]}))
T.add_edges_from(edges)
return T
def build_crm(roadmap_dir='./build_roadmap/roadmap_2D.p',
wsn_rate_dir='./wsn_routing/rate_map.p',
ws_img_dir='./build_roadmap/ws_img.p'):
'''
build combined roadmap, crm
via load roadmap and merge in wsn_rate info
'''
roadmap_edges = pickle.load(open(roadmap_dir, 'rb'))
wsn_rate, wifis_loc, sink_loc = pickle.load(open(wsn_rate_dir, 'rb'))
img = pickle.load(open(ws_img_dir, 'rb'))
crm = DiGraph(name='combined_model',
ws_img=img,
wifis=wifis_loc,
sink=sink_loc)
for e in roadmap_edges:
(f_node, t_node) = e
near_f_node = min(wsn_rate.keys(), key=lambda p: dist_2D(p, f_node))
f_node_rate = wsn_rate[near_f_node]
crm.add_node(f_node, rate=f_node_rate)
near_t_node = min(wsn_rate.keys(), key=lambda p: dist_2D(p, t_node))
t_node_rate = wsn_rate[near_t_node]
crm.add_node(t_node, rate=t_node_rate)
# interpolation
dist_e = dist_2D(f_node, t_node)
f_t_rate = intp_rate(f_node_rate, t_node_rate)
crm.add_edge(f_node, t_node, rate=f_t_rate, dist=dist_e)
t_f_rate = intp_rate(t_node_rate, f_node_rate)
crm.add_edge(t_node, f_node, rate=t_f_rate, dist=dist_e)
print '---crm constructed from %s and %s, %d nodes, %d edges---' % (
roadmap_dir, wsn_rate_dir, len(crm.nodes()), len(crm.edges()))
return crm
def explicitPath(self,
G: DiGraph,
T: DiGraph,
ksp: List[tuple],
target,
weight="weight") -> List[tuple]:
Edges = []
Edges_append = Edges.append
totalCost = 0
if self.prefPath >= 0:
edges = ksp[self.prefPath].edges
lastEdgeNum = -1
heapSidetrack = self.heap.sidetrack
for i in reversed(range(len(edges))):
currentEdge = edges[i]
if currentEdge.toNode == heapSidetrack.fromNode:
lastEdgeNum = i
break
for i in range(lastEdgeNum + 1):
Edges_append(edges[i])
totalCost += edges[i].weight
Edges_append(self.heap.sidetrack)
totalCost += self.heap.sidetrack.weight
current = self.heap.sidetrack.toNode
T_nodes = T.nodes()
while current != target:
nxt = T_nodes[current]["parent"]
edgeWeight = T_nodes[current]["distance"] - T_nodes[nxt]["distance"]
Edges_append(Edge(current, nxt, weight=edgeWeight))
totalCost += edgeWeight
current = nxt
return explicitPath(edges=Edges, totalCost=totalCost)
def DuoBA_from_ltls(hard_spec, soft_spec):
hard_buchi = buchi_from_ltl(hard_spec, 'hard_buchi')
soft_buchi = buchi_from_ltl(soft_spec, 'soft_buchi')
hard_symbols = hard_buchi.graph['symbols']
soft_symbols = soft_buchi.graph['symbols']
symbols = set(hard_symbols).union(set(soft_symbols))
DuoBA = DiGraph(type='safe_buchi',
hard=hard_buchi,
soft=soft_buchi,
symols=symbols)
initial = set()
accept = set()
for (h_node, s_node, l) in cartesian_product(hard_buchi.nodes(),
soft_buchi.nodes(), [1, 2]):
DuoNode = (h_node, s_node, l)
DuoBA.add_node(DuoNode, hard=h_node, soft=s_node, level=l)
if (h_node in hard_buchi.graph['initial']
and s_node in soft_buchi.graph['initial'] and l == 1):
initial.add(DuoNode)
if (h_node in hard_buchi.graph['accept'] and l == 1):
accept.add(DuoNode)
DuoBA.graph['accept'] = accept
DuoBA.graph['initial'] = initial
for f_duonode in DuoBA.nodes():
for t_duonode in DuoBA.nodes():
f_h_node, f_s_node, f_level = check_duo_attr(DuoBA, f_duonode)
t_h_node, t_s_node, t_level = check_duo_attr(DuoBA, t_duonode)
if (t_h_node not in DuoBA.graph['hard'].neighbors(f_h_node) or
t_s_node not in DuoBA.graph['soft'].neighbors(f_s_node)):
continue
# relaxed because no common input alphabets are enabled
hardguard = DuoBA.graph['hard'].edges[f_h_node, t_h_node]['guard']
softguard = DuoBA.graph['soft'].edges[f_s_node, t_s_node]['guard']
if ((f_h_node not in DuoBA.graph['hard'].graph['accept']
and f_level == 1 and t_level == 1)
or (f_h_node in DuoBA.graph['hard'].graph['accept']
and f_level == 1 and t_level == 2)
or (f_s_node not in DuoBA.graph['soft'].graph['accept']
and f_level == 2 and t_level == 2)
or (f_s_node in DuoBA.graph['soft'].graph['accept']
and f_level == 2 and t_level == 1)):
DuoBA.add_edge(f_duonode,
t_duonode,
hardguard=hardguard,
softguard=softguard)
return DuoBA
def DuoBA_from_ltls(hard_spec, soft_spec):
hard_buchi = buchi_from_ltl(hard_spec, 'hard_buchi')
soft_buchi = buchi_from_ltl(soft_spec, 'soft_buchi')
hard_symbols = hard_buchi.graph['symbols']
soft_symbols = soft_buchi.graph['symbols']
symbols = set(hard_symbols).union(set(soft_symbols))
DuoBA = DiGraph(type='safe_buchi', hard=hard_buchi, soft=soft_buchi, symols=symbols)
initial = set()
accept = set()
for (h_node, s_node, l) in cartesian_product(hard_buchi.nodes(), soft_buchi.nodes(), [1, 2]):
DuoNode = (h_node, s_node, l)
DuoBA.add_node(DuoNode,hard=h_node, soft=s_node, level=l)
if (h_node in hard_buchi.graph['initial'] and
s_node in soft_buchi.graph['initial'] and l == 1):
initial.add(DuoNode)
if (h_node in hard_buchi.graph['accept'] and l == 1):
accept.add(DuoNode)
DuoBA.graph['accept'] = accept
DuoBA.graph['initial'] = initial
for f_duonode in DuoBA.nodes():
for t_duonode in DuoBA.nodes():
f_h_node, f_s_node, f_level = check_duo_attr(DuoBA, f_duonode)
t_h_node, t_s_node, t_level = check_duo_attr(DuoBA, t_duonode)
if (t_h_node not in DuoBA.graph['hard'].neighbors(f_h_node) or
t_s_node not in DuoBA.graph['soft'].neighbors(f_s_node)):
continue
# relaxed because no common input alphabets are enabled
hardguard = DuoBA.graph['hard'].edges[f_h_node,t_h_node]['guard']
softguard = DuoBA.graph['soft'].edges[f_s_node,t_s_node]['guard']
if ((f_h_node not in DuoBA.graph['hard'].graph['accept'] and
f_level == 1 and t_level == 1) or
(f_h_node in DuoBA.graph['hard'].graph['accept'] and
f_level == 1 and t_level == 2) or
(f_s_node not in DuoBA.graph['soft'].graph['accept'] and
f_level == 2 and t_level == 2) or
(f_s_node in DuoBA.graph['soft'].graph['accept'] and
f_level == 2 and t_level == 1)):
DuoBA.add_edge(f_duonode, t_duonode, hardguard=hardguard, softguard=softguard)
return DuoBA
class BoardGraph(object):
def walk(self, board, size_limit=maxint):
pending_nodes = []
self.graph = DiGraph()
self.start = board.display(cropped=True)
self.graph.add_node(self.start)
pending_nodes.append(self.start)
self.min_domino_count = len(board.dominoes)
while pending_nodes:
if len(self.graph) >= size_limit:
raise GraphLimitExceeded(size_limit)
state = pending_nodes.pop()
board = Board.create(state, border=1)
dominoes = set(board.dominoes)
self.min_domino_count = min(self.min_domino_count, len(dominoes))
for domino in dominoes:
dx, dy = domino.direction
self.try_move(state, domino, dx, dy, pending_nodes)
self.try_move(state, domino, -dx, -dy, pending_nodes)
self.last = state
return set(self.graph.nodes())
def try_move(self, old_state, domino, dx, dy, pending_states):
try:
new_state = self.move(domino, dx, dy)
move = domino.describe_move(dx, dy)
if not self.graph.has_node(new_state):
# new node
self.graph.add_node(new_state)
pending_states.append(new_state)
self.graph.add_edge(old_state, new_state, move=move)
except BoardError:
pass
def move(self, domino, dx, dy):
""" Move a domino and calculate the new board state.
Afterward, put the board back in its original state.
@return: the new board state
@raise BoardError: if the move is illegal
"""
domino.move(dx, dy)
try:
board = domino.head.board
if not board.isConnected():
raise BoardError('Board is not connected.')
if board.hasLoner():
raise BoardError('Board has a lonely domino.')
return board.display(cropped=True)
finally:
domino.move(-dx, -dy)
def setDelta(self,
G: DiGraph,
T: DiGraph,
weight="weight") -> Dict[tuple, float]:
T_nodes = T.nodes()
G_edges = G.edges()
for edge in G_edges:
_from, to = edge
T_from, T_to = T_nodes[_from], T_nodes[to]
G_edge = G_edges[edge]
tp = T_from["parent"]
# 全てのedge \in G\Tに対してdelta(potential)を計算する
if tp is None or tp != to: # tp = to <-> (_from, tp) == (_from, to)
tmp = G_edge[weight] + T_to["distance"] - T_from["distance"]
G_edge["delta"] = tmp if tmp == tmp else 0
def construct_arg_crm(self):
arg_crm = DiGraph()
for wp_f in self.crm.nodes_iter():
for d_f in iter(self.data_set):
s_f = (wp_f, d_f)
arg_crm.add_node(s_f)
for wp_t in self.crm.neighbors(wp_f):
d_evolve = d_f - self.crm.edge[wp_f][wp_t][
'rate'] * self.sample_time - 2 * self.quant_size
d_t = [d for d in self.data_set if d >= d_evolve][0]
s_t = (wp_t, d_t)
arg_crm.add_edge(s_f, s_t, weight=1.0)
print '---static argumented crm constructed: %d nodes, %d edges----' % (
len(arg_crm.nodes()), len(arg_crm.edges()))
self.arg_crm = arg_crm
def get_example_5():
g = DiGraph()
g.add_edges_from([(0, 1), (0, 2), (1, 3), (1, 4), (2, 4), (2, 5), (2, 6)])
for s, t in g.edges():
g[s][t]['c'] = 1
g[1][4]['c'] = 0
g[2][4]['c'] = 0
g[2][6]['c'] = 3
for n in g.nodes():
g.node[n]['r'] = 1
g.node[3]['r'] = 10
g.node[4]['r'] = 100
g.node[5]['r'] = 11
U = [10]
# sub-optimal answer actually
expected_edge_set = [[(0, 2), (2, 4), (2, 5), (2, 6)]]
return (g, U, expected_edge_set)
def find_SCCs(mdp, Sneg):
#----simply find strongly connected components----
print 'Remaining states size', len(Sneg)
SCC = set()
simple_digraph = DiGraph()
A = dict()
for s in mdp.nodes():
A[s] = mdp.node[s]['act'].copy()
for s_f in Sneg:
if s_f not in simple_digraph:
simple_digraph.add_node(s_f)
for s_t in mdp.successors_iter(s_f):
if s_t in Sneg:
simple_digraph.add_edge(s_f,s_t)
print "SubGraph of one Sf: %s states and %s edges" %(str(len(simple_digraph.nodes())), str(len(simple_digraph.edges())))
sccs = strongly_connected_component_subgraphs(simple_digraph)
for scc in sccs:
SCC.add(frozenset(scc.nodes()))
return SCC, A
def create_graph_with_new_data():
'''
A DiGraph object holds directed edges
- Parallel edges still aren't allowed
- For a Digraph, two edges are parallel if they connect the same ordered pair of vertices
- Thus, two edges that connect the same vertices but go in different directions are not parallel
'''
# Create with edge list
g = DiGraph([(1, 5), (5, 1)])
#g = Graph([(1, 5), (5, 1)])
g.add_node(6)
print(g.nodes()) # [1, 5, 6]
# These are two distinct edges
print(g.edges()) # [(1, 5), (5, 1)]
print(g.neighbors(1)) # [5]
print(g.neighbors(5)) # [1]
g.edge[1][5]['foo'] = 'bar'
print(g.edge[1][5]) # {'foo': 'bar'}
print(g.edge[5][1]) # {}
def get_example_5():
g = DiGraph()
g.add_edges_from([(0, 1), (0, 2), (1, 3), (1, 4),
(2, 4), (2, 5), (2, 6)])
for s, t in g.edges():
g[s][t]['c'] = 1
g[1][4]['c'] = 0
g[2][4]['c'] = 0
g[2][6]['c'] = 3
for n in g.nodes():
g.node[n]['r'] = 1
g.node[3]['r'] = 10
g.node[4]['r'] = 100
g.node[5]['r'] = 11
U = [10]
# sub-optimal answer actually
expected_edge_set = [[(0, 2), (2, 4), (2, 5), (2, 6)]]
return (g, U, expected_edge_set)
def eppstein_ksp(self,
G: DiGraph,
source: Any,
target: Any,
K: int = 1,
weight="weight") -> List[NamedTuple]:
self.target = target
T = ConstructShortestPathTree(
G.reverse(), target, weight=weight) # reversed shortest path tree
self.setDelta(G, T, weight=weight)
nodeHeaps = dict()
outrootHeaps = dict()
for node in G.nodes():
self.computeH_out(node, G, nodeHeaps, weight=weight)
rootArrayHeap = EppsteinArrayHeap()
self.computeH_T_recursive(target, rootArrayHeap, nodeHeaps,
outrootHeaps, T)
hg = EppsteinHeap(Edge(source, source, weight=0))
ksp = []
pathPQ = []
heappush(pathPQ, EppsteinPath(hg, -1, T.nodes()[source]["distance"]))
k = 0
while k < K and pathPQ:
# Checked
kpathImplicit = heappop(pathPQ)
kpath = kpathImplicit.explicitPath(G,
T,
ksp,
target,
weight=weight)
ksp.append(kpath)
self.addHeapEdgeChildrenToQueue(kpathImplicit, ksp,
pathPQ) # heap edge
self.addCrossEdgeChildToQueue(outrootHeaps, kpathImplicit, k, ksp,
pathPQ) # cross edge
k += 1
return ksp
def get_example_4():
g = DiGraph()
g.add_edges_from([
(0, 1),
(1, 2),
(2, 3),
(2, 14), # tree 1
(2, 15),
(3, 16),
(3, 17),
(0, 4),
(4, 5),
(4, 6), # tree 2
(5, 11),
(6, 11),
(6, 12),
(6, 13),
(0, 7),
(7, 8),
(7, 9), # tree 3
(8, 10),
(8, 11),
(9, 12),
(9, 13)
])
for s, t in g.edges():
g[s][t]['c'] = 1
for n in g.nodes():
g.node[n]['r'] = 1
g.node[10]['r'] = 2
U = [7]
expected_edge_set = [[
(0, 7),
(7, 8),
(7, 9), # tree 3
(8, 10),
(8, 11),
(9, 12),
(9, 13)
]]
return (g, U, expected_edge_set)
def get_example_4():
g = DiGraph()
g.add_edges_from([(0, 1), (1, 2), (2, 3), (2, 14), # tree 1
(2, 15), (3, 16), (3, 17),
(0, 4), (4, 5), (4, 6), # tree 2
(5, 11), (6, 11), (6, 12), (6, 13),
(0, 7), (7, 8), (7, 9), # tree 3
(8, 10), (8, 11), (9, 12), (9, 13)])
for s, t in g.edges():
g[s][t]['c'] = 1
for n in g.nodes():
g.node[n]['r'] = 1
g.node[10]['r'] = 2
U = [7]
expected_edge_set = [
[(0, 7), (7, 8), (7, 9), # tree 3
(8, 10), (8, 11), (9, 12), (9, 13)]
]
return (g, U, expected_edge_set)
def get_example_6():
# IN-OPTIMAL CASE
g = DiGraph()
g.add_edges_from([(0, 1), (0, 2), (1, 3), (1, 4), (2, 4), (2, 5)])
for s, t in g.edges():
g[s][t]['c'] = 0
g[1][3]['c'] = 4
g[1][4]['c'] = 4
g[2][4]['c'] = 2
g[2][5]['c'] = 1
for n in g.nodes():
g.node[n]['r'] = 0
g.node[3]['r'] = 1
g.node[4]['r'] = 100
g.node[5]['r'] = 1
U = [7]
# sub-optimal answer actually
expected_edge_set = [[(0, 2), (2, 4), (2, 5)]]
return (g, U, expected_edge_set)
def get_example_6():
# IN-OPTIMAL CASE
g = DiGraph()
g.add_edges_from([(0, 1), (0, 2), (1, 3),
(1, 4), (2, 4), (2, 5)])
for s, t in g.edges():
g[s][t]['c'] = 0
g[1][3]['c'] = 4
g[1][4]['c'] = 4
g[2][4]['c'] = 2
g[2][5]['c'] = 1
for n in g.nodes():
g.node[n]['r'] = 0
g.node[3]['r'] = 1
g.node[4]['r'] = 100
g.node[5]['r'] = 1
U = [7]
# sub-optimal answer actually
expected_edge_set = [[(0, 2), (2, 4), (2, 5)]]
return (g, U, expected_edge_set)
def get_example_3():
"""get a binarized example, whose original graph is
more complicated than the above example
"""
g = DiGraph()
g.add_nodes_from(range(1, 10))
g.add_edges_from([(1, 2), (1, 3), (1, 7),
(2, 4), (2, 5), (2, 6),
(2, 7), (3, 8), (3, 9)])
rewards = range(1, 10)
for r, n in zip(rewards, g.nodes()):
g.node[n]['r'] = r
# all edges have cost 2 except 1 -> 2 and 1 -> 3(cost 1)
for s, t in g.edges():
g[s][t]['c'] = 2
g[1][2]['c'] = 1
g[1][3]['c'] = 1
g = binarize_dag(g,
vertex_weight_key='r',
edge_weight_key='c',
dummy_node_name_prefix='d_')
# parameters and expected output
U = [0, 2, 3, 4, 100]
expected_edges_set = [
[],
[(1, 7)],
[(1, 'd_1'), ('d_1', 3), (3, 9)],
[(1, 'd_1'), ('d_1', 3), (3, 9), ('d_1', 2)],
# (1, 7) removed to make it a tree
list(set(g.edges()) - set([(1, 7)]))
]
return (g, U, expected_edges_set)
def find_MECs(mdp, Sneg):
#----implementation of Alg.47 P866 of Baier08----
print 'Remaining states size', len(Sneg)
U = mdp.graph['U']
A = dict()
for s in Sneg:
A[s] = mdp.node[s]['act'].copy()
if not A[s]:
print "Isolated state"
MEC = set()
MECnew = set()
MECnew.add(frozenset(Sneg))
#----
k = 0
while MEC != MECnew:
print "<============iteration %s============>" %k
k +=1
MEC = MECnew
MECnew = set()
print "MEC size: %s" %len(MEC)
print "MECnew size: %s" %len(MECnew)
for T in MEC:
R = set()
T_temp = set(T)
simple_digraph = DiGraph()
for s_f in T_temp:
if s_f not in simple_digraph:
simple_digraph.add_node(s_f)
for s_t in mdp.successors_iter(s_f):
if s_t in T_temp:
simple_digraph.add_edge(s_f,s_t)
print "SubGraph of one MEC: %s states and %s edges" %(str(len(simple_digraph.nodes())), str(len(simple_digraph.edges())))
Sccs = strongly_connected_component_subgraphs(simple_digraph)
i = 0
for Scc in Sccs:
i += 1
if (len(Scc.edges())>=1):
for s in Scc.nodes():
U_to_remove = set()
for u in A[s]:
for t in mdp.successors_iter(s):
if ((u in mdp.edge[s][t]['prop'].keys()) and (t not in Scc.nodes())):
U_to_remove.add(u)
A[s].difference_update(U_to_remove)
if not A[s]:
R.add(s)
while R:
s = R.pop()
T_temp.remove(s)
for f in mdp.predecessors(s):
if f in T_temp:
A[f].difference_update(set(mdp.edge[f][s]['prop'].keys()))
if not A[f]:
R.add(f)
New_Sccs = strongly_connected_component_subgraphs(simple_digraph)
j = 0
for Scc in New_Sccs:
j += 1
if (len(Scc.edges()) >= 1):
common = set(Scc.nodes()).intersection(T_temp)
if common:
MECnew.add(frozenset(common))
#---------------
print 'Final MEC and MECnew size:', len(MEC)
return MEC, A
class Buchi(object):
"""
construct buchi automaton graph
"""
def __init__(self, task):
"""
initialization
:param task: task specified in LTL
"""
# task specified in LTL
self.formula = task.formula
self.subformula = task.subformula
self.number_of_robots = task.number_of_robots
# graph of buchi automaton
self.buchi_graph = DiGraph(type='buchi', init=[], accept=[])
# minimal length (in terms of number of transitions) between a pair of nodes
self.min_length = dict()
def construct_buchi_graph(self):
"""
parse the output of the program ltl2ba and build the buchi automaton
"""
program = 'online'
if program == 'offline':
# directory of the program ltl2ba
dirname = os.path.dirname(__file__)
# output of the program ltl2ba
output = subprocess.check_output(dirname + "/./ltl2ba -f \"" +
self.formula + "\"",
shell=True).decode("utf-8")
# find all states/nodes in the buchi automaton
states = r'\n(\w+):'
if_fi = r':\n\tif(.*?)fi'
elif program == 'online':
para = {
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
'formule': self.formula,
'convert': 'Convert',
'image': 'on',
'descr': 'on',
'flysimpl': 'on',
'latsimpl': 'on',
'sccsimpl': 'on',
'fjtofj': 'on'
}
html_doc = requests.post(
'http://www.lsv.fr/~gastin/ltl2ba/index.php', data=para)
# print(html_doc.text)
output = BeautifulSoup(html_doc.text, 'html.parser').tt.string
states = r'\n(\w+) :'
if_fi = r' :(.*?)fi'
# find all states/nodes in the buchi automaton
state_re = re.compile(states)
state_group = re.findall(state_re, output)
# find initial and accepting states
init = [s for s in state_group if 'init' in s]
accept = [s for s in state_group if 'accept' in s]
# finish the inilization of the graph of the buchi automaton
self.buchi_graph.graph['init'] = init
self.buchi_graph.graph['accept'] = accept
order_key = list(self.subformula.keys())
order_key.sort(reverse=True)
# for each state/node, find it transition relations
for state in state_group:
# add node
self.buchi_graph.add_node(state)
# loop over all transitions starting from current state
state_if_fi = re.findall(state + if_fi, output, re.DOTALL)
if state_if_fi:
relation_group = re.findall(r':: (\(.*?\)) -> goto (\w+)\n\t',
state_if_fi[0])
for symbol, next_state in relation_group:
sym = symbol
# delete edges with multiple subformulas
num_pos = 0
for sym in symbol.split(' && '):
if '!' not in sym:
num_pos += 1
if num_pos >= 2: continue
# whether the edge is feasible in terms of atomic propositions
for k in order_key:
symbol = symbol.replace('e{0}'.format(k),
self.subformula[k])
# get the trurh assignment
truth_table = self.get_truth_assignment(symbol)
# infeasible transition
if not truth_table: continue
# add edge
self.buchi_graph.add_edge(state,
next_state,
truth=truth_table,
symbol=sym)
else:
state_skip = re.findall(state + r'(.*?)skip\n', output,
re.DOTALL)
if state_skip:
self.buchi_graph.add_edge(state, state, truth='1')
def get_truth_assignment(self, symbol):
"""
get one set of truth assignment that makes the symbol true
:param symbol: logical expression which controls the transition
:return: a set of truth assignment enables the symbol
"""
# empty symbol
if symbol == '(1)':
return '1'
# non-empty symbol
else:
exp = symbol.replace('||', '|').replace('&&',
'&').replace('!', '~')
exp_dnf = to_dnf(exp).__str__()
# loop over each clause
for clause in exp_dnf.split(' | '):
truth_table = dict()
clause = clause.strip('(').strip(')')
literals = clause.split(' & ')
# loop over each literal
for literal in literals:
if '~' not in literal:
truth_table[literal] = True
else:
truth_table[literal[1:]] = False
# whether exist a literal with positive and negative version
if len(literals) != len(truth_table):
continue
else:
# exist a robot with two positive literals
true_key = [
truth.split('_')[1] for truth in truth_table.keys()
if truth_table[truth]
]
if len(true_key) != len(set(true_key)):
continue
else:
# print(clause, truth_table)
return truth_table
return dict()
def get_minimal_length(self):
"""
search the shortest path from a node to another, i.e., # of transitions in the path
:return:
"""
# loop over pairs of buchi states
for head_node in self.buchi_graph.nodes():
for tail_node in self.buchi_graph.nodes():
# head_node = tail_node, and tail_node is an accepting state
if head_node != tail_node and 'accept' in tail_node:
try:
length, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph,
source=head_node,
target=tail_node)
# couldn't find a path from head_node to tail_node
except nx.exception.NetworkXNoPath:
length = np.inf
self.min_length[(head_node, tail_node)] = length
# head_node != tail_node and tail_node is an accepting state
# move 1 step forward to all reachable states of head_node then calculate the minimal length
elif head_node == tail_node and 'accept' in tail_node:
length = np.inf
for suc in self.buchi_graph.succ[head_node]:
try:
len1, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph, source=suc, target=tail_node)
except nx.exception.NetworkXNoPath:
len1 = np.inf
if len1 < length:
length = len1 + 1
self.min_length[(head_node, tail_node)] = length
def get_feasible_accepting_state(self):
"""
get feasbile accepting/final state, or check whether an accepting state is feaasible
:return:
"""
accept = self.buchi_graph.graph['accept']
self.buchi_graph.graph['accept'] = []
for ac in accept:
for init in self.buchi_graph.graph['init']:
if self.min_length[(init, ac)] < np.inf and self.min_length[
(ac, ac)] < np.inf:
self.buchi_graph.graph['accept'].append(ac)
break
if not self.buchi_graph.graph['accept']:
print('No feasible accepting states!!!!!!!!! ')
exit()
def robot2region(self, symbol):
"""
pair of robot and corresponding regions in the expression
:param symbol: logical expression
:return: robot index : regions
eg: input: exp = 'l1_1 & l3_1 & l4_1 & l4_6 | l3_4 & l5_6'
output: {1: ['l1_1', 'l3_1', 'l4_1'], 4: ['l3_4'], 6: ['l4_6', 'l5_6']}
"""
robot_region = dict()
for r in range(self.number_of_robots):
findall = re.findall(r'(l\d+?_{0})[^0-9]'.format(r + 1), symbol)
if findall:
robot_region[str(r + 1)] = findall
return robot_region
class tree(object):
""" construction of prefix and suffix tree
"""
def __init__(self, ts, buchi_graph, init, base=1e3, seg='pre'):
"""
:param ts: transition system
:param buchi_graph: Buchi graph
:param init: product initial state
"""
self.robot = 1
self.goals = []
self.ts = ts
self.buchi_graph = buchi_graph
self.init = init
self.dim = len(self.ts['workspace'])
self.tree = DiGraph(type='PBA', init=init)
label = self.label(init[0])
if label != '':
label = label + '_' + str(1)
# accepting state before current node
acc = set()
if 'accept' in init[1]:
acc.add(init)
self.tree.add_node(init, cost=0, label=label, acc=acc)
# already used skilles
self.used = set()
self.base = base
self.seg = seg
self.found = 10
def sample(self, num):
"""
sample point from the workspace
:return: sampled point, tuple
"""
q_rand = list(self.tree.nodes())[np.random.randint(
self.tree.number_of_nodes(), size=1)[0]]
x_rand = [0, 0]
# parallel to one axis
line = np.random.randint(2, size=1)[0]
x_rand[line] = q_rand[0][line]
# sample another component
r = round(1 / num, 10)
# x_rand[1-line] = int(np.random.uniform(0, 1, size=1)[0]/r) * r + r/2
x_rand[1 - line] = round(
np.random.randint(num, size=1)[0] * r + r / 2, 10)
return tuple(x_rand)
def acpt_check(self, q_min, q_new):
"""
check the accepting state in the patg leading to q_new
:param q_min:
:param q_new:
:return:
"""
changed = False
acc = set(self.tree.nodes[q_min]['acc']) # copy
if 'accept' in q_new[1]:
acc.add(q_new)
# print(acc)
changed = True
return acc, changed
def extend(self, q_new, prec_list, succ_list, label_new):
"""
:param: q_new: new state form: tuple (mulp, buchi)
:param: near_v: near state form: tuple (mulp, buchi)
:param: obs_check: check obstacle free form: dict { (mulp, mulp): True }
:param: succ: list of successor of the root
:return: extending the tree
"""
added = 0
cost = np.inf
q_min = ()
for pre in prec_list:
if pre in succ_list: # do not extend if there is a corresponding root
continue
c = self.tree.nodes[pre]['cost'] + np.abs(
q_new[0][0] - pre[0][0]) + np.abs(q_new[0][1] - pre[0][1])
if c < cost:
added = 1
q_min = pre
cost = c
if added == 1:
self.tree.add_node(q_new, cost=cost, label=label_new)
self.tree.nodes[q_new]['acc'] = set(
self.acpt_check(q_min, q_new)[0])
self.tree.add_edge(q_min, q_new)
return added
def rewire(self, q_new, succ_list):
"""
:param: q_new: new state form: tuple (mul, buchi)
:param: near_v: near state form: tuple (mul, buchi)
:param: obs_check: check obstacle free form: dict { (mulp, mulp): True }
:return: rewiring the tree
"""
for suc in succ_list:
# root
if suc == self.init:
continue
c = self.tree.nodes[q_new]['cost'] + np.abs(
q_new[0][0] - suc[0][0]) + np.abs(q_new[0][1] - suc[0][1])
delta_c = self.tree.nodes[suc]['cost'] - c
# update the cost of node in the subtree rooted at near_vertex
if delta_c > 0:
self.tree.remove_edge(list(self.tree.pred[suc].keys())[0], suc)
self.tree.add_edge(q_new, suc)
edges = dfs_labeled_edges(self.tree, source=suc)
acc, changed = self.acpt_check(q_new, suc)
self.tree.nodes[suc]['acc'] = set(acc)
for u, v, d in edges:
if d == 'forward':
self.tree.nodes[v][
'cost'] = self.tree.nodes[v]['cost'] - delta_c
if changed:
self.tree.nodes[v]['acc'] = set(
self.acpt_check(u, v)[0]) # copy
# better to research the goal but abandon the implementation
def prec(self, q_new, label_new, obs_check):
"""
find the predcessor of q_new
:param q_new: new product state
:return: label_new: label of new
"""
p_prec = []
for vertex in self.tree.nodes:
if q_new != vertex and self.obs_check(vertex, q_new[0], label_new, obs_check) \
and self.checkTranB(vertex[1], self.tree.nodes[vertex]['label'], q_new[1]):
p_prec.append(vertex)
return p_prec
def succ(self, q_new, label_new, obs_check):
"""
find the successor of q_new
:param q_new: new product state
:return: label_new: label of new
"""
p_succ = []
for vertex in self.tree.nodes:
if q_new != vertex and self.obs_check(vertex, q_new[0], label_new, obs_check) \
and self.checkTranB(q_new[1], self.tree.nodes[q_new]['label'], vertex[1]):
p_succ.append(vertex)
return p_succ
def obs_check(self, q_tree, x_new, label_new, obs_check):
"""
check whether obstacle free along the line from q_tree to x_new
:param q_tree: vertex in the tree
:param x_new:
:param label_new:
:return: true or false
"""
if q_tree[0][0] != x_new[0] and q_tree[0][1] != x_new[1]:
return False
if (q_tree[0], x_new) in obs_check.keys():
return obs_check[(q_tree[0], x_new)]
if (x_new, q_tree[0]) in obs_check.keys():
return obs_check[(x_new, q_tree[0])]
# the line connecting two points crosses an obstacle
for (obs, boundary) in iter(self.ts['obs'].items()):
if LineString([Point(q_tree[0]),
Point(x_new)]).intersects(boundary):
obs_check[(q_tree[0], x_new)] = False
obs_check[(x_new, q_tree[0])] = False
return False
for (region, boundary) in iter(self.ts['region'].items()):
if LineString([Point(q_tree[0]), Point(x_new)]).intersects(boundary) \
and region + '_' + str(1) != label_new \
and region + '_' + str(1) != self.tree.nodes[q_tree]['label']:
obs_check[(q_tree[0], x_new)] = False
obs_check[(x_new, q_tree[0])] = False
return False
obs_check[(q_tree[0], x_new)] = True
obs_check[(x_new, q_tree[0])] = True
return True
def label(self, x):
"""
generating the label of position state
:param x: position
:return: label
"""
point = Point(x)
# whether x lies within obstacle
for (obs, boundary) in iter(self.ts['obs'].items()):
if point.within(boundary):
return obs
# whether x lies within regions
for (region, boundary) in iter(self.ts['region'].items()):
if point.within(boundary):
return region
# x lies within unlabeled region
return ''
def checkTranB(self, b_state, x_label, q_b_new):
""" decide valid transition, whether b_state --L(x)---> q_b_new
:param b_state: buchi state
:param x_label: label of x
:param q_b_new buchi state
:return True satisfied
"""
b_state_succ = self.buchi_graph.succ[b_state]
# q_b_new is not the successor of b_state
if q_b_new not in b_state_succ:
return False
truth = self.buchi_graph.edges[(b_state, q_b_new)]['truth']
if self.t_satisfy_b_truth(x_label, truth):
return True
return False
def t_satisfy_b_truth(self, x_label, truth):
"""
check whether transition enabled under current label
:param x_label: current label
:param truth: truth value making transition enabled
:return: true or false
"""
if truth == '1':
return True
true_label = [
truelabel for truelabel in truth.keys() if truth[truelabel]
]
for label in true_label:
if label not in x_label:
return False
false_label = [
falselabel for falselabel in truth.keys() if not truth[falselabel]
]
for label in false_label:
if label in x_label:
return False
return True
def findpath(self, goals):
"""
find the path backwards
:param goals: goal state
:return: dict path : cost
"""
paths = OrderedDict()
for i in range(len(goals)):
goal = goals[i]
path = [goal]
s = goal
while s != self.init:
s = list(self.tree.pred[s].keys())[0]
path.insert(0, s)
paths[i] = [self.tree.nodes[goal]['cost'], path]
return paths
class StadynaMcgAnalysis:
def __init__(self):
self.androGuardObjects = []
self.nodes = {}
self.nodes_id = {}
self.entry_nodes = []
self.G = DiGraph()
# self.internal_methods = []
#self.GI = DiGraph()
def analyseFile(self, vmx, apk):
vm = vmx.get_vm()
self.androGuardObjects.append((apk, vm, vmx))
# self.internal_methods.extend(vm.get_methods())
#creating real internal nodes
internal_called_methods = vmx.get_tainted_packages(
).stadyna_get_internal_called_methods()
for method in internal_called_methods:
class_name, method_name, descriptor = method
nodeType = None
if method_name == "<clinit>":
nodeType = NODE_STATIC_INIT
elif method_name == "<init>":
nodeType = NODE_CONSTRUCTOR
else:
nodeType = NODE_METHOD
n = self._get_node(nodeType, (class_name, method_name, descriptor))
n.set_attribute(ATTR_CLASS_NAME, class_name)
n.set_attribute(ATTR_METHOD_NAME, method_name)
n.set_attribute(ATTR_DESCRIPTOR, descriptor)
self.G.add_node(n.id)
#creating real edges (nodes are already there)
#currently we are working only with internal packages.
for j in vmx.get_tainted_packages().get_internal_packages():
src_class_name, src_method_name, src_descriptor = j.get_src(
vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(
vm.get_class_manager())
n1 = self._get_existed_node(
(src_class_name, src_method_name, src_descriptor))
# n1.set_attribute(ATTR_CLASS_NAME, src_class_name)
# n1.set_attribute(ATTR_METHOD_NAME, src_method_name)
# n1.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
n2 = self._get_existed_node(
(dst_class_name, dst_method_name, dst_descriptor))
# n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
# n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
# n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#adding fake class nodes
for method in internal_called_methods:
src_class_name, src_method_name, src_descriptor = method
if src_method_name == "<init>" or src_method_name == "<clinit>":
n1 = self._get_existed_node(
(src_class_name, src_method_name, src_descriptor))
n2 = self._get_node(NODE_FAKE_CLASS, src_class_name, None,
False)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
if src_method_name == "<clinit>":
self.G.add_edge(n1.id, n2.id)
elif src_method_name == "<init>":
self.G.add_edge(n2.id, n1.id)
#real (external) reflection invoke nodes
reflection_invoke_paths = analysis.seccon_get_invoke_method_paths(vmx)
for j in reflection_invoke_paths:
src_class_name, src_method_name, src_descriptor = j.get_src(
vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(
vm.get_class_manager())
n1 = self._get_existed_node(
(src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning(
"Cannot find the node [%s], where reflection invoke is called!"
% (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name,
src_descriptor, dst_class_name,
dst_method_name, dst_descriptor,
POSTFIX_REFL_INVOKE)
n2 = self._get_node(NODE_REFL_INVOKE, key, LABEL_REFL_INVOKE, True)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge(n1.id, n2.id)
#real (external) reflection new instance nodes
reflection_newInstance_paths = analysis.seccon_get_newInstance_method_paths(
vmx)
for j in reflection_newInstance_paths:
src_class_name, src_method_name, src_descriptor = j.get_src(
vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(
vm.get_class_manager())
n1 = self._get_existed_node(
(src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning(
"Cannot find the node [%s], where reflection new instance is called!"
% (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name,
src_descriptor, dst_class_name,
dst_method_name, dst_descriptor,
POSTFIX_REFL_NEWINSTANCE)
n2 = self._get_node(NODE_REFL_NEWINSTANCE, key,
LABEL_REFL_NEWINSTANCE, True)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge(n1.id, n2.id)
#adding fake entry points
if apk != None:
for i in apk.get_activities():
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node(
(j, "onCreate", "(Landroid/os/Bundle;)V"))
if n1 != None:
key = "%s %s %s %s" % (j, "onCreate",
"(Landroid/os/Bundle;)V",
POSTFIX_ACTIVITY)
n2 = self._get_node(NODE_FAKE_ACTIVITY, key,
LABEL_ACTIVITY, False)
self.G.add_edge(n2.id, n1.id)
self.entry_nodes.append(n1.id)
for i in apk.get_services():
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node((j, "onCreate", "()V"))
if n1 != None:
key = "%s %s %s %s" % (j, "onCreate", "()V",
POSTFIX_SERVICE)
n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_SERVICE,
False)
self.G.add_edge(n2.id, n1.id)
self.entry_nodes.append(n1.id)
for i in apk.get_receivers():
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node(
(j, "onReceive",
"(Landroid/content/Context;Landroid/content/Intent;)V"))
if n1 != None:
key = "%s %s %s %s" % (
j, "onReceive",
"(Landroid/content/Context;Landroid/content/Intent;)V",
POSTFIX_RECEIVER)
n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_RECEIVER,
False)
self.G.add_edge(n2.id, n1.id)
self.entry_nodes.append(n1.id)
#fake permissions
list_permissions = vmx.stadyna_get_permissions([])
for x in list_permissions:
for j in list_permissions[x]:
if isinstance(j, PathVar):
continue
src_class_name, src_method_name, src_descriptor = j.get_src(
vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(
vm.get_class_manager())
n1 = self._get_existed_node(
(src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning(
"Cannot find node [%s %s %s] for permission [%s]!" %
(src_class_name, src_method_name, src_descriptor, x))
continue
#SOURCE, DEST, POSTFIX, PERMISSION_NAME
key = "%s %s %s %s %s %s %s %s" % (
src_class_name, src_method_name, src_descriptor,
dst_class_name, dst_method_name, dst_descriptor,
POSTFIX_PERM, x)
n2 = self._get_node(NODE_FAKE_PERMISSION, key, x, False)
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n2.set_attribute(ATTR_PERM_NAME, x)
n2.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[x][0])
self.G.add_edge(n1.id, n2.id)
#fake DexClassLoader nodes
dyn_code_loading = analysis.seccon_get_dyncode_loading_paths(vmx)
for j in dyn_code_loading:
src_class_name, src_method_name, src_descriptor = j.get_src(
vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(
vm.get_class_manager())
n1 = self._get_existed_node(
(src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning(
"Cannot find dexload node [%s]!" %
(src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name,
src_descriptor, dst_class_name,
dst_method_name, dst_descriptor,
POSTFIX_DEXLOAD)
n2 = self._get_node(NODE_FAKE_DEXLOAD, key, LABEL_DEXLOAD, False)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge(n1.id, n2.id)
# Specific Java/Android library
for c in vm.get_classes():
#if c.get_superclassname() == "Landroid/app/Service;" :
# n1 = self._get_node( c.get_name(), "<init>", "()V" )
# n2 = self._get_node( c.get_name(), "onCreate", "()V" )
# self.G.add_edge( n1.id, n2.id )
if c.get_superclassname(
) == "Ljava/lang/Thread;" or c.get_superclassname(
) == "Ljava/util/TimerTask;":
for i in vm.get_method("run"):
if i.get_class_name() == c.get_name():
n1 = self._get_node(NODE_METHOD,
(i.get_class_name(), i.get_name(),
i.get_descriptor()))
n2 = self._get_node(
NODE_METHOD,
(i.get_class_name(), "start", i.get_descriptor()))
# link from start to run
self.G.add_edge(n2.id, n1.id)
#n2.add_edge( n1, {} )
# link from init to start
for init in vm.get_method("<init>"):
if init.get_class_name() == c.get_name():
#TODO: Leaving _get_existed_node to check if all the nodes are included
#It is possible that internal_packages does not contain this node. Leaving _get_existed_node to check this
n3 = self._get_node(
NODE_CONSTRUCTOR,
(init.get_class_name(), "<init>",
init.get_descriptor()))
self.G.add_edge(n3.id, n2.id)
#n3.add_edge( n2, {} )
def addInvokePath(self, src, through, dst):
src_class_name, src_method_name, src_descriptor = src
dst_class_name, dst_method_name, dst_descriptor = dst
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (
src_class_name, src_method_name, src_descriptor,
through_class_name, through_method_name, through_descriptor,
POSTFIX_REFL_INVOKE)
n1 = self._get_existed_node(key)
if n1 == None:
logger.warning(
"Something wrong has happened! Could not find invoke Node in Graph with key [%s]"
% str(key))
return
n2 = self._get_node(NODE_METHOD,
(dst_class_name, dst_method_name, dst_descriptor))
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#check if called method calls protected feature
data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor)
if data in DVM_PERMISSIONS_BY_API_CALLS:
logger.info(
"BINGOOOOOOO! The protected method is called through reflection!"
)
perm = DVM_PERMISSIONS_BY_API_CALLS[data]
key1 = "%s %s %s %s %s %s %s %s" % (
through_class_name, through_method_name, through_descriptor,
dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM,
perm)
n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False)
n3.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n3.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n3.set_attribute(ATTR_PERM_NAME, perm)
n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[perm][0])
self.G.add_edge(n2.id, n3.id)
def addNewInstancePath(self, src, through, dst):
src_class_name, src_method_name, src_descriptor = src
dst_class_name, dst_method_name, dst_descriptor = dst
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (
src_class_name, src_method_name, src_descriptor,
through_class_name, through_method_name, through_descriptor,
POSTFIX_REFL_NEWINSTANCE)
n1 = self._get_existed_node(key)
if n1 == None:
logger.error(
"Something wrong has happened! Could not find Node in Graph with key [%s]"
% str(key))
return
n2 = self._get_node(NODE_CONSTRUCTOR,
(dst_class_name, dst_method_name, dst_descriptor))
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#we also need to add link to the class node
#TODO: Think in the future what to do with this
n_class = self._get_node(NODE_FAKE_CLASS, dst_class_name, None, False)
n_class.set_attribute(ATTR_CLASS_NAME, dst_class_name)
self.G.add_edge(n_class.id, n2.id)
#checking if we need to add additional permission nodes
data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor)
if data in DVM_PERMISSIONS_BY_API_CALLS:
logger.info(
"BINGOOOOOOO! The protected method is called through reflection!"
)
perm = DVM_PERMISSIONS_BY_API_CALLS[data]
key1 = "%s %s %s %s %s %s %s %s" % (
through_class_name, through_method_name, through_descriptor,
dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM,
perm)
n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False)
n3.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n3.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n3.set_attribute(ATTR_PERM_NAME, perm)
n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[perm][0])
self.G.add_edge(n2.id, n3.id)
def addDexloadPath(self, src, through, filename):
src_class_name, src_method_name, src_descriptor = src
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name,
src_descriptor, through_class_name,
through_method_name,
through_descriptor, POSTFIX_DEXLOAD)
n1 = self._get_existed_node(key)
if n1 == None:
logger.error(
"Something wrong has happened! Could not find Node in Graph with key [%s]"
% str(key))
return
n2 = self._get_node(NODE_FAKE_DEXLOAD_FILE, filename, filename, False)
n2.set_attribute(ATTR_DEXLOAD_FILENAME, filename)
self.G.add_edge(n1.id, n2.id)
def _get_node(self, nType, key, label=None, real=True):
node_key = None
if isinstance(key, basestring):
node_key = key
elif isinstance(key, tuple):
node_key = "%s %s %s" % key
else:
logger.error("Unknown instance type of key!!!")
if node_key not in self.nodes.keys():
new_node = NodeS(len(self.nodes), nType, node_key, label, real)
self.nodes[node_key] = new_node
self.nodes_id[new_node.id] = new_node
return self.nodes[node_key]
def _get_existed_node(self, key):
node_key = None
if isinstance(key, basestring):
node_key = key
elif isinstance(key, tuple):
node_key = "%s %s %s" % key
else:
logger.error("Unknown instance type of key!!!")
try:
return self.nodes[node_key]
except KeyError:
logger.error("Could not find existed node [%s]!" % node_key)
return None
def export_to_gexf(self):
buff = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
buff += "<gexf xmlns=\"http://www.gephi.org/gexf\" xmlns:viz=\"http://www.gephi.org/gexf/viz\">\n"
buff += "<graph type=\"static\">\n"
buff += "<attributes class=\"node\" type=\"static\">\n"
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_TYPE, ID_ATTRIBUTES[ATTR_TYPE])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_CLASS_NAME, ID_ATTRIBUTES[ATTR_CLASS_NAME])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_METHOD_NAME, ID_ATTRIBUTES[ATTR_METHOD_NAME])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_DESCRIPTOR, ID_ATTRIBUTES[ATTR_DESCRIPTOR])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_REAL, ID_ATTRIBUTES[ATTR_REAL])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_PERM_NAME, ID_ATTRIBUTES[ATTR_PERM_NAME])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_PERM_LEVEL, ID_ATTRIBUTES[ATTR_PERM_LEVEL])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (
ATTR_DEXLOAD_FILENAME, ID_ATTRIBUTES[ATTR_DEXLOAD_FILENAME])
buff += "</attributes>\n"
# buff += "<attributes class=\"node\" type=\"static\">\n"
# buff += "<attribute id=\"%d\" title=\"type\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "type"]
# buff += "<attribute id=\"%d\" title=\"class_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "class_name"]
# buff += "<attribute id=\"%d\" title=\"method_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "method_name"]
# buff += "<attribute id=\"%d\" title=\"descriptor\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "descriptor"]
#
#
# buff += "<attribute id=\"%d\" title=\"permissions\" type=\"integer\" default=\"0\"/>\n" % ID_ATTRIBUTES[ "permissions"]
# buff += "<attribute id=\"%d\" title=\"permissions_level\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "permissions_level"]
#
# buff += "<attribute id=\"%d\" title=\"dynamic_code\" type=\"boolean\" default=\"false\"/>\n" % ID_ATTRIBUTES[ "dynamic_code"]
#
# buff += "</attributes>\n"
buff += "<nodes>\n"
for node in self.G.nodes():
buff += "<node id=\"%d\" label=\"%s\">\n" % (
node, escape(self.nodes_id[node].label))
buff += self.nodes_id[node].get_attributes_gexf()
buff += "</node>\n"
buff += "</nodes>\n"
buff += "<edges>\n"
nb = 0
for edge in self.G.edges():
buff += "<edge id=\"%d\" source=\"%d\" target=\"%d\"/>\n" % (
nb, edge[0], edge[1])
nb += 1
buff += "</edges>\n"
buff += "</graph>\n"
buff += "</gexf>\n"
return buff
def get_current_real_node_count(self):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_REAL) == "True":
count += 1
return count
def get_current_node_count(self):
return len(self.G.nodes())
def get_current_edge_count(self):
return len(self.G.edges())
def get_current_permission_level_node_count(self, permission_level):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(
ATTR_PERM_LEVEL) == permission_level:
count += 1
return count
def get_current_protected_node_count(self):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) != None:
count += 1
return count
class Buchi(object):
"""
construct buchi automaton graph
"""
def __init__(self, task, workspace):
"""
initialization
:param task: task specified in LTL
"""
# task specified in LTL
self.formula = task.formula
self.type_num = workspace.type_num
# graph of buchi automaton
self.buchi_graph = DiGraph(type='buchi', init=[], accept=[])
def construct_buchi_graph(self):
"""
parse the output of the program ltl2ba and build the buchi automaton
"""
# directory of the program ltl2ba
dirname = os.path.dirname(__file__)
# output of the program ltl2ba
output = subprocess.check_output(dirname + "/./ltl2ba -f \"" +
self.formula + "\"",
shell=True).decode("utf-8")
# find all states/nodes in the buchi automaton
state_re = re.compile(r'\n(\w+):\n\t')
state_group = re.findall(state_re, output)
# find initial and accepting states
init = [s for s in state_group if 'init' in s]
# treat the node accept_init as init node
accept = [s for s in state_group if 'accept' in s]
# finish the inilization of the graph of the buchi automaton
self.buchi_graph.graph['init'] = init
self.buchi_graph.graph['accept'] = accept
# for each state/node, find it transition relations
for state in state_group:
# add node
self.buchi_graph.add_node(state, label='', formula='')
# loop over all transitions starting from current state
state_if_fi = re.findall(state + r':\n\tif(.*?)fi', output,
re.DOTALL)
if state_if_fi:
relation_group = re.findall(r':: (\(.*?\)) -> goto (\w+)\n\t',
state_if_fi[0])
for symbol, next_state in relation_group:
symbol = symbol.replace('||', '|').replace('&&', '&')
edge_label = self.merge_check_feasible(symbol)
if edge_label:
# update node, no edges for selfloop
if state == next_state:
self.buchi_graph.nodes[state]['label'] = edge_label
self.buchi_graph.nodes[state]['formula'] = to_dnf(
symbol)
# if 'accept' in state:
# print(edge_label)
continue
# add edge
self.buchi_graph.add_edge(state,
next_state,
label=edge_label,
formula=to_dnf(symbol))
else:
state_skip = re.findall(state + r':\n\tskip\n', output,
re.DOTALL)
if state_skip:
self.buchi_graph.nodes[state]['label'] = 'skip'
self.buchi_graph.nodes[state]['formula'] = 'skip'
self.delete_node_no_selfloop_except_init_accept()
def delete_node_no_selfloop_except_init_accept(self):
remove_node = []
remove_edge = []
for node in self.buchi_graph.nodes():
# if no selfloop
if self.buchi_graph.nodes[node]['label'] == '':
# delete if not init or accept
if 'init' not in node and 'accept' not in node:
remove_node.append(node)
# init or accept in node then only keep the edge with label '1'
elif 'init' in node: # or 'accept' in node:
for n in self.buchi_graph.succ[node]:
if self.buchi_graph.edges[(node, n)]['label'] != '1':
remove_edge.append((node, n))
# if there is self loop but not equal 1, only keep the edge with label '1'
elif 'init' in node and self.buchi_graph.nodes[node][
'label'] != '1':
for n in self.buchi_graph.succ[node]:
if self.buchi_graph.edges[(node, n)]['label'] != '1':
remove_edge.append((node, n))
self.buchi_graph.remove_edges_from(remove_edge)
self.buchi_graph.remove_nodes_from(remove_node)
def get_init_accept(self):
"""
search the shortest path from a node to another, i.e., # of transitions in the path
:return:
"""
# shortest path for each accepting state to itself, including self-loop
accept_accept = dict()
for accept in self.buchi_graph.graph['accept']:
# 0 if with self loop or without outgoing edges
if self.buchi_graph.nodes[accept][
'formula'] == 'skip' or self.buchi_graph.nodes[accept][
'formula']:
accept_accept[accept] = 0
continue
# find the shortest path back to itself
length = np.inf
for suc in self.buchi_graph.succ[accept]:
try:
len1, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph, source=suc, target=accept)
except nx.exception.NetworkXNoPath:
len1 = np.inf
if len1 < length:
length = len1 + 1
accept_accept[accept] = length
# shortest path from initial to accept
init_state = None
accept_state = None
length = np.inf
for init in self.buchi_graph.graph['init']:
for accept in self.buchi_graph.graph['accept']:
# if same, find the shorest path to itself
if init == accept:
len1 = np.inf
for suc in self.buchi_graph.succ[init]:
try:
len2, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph, source=suc, target=accept)
except nx.exception.NetworkXNoPath:
len2 = np.inf
if len2 + 1 < len1:
len1 = len2 + 1
# else different, find the shorest path to accept
else:
try:
len1, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph, source=init, target=accept)
except nx.exception.NetworkXNoPath:
continue
# init_to_accept + accept_to_accept
len1 = len1 + accept_accept[accept]
# update
if len1 < length:
init_state = init
accept_state = accept
length = len1
# self loop around the accepting state
self_loop_label = self.buchi_graph.nodes[accept_state]['label']
return init_state, accept_state, self_loop_label != '1' and self_loop_label != ''
def merge_check_feasible(self, symbol):
# [[[literal], .. ], [[literal], .. ], [[literal], .. ]]
# clause clause clause
pruned_clause = []
if symbol == '(1)':
return '1'
# non-empty symbol
else:
clause = symbol.split(' | ')
# merge
for c in clause:
literals_ = c.strip('(').strip(')').split(' & ')
literals_.sort()
literals_.reverse()
i = 0
literals = []
while i < len(literals_):
curr = literals_[i].split('_')
literals.append(curr)
# loop over subsequent elements
j = i + 1
while j < len(literals_):
subsq = literals_[j].split('_')
if curr[0] != subsq[0]:
break
# same # of robots of same type with indicator 0
if curr[1] == subsq[1] and curr[2] >= subsq[2] and int(
curr[3]) + int(subsq[3]) == 0:
del literals_[j]
continue
j += 1
# make it number
curr[1] = int(curr[1])
curr[2] = int(curr[2])
curr[3] = int(curr[3])
i += 1
# if two literal use the same robot, infeasible
if sum([l[3] for l in literals]) != sum(
set([l[3] for l in literals])):
continue
# # check feasibility, whether required robots of same type for one literal exceeds provided robots
for i in range(len(literals)):
if self.type_num[literals[i][1]] < literals[i][2]:
raise ValueError(
'{0} required robots exceeds provided robots'.
format(literals[i]))
# check feasibility, the required number of robots of same type for all literals exceed provided robots
for robot_type, num in self.type_num.items():
if num < sum(
[l[2] for l in literals if l[1] == robot_type]):
literals = []
break
if literals:
pruned_clause.append(literals)
return pruned_clause
def get_subgraph(self, head, tail, is_nonempty_self_loop):
# node set
nodes = self.find_all_nodes(head, tail)
subgraph = self.buchi_graph.subgraph(nodes).copy()
subgraph.graph['init'] = head
subgraph.graph['accept'] = tail
# remove all outgoing edges of the accepting state from subgraph for the prefix part
if head != tail:
remove_edge = list(subgraph.edges(tail))
subgraph.remove_edges_from(remove_edge)
# prune the subgraph
self.prune_subgraph(subgraph)
# get all paths in the pruned subgraph
paths = []
if head != tail:
paths = list(
nx.all_simple_paths(subgraph, source=head, target=tail))
else:
for s in subgraph.succ[head]:
paths = paths + [[head] + p for p in list(
nx.all_simple_paths(subgraph, source=s, target=tail))]
# if self loop around the accepting state, then create an element with edge label ,
# solving prefix and suffix together
if is_nonempty_self_loop and subgraph.nodes[tail]['label'] != 'skip':
subgraph.add_edge(tail,
tail,
label=subgraph.nodes[tail]['label'],
formula=subgraph.nodes[tail]['formula'])
for path in paths:
path.append(tail)
return subgraph, paths
def get_element(self, pruned_subgraph):
edges = list(pruned_subgraph.edges)
curr_element = 0
# map each edge to element
edge2element = dict()
element2edge = dict()
while len(edges) > 0:
# put all elements with same node label and edge label into one list
curr = edges[0]
self_loop_label = [curr]
del edges[0]
j = 0
while j < len(edges):
if pruned_subgraph.nodes[curr[0]]['formula'] == pruned_subgraph.nodes[edges[j][0]]['formula'] \
and pruned_subgraph.edges[curr]['formula'] == pruned_subgraph.edges[edges[j]]['formula']:
self_loop_label.append(edges[j])
del edges[j]
continue
else:
j += 1
# partition into two groups
dependent = []
independent = [self_loop_label[0]]
del self_loop_label[0]
for edge in self_loop_label:
# pick the first element
is_independent = True
for ind in independent:
if nx.has_path(pruned_subgraph, ind[1], edge[0]) or \
nx.has_path(pruned_subgraph, edge[1], ind[0]):
# if there is a path between the first element and current element
is_independent = False
break
# address the first element
if is_independent:
independent.append(edge)
else:
dependent.append(edge)
# map edge/element to element/edge
if independent:
curr_element += 1
element2edge[curr_element] = independent[0]
for edge in independent:
edge2element[edge] = curr_element
if dependent:
for edge in dependent:
curr_element += 1
edge2element[edge] = curr_element
element2edge[curr_element] = edge
return edge2element, element2edge
def element2label2eccl(self, element2edge, pruned_subgraph):
element_component2label = dict()
# colloect eccl that correponds to the same label
for element in element2edge.keys():
# node label
self_loop_label = pruned_subgraph.nodes[element2edge[element]
[0]]['label']
if self_loop_label and self_loop_label != '1':
element_component2label[(element,
0)] = self.element2label2eccl_helper(
element, 0, self_loop_label)
# edge label
edge_label = pruned_subgraph.edges[element2edge[element]]['label']
if edge_label != '1':
element_component2label[(element,
1)] = self.element2label2eccl_helper(
element, 1, edge_label)
return element_component2label
def element2label2eccl_helper(self, element, component, label):
label2eccl = dict()
for c, clause in enumerate(label):
for l, literal in enumerate(clause):
region_type = tuple(literal[0:2])
if region_type in label2eccl.keys():
label2eccl[region_type].append((element, component, c, l))
else:
label2eccl[region_type] = [(element, component, c, l)]
return label2eccl
def map_path_to_element_sequence(self, edge2element, element2edge,
pruned_subgraph, paths):
element_sequences = []
# put all path that share the same set of elements into one group
for path in paths:
element_sequence = []
for i in range(len(path) - 1):
element_sequence.append(edge2element[(path[i], path[i + 1])])
is_added = False
for i in range(len(element_sequences)):
if set(element_sequences[i][0]) == set(element_sequence):
element_sequences[i].append(element_sequence)
is_added = True
break
if not is_added:
element_sequences.append([element_sequence])
graph_with_maximum_width = DiGraph()
width = 0
height = 0
# for each group, find one poset
for ele_seq in element_sequences:
# all pairs of elements from the first element
linear_order = []
for i in range(len(ele_seq[0])):
for j in range(i + 1, len(ele_seq[0])):
linear_order.append((ele_seq[0][i], ele_seq[0][j]))
# remove contradictive pairs
for i in range(1, len(ele_seq)):
for j in range(len(ele_seq[1]) - 1):
if (ele_seq[i][j + 1], ele_seq[i][j]) in linear_order:
linear_order.remove((ele_seq[i][j + 1], ele_seq[i][j]))
# hasse diagram
hasse = DiGraph()
hasse.add_nodes_from(ele_seq[0])
hasse.add_edges_from(linear_order)
self.prune_subgraph(hasse)
try:
w = max([len(o) for o in nx.antichains(hasse)])
except nx.exception.NetworkXUnfeasible:
print(hasse.edges)
# h = nx.dag_longest_path_length(hasse)
h = len([
e for e in hasse.nodes
if pruned_subgraph.nodes[element2edge[e][0]]['label'] != '1'
])
if w > width or (w == width and h > height):
graph_with_maximum_width = hasse
width = w
height = h
# print(height)
return {edge for edge in graph_with_maximum_width.edges}, list(graph_with_maximum_width.nodes), \
graph_with_maximum_width
def prune_subgraph(self, subgraph):
original_edges = list(subgraph.edges)
for edge in original_edges:
attr = subgraph.get_edge_data(edge[0], edge[1])
subgraph.remove_edge(edge[0], edge[1])
if nx.has_path(subgraph, edge[0], edge[1]):
continue
else:
subgraph.add_edge(edge[0], edge[1], **attr)
def element2label2eccl_suffix(self, element2edge, pruned_subgraph):
stage_element_component2label = dict()
# colloect eccl that correponds to the same label
for element in element2edge.keys():
# node label
self_loop_label = pruned_subgraph.nodes[element2edge[element]
[0]]['label']
if self_loop_label and self_loop_label != '1':
stage_element_component2label[(
1, element, 0)] = self.element2label2eccl_helper_suffix(
1, element, 0, self_loop_label)
stage_element_component2label[(
2, element, 0)] = self.element2label2eccl_helper_suffix(
2, element, 0, self_loop_label)
# edge label
edge_label = pruned_subgraph.edges[element2edge[element]]['label']
if edge_label != '1':
stage_element_component2label[(
1, element, 1)] = self.element2label2eccl_helper_suffix(
1, element, 1, edge_label)
stage_element_component2label[(
2, element, 1)] = self.element2label2eccl_helper_suffix(
2, element, 1, edge_label)
return stage_element_component2label
def element2label2eccl_helper_suffix(self, stage, element, component,
label):
label2eccl = dict()
for c, clause in enumerate(label):
for l, literal in enumerate(clause):
region_type = tuple(literal[0:2])
if region_type in label2eccl.keys():
label2eccl[region_type].append(
(stage, element, component, c, l))
else:
label2eccl[region_type] = [(stage, element, component, c,
l)]
return label2eccl
def find_all_nodes(self, head, tail):
front = set({head})
explored = set()
in_between = set({head})
while True:
try:
node = front.pop()
explored.add(node)
for s in self.buchi_graph.succ[node]:
if nx.has_path(
self.buchi_graph, s,
tail) and s not in explored and s not in front:
front.add(s)
in_between.add(s)
except KeyError:
break
return in_between
def plotly_lookml(G: DiGraph,
color_map: list[str],
plot_layout: str = "fdp") -> go.Figure:
"""Create an interactive plotly figure for the input `DiGraph` network
Code source: https://plotly.com/python/network-graphs/
Args:
G: the DiGraph() network for LookML nodes
color_map: color names for each node in `G`
plot_layout: layout of the nodes positions using Pydot and Graphviz (options: 'dot', 'twopi', 'fdp', 'sfdp', 'circo')
Returns:
the interactive plotly figure with the LookML network rendered
"""
# build layout (i.e. node coordinates / positions)
pos = graphviz_layout(G, prog=plot_layout)
edge_x = []
edge_y = []
for edge in G.edges():
x0, y0 = pos[edge[0]]
x1, y1 = pos[edge[1]]
edge_x.append(x0)
edge_x.append(x1)
edge_x.append(None)
edge_y.append(y0)
edge_y.append(y1)
edge_y.append(None)
edge_trace = go.Scatter(
x=edge_x,
y=edge_y,
line=dict(width=0.5, color="#888"),
hoverinfo="none",
mode="lines",
)
node_x = []
node_y = []
for node in G.nodes():
x, y = pos[node]
node_x.append(x)
node_y.append(y)
node_trace = go.Scatter(
x=node_x,
y=node_y,
mode="markers",
hoverinfo="text",
marker=dict(
# colorscale options
# 'Greys' | 'YlGnBu' | 'Greens' | 'YlOrRd' | 'Bluered' | 'RdBu' |
# 'Reds' | 'Blues' | 'Picnic' | 'Rainbow' | 'Portland' | 'Jet' |
# 'Hot' | 'Blackbody' | 'Earth' | 'Electric' | 'Viridis' |
colorscale="YlGnBu",
color=[],
size=10,
line_width=2,
),
)
# Color node point
node_text = []
for node, adjacencies in G.adjacency():
node_text.append(node)
node_trace.marker.color = color_map
node_trace.text = node_text
def layout():
layout = go.Layout(
title="LookML Content Relationships Network",
titlefont_size=16,
showlegend=False,
hovermode="closest",
margin=dict(b=20, l=5, r=5, t=40),
xaxis=dict(showgrid=False,
zeroline=False,
showticklabels=False),
yaxis=dict(showgrid=False,
zeroline=False,
showticklabels=False),
)
return layout
# Create Network Graph
fig = go.Figure(data=[edge_trace, node_trace], layout=layout())
fig.show()
return fig
def load(fname):
def clean_bool(string):
if string == "0":
return None
else:
return string
def to_bool(string):
if string == "1" or string == "True":
return True
elif string == "0" or string == "False":
return False
else:
return string
def to_float(string):
if string == "None":
return None
try:
return float(string)
except:
return string
mode = "node0"
nodes = []
edges = []
pointers = set()
outputs = None
inputs = None
named_ranges = {}
infile = gzip.GzipFile(fname, 'r')
for line in infile.read().splitlines():
if line == "====":
mode = "node0"
continue
if line == "-----":
cellmap_temp = {n.address(): n for n in nodes}
Range = RangeFactory(cellmap_temp)
mode = "node0"
continue
elif line == "edges":
cellmap = {n.address(): n for n in nodes}
mode = "edges"
continue
elif line == "outputs":
mode = "outputs"
continue
elif line == "inputs":
mode = "inputs"
continue
elif line == "named_ranges":
mode = "named_ranges"
continue
if mode == "node0":
[
address, formula, python_expression, is_range, is_named_range,
is_pointer, should_eval
] = line.split(SEP)
formula = clean_bool(formula)
python_expression = clean_bool(python_expression)
is_range = to_bool(is_range)
is_named_range = to_bool(is_named_range)
is_pointer = to_bool(is_pointer)
should_eval = should_eval
mode = "node1"
elif mode == "node1":
if is_range:
reference = json.loads(
line
) if is_pointer else line # in order to be able to parse dicts
vv = Range(reference)
if is_pointer:
if not is_named_range:
address = vv.name
pointers.add(address)
cell = Cell(address, None, vv, formula, is_range,
is_named_range, should_eval)
cell.python_expression = python_expression
nodes.append(cell)
else:
value = to_bool(to_float(line))
cell = Cell(address, None, value, formula, is_range,
is_named_range, should_eval)
cell.python_expression = python_expression
if formula:
if 'OFFSET' in formula or 'INDEX' in formula:
pointers.add(address)
cell.compile()
nodes.append(cell)
elif mode == "edges":
source, target = line.split(SEP)
edges.append((cellmap[source], cellmap[target]))
elif mode == "outputs":
outputs = line.split(SEP)
elif mode == "inputs":
inputs = line.split(SEP)
elif mode == "named_ranges":
k, v = line.split(SEP)
named_ranges[k] = v
G = DiGraph(data=edges)
print "Graph loading done, %s nodes, %s edges, %s cellmap entries" % (len(
G.nodes()), len(G.edges()), len(cellmap))
return (G, cellmap, named_ranges, pointers, outputs, inputs)
class StadynaMcgAnalysis:
def __init__(self):
self.androGuardObjects = []
self.nodes = {}
self.nodes_id = {}
self.entry_nodes = []
self.G = DiGraph()
# self.internal_methods = []
#self.GI = DiGraph()
def analyseFile(self, vmx, apk):
vm = vmx.get_vm()
self.androGuardObjects.append((apk, vm, vmx))
# self.internal_methods.extend(vm.get_methods())
#creating real internal nodes
internal_called_methods = vmx.get_tainted_packages().stadyna_get_internal_called_methods()
for method in internal_called_methods:
class_name, method_name, descriptor = method
nodeType = None
if method_name == "<clinit>":
nodeType = NODE_STATIC_INIT
elif method_name == "<init>":
nodeType = NODE_CONSTRUCTOR
else:
nodeType = NODE_METHOD
n = self._get_node(nodeType, (class_name, method_name, descriptor))
n.set_attribute(ATTR_CLASS_NAME, class_name)
n.set_attribute(ATTR_METHOD_NAME, method_name)
n.set_attribute(ATTR_DESCRIPTOR, descriptor)
self.G.add_node(n.id)
#creating real edges (nodes are already there)
#currently we are working only with internal packages.
for j in vmx.get_tainted_packages().get_internal_packages():
src_class_name, src_method_name, src_descriptor = j.get_src(vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(vm.get_class_manager())
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
# n1.set_attribute(ATTR_CLASS_NAME, src_class_name)
# n1.set_attribute(ATTR_METHOD_NAME, src_method_name)
# n1.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
n2 = self._get_existed_node((dst_class_name, dst_method_name, dst_descriptor))
# n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
# n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
# n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#adding fake class nodes
for method in internal_called_methods:
src_class_name, src_method_name, src_descriptor = method
if src_method_name == "<init>" or src_method_name == "<clinit>":
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
n2 = self._get_node(NODE_FAKE_CLASS, src_class_name, None, False)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
if src_method_name == "<clinit>":
self.G.add_edge(n1.id, n2.id)
elif src_method_name == "<init>":
self.G.add_edge(n2.id, n1.id)
#real (external) reflection invoke nodes
reflection_invoke_paths = analysis.seccon_get_invoke_method_paths(vmx)
for j in reflection_invoke_paths:
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find the node [%s], where reflection invoke is called!" % (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_INVOKE)
n2 = self._get_node(NODE_REFL_INVOKE, key, LABEL_REFL_INVOKE, True)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge( n1.id, n2.id )
#real (external) reflection new instance nodes
reflection_newInstance_paths = analysis.seccon_get_newInstance_method_paths(vmx)
for j in reflection_newInstance_paths:
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find the node [%s], where reflection new instance is called!" % (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_NEWINSTANCE)
n2 = self._get_node(NODE_REFL_NEWINSTANCE, key, LABEL_REFL_NEWINSTANCE, True)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge( n1.id, n2.id )
#adding fake entry points
if apk != None:
for i in apk.get_activities() :
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node((j, "onCreate", "(Landroid/os/Bundle;)V"))
if n1 != None:
key = "%s %s %s %s" % (j, "onCreate", "(Landroid/os/Bundle;)V", POSTFIX_ACTIVITY)
n2 = self._get_node(NODE_FAKE_ACTIVITY, key, LABEL_ACTIVITY, False)
self.G.add_edge( n2.id, n1.id )
self.entry_nodes.append( n1.id )
for i in apk.get_services() :
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node( (j, "onCreate", "()V") )
if n1 != None :
key = "%s %s %s %s" % (j, "onCreate", "()V", POSTFIX_SERVICE)
n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_SERVICE, False)
self.G.add_edge( n2.id, n1.id )
self.entry_nodes.append( n1.id )
for i in apk.get_receivers() :
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node( (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V") )
if n1 != None :
key = "%s %s %s %s" % (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V", POSTFIX_RECEIVER)
n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_RECEIVER, False)
self.G.add_edge( n2.id, n1.id )
self.entry_nodes.append( n1.id )
#fake permissions
list_permissions = vmx.stadyna_get_permissions([])
for x in list_permissions:
for j in list_permissions[x]:
if isinstance(j, PathVar):
continue
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find node [%s %s %s] for permission [%s]!" % (src_class_name, src_method_name, src_descriptor, x))
continue
#SOURCE, DEST, POSTFIX, PERMISSION_NAME
key = "%s %s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, x)
n2 = self._get_node(NODE_FAKE_PERMISSION, key, x, False)
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n2.set_attribute(ATTR_PERM_NAME, x)
n2.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ x ][0])
self.G.add_edge(n1.id, n2.id)
#fake DexClassLoader nodes
dyn_code_loading = analysis.seccon_get_dyncode_loading_paths(vmx)
for j in dyn_code_loading:
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find dexload node [%s]!" % (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_DEXLOAD)
n2 = self._get_node(NODE_FAKE_DEXLOAD, key, LABEL_DEXLOAD, False)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge( n1.id, n2.id )
# Specific Java/Android library
for c in vm.get_classes():
#if c.get_superclassname() == "Landroid/app/Service;" :
# n1 = self._get_node( c.get_name(), "<init>", "()V" )
# n2 = self._get_node( c.get_name(), "onCreate", "()V" )
# self.G.add_edge( n1.id, n2.id )
if c.get_superclassname() == "Ljava/lang/Thread;" or c.get_superclassname() == "Ljava/util/TimerTask;" :
for i in vm.get_method("run") :
if i.get_class_name() == c.get_name() :
n1 = self._get_node(NODE_METHOD, (i.get_class_name(), i.get_name(), i.get_descriptor()))
n2 = self._get_node(NODE_METHOD, (i.get_class_name(), "start", i.get_descriptor()))
# link from start to run
self.G.add_edge( n2.id, n1.id )
#n2.add_edge( n1, {} )
# link from init to start
for init in vm.get_method("<init>") :
if init.get_class_name() == c.get_name():
#TODO: Leaving _get_existed_node to check if all the nodes are included
#It is possible that internal_packages does not contain this node. Leaving _get_existed_node to check this
n3 = self._get_node(NODE_CONSTRUCTOR, (init.get_class_name(), "<init>", init.get_descriptor()))
self.G.add_edge( n3.id, n2.id )
#n3.add_edge( n2, {} )
def addInvokePath(self, src, through, dst):
src_class_name, src_method_name, src_descriptor = src
dst_class_name, dst_method_name, dst_descriptor = dst
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_INVOKE)
n1 = self._get_existed_node(key)
if n1 == None:
logger.warning("Something wrong has happened! Could not find invoke Node in Graph with key [%s]" % str(key))
return
n2 = self._get_node(NODE_METHOD, (dst_class_name, dst_method_name, dst_descriptor))
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#check if called method calls protected feature
data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor)
if data in DVM_PERMISSIONS_BY_API_CALLS:
logger.info("BINGOOOOOOO! The protected method is called through reflection!")
perm = DVM_PERMISSIONS_BY_API_CALLS[ data ]
key1 = "%s %s %s %s %s %s %s %s" % (through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm)
n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False)
n3.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n3.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n3.set_attribute(ATTR_PERM_NAME, perm)
n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ perm ][0])
self.G.add_edge(n2.id, n3.id)
def addNewInstancePath(self, src, through, dst):
src_class_name, src_method_name, src_descriptor = src
dst_class_name, dst_method_name, dst_descriptor = dst
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_NEWINSTANCE)
n1 = self._get_existed_node(key)
if n1 == None:
logger.error("Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key))
return
n2 = self._get_node(NODE_CONSTRUCTOR, (dst_class_name, dst_method_name, dst_descriptor))
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#we also need to add link to the class node
#TODO: Think in the future what to do with this
n_class = self._get_node(NODE_FAKE_CLASS, dst_class_name, None, False)
n_class.set_attribute(ATTR_CLASS_NAME, dst_class_name)
self.G.add_edge(n_class.id, n2.id)
#checking if we need to add additional permission nodes
data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor)
if data in DVM_PERMISSIONS_BY_API_CALLS:
logger.info("BINGOOOOOOO! The protected method is called through reflection!")
perm = DVM_PERMISSIONS_BY_API_CALLS[ data ]
key1 = "%s %s %s %s %s %s %s %s" % (through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm)
n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False)
n3.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n3.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n3.set_attribute(ATTR_PERM_NAME, perm)
n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ perm ][0])
self.G.add_edge(n2.id, n3.id)
def addDexloadPath(self, src, through, filename):
src_class_name, src_method_name, src_descriptor = src
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_DEXLOAD)
n1 = self._get_existed_node(key)
if n1 == None:
logger.error("Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key))
return
n2 = self._get_node(NODE_FAKE_DEXLOAD_FILE, filename, filename, False)
n2.set_attribute(ATTR_DEXLOAD_FILENAME, filename)
self.G.add_edge(n1.id, n2.id)
def _get_node(self, nType, key, label=None, real=True):
node_key = None
if isinstance(key, basestring):
node_key = key
elif isinstance(key, tuple):
node_key = "%s %s %s" % key
else:
logger.error("Unknown instance type of key!!!")
if node_key not in self.nodes.keys():
new_node = NodeS(len(self.nodes), nType, node_key, label, real)
self.nodes[node_key] = new_node
self.nodes_id[new_node.id] = new_node
return self.nodes[node_key]
def _get_existed_node(self, key):
node_key = None
if isinstance(key, basestring):
node_key = key
elif isinstance(key, tuple):
node_key = "%s %s %s" % key
else:
logger.error("Unknown instance type of key!!!")
try:
return self.nodes[node_key]
except KeyError:
logger.error("Could not find existed node [%s]!" % node_key)
return None
def export_to_gexf(self) :
buff = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
buff += "<gexf xmlns=\"http://www.gephi.org/gexf\" xmlns:viz=\"http://www.gephi.org/gexf/viz\">\n"
buff += "<graph type=\"static\">\n"
buff += "<attributes class=\"node\" type=\"static\">\n"
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_TYPE, ID_ATTRIBUTES[ ATTR_TYPE ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_CLASS_NAME, ID_ATTRIBUTES[ ATTR_CLASS_NAME ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_METHOD_NAME, ID_ATTRIBUTES[ ATTR_METHOD_NAME ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_DESCRIPTOR, ID_ATTRIBUTES[ ATTR_DESCRIPTOR ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_REAL, ID_ATTRIBUTES[ ATTR_REAL ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_PERM_NAME, ID_ATTRIBUTES[ ATTR_PERM_NAME ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_PERM_LEVEL, ID_ATTRIBUTES[ ATTR_PERM_LEVEL ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_DEXLOAD_FILENAME, ID_ATTRIBUTES[ ATTR_DEXLOAD_FILENAME ])
buff += "</attributes>\n"
# buff += "<attributes class=\"node\" type=\"static\">\n"
# buff += "<attribute id=\"%d\" title=\"type\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "type"]
# buff += "<attribute id=\"%d\" title=\"class_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "class_name"]
# buff += "<attribute id=\"%d\" title=\"method_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "method_name"]
# buff += "<attribute id=\"%d\" title=\"descriptor\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "descriptor"]
#
#
# buff += "<attribute id=\"%d\" title=\"permissions\" type=\"integer\" default=\"0\"/>\n" % ID_ATTRIBUTES[ "permissions"]
# buff += "<attribute id=\"%d\" title=\"permissions_level\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "permissions_level"]
#
# buff += "<attribute id=\"%d\" title=\"dynamic_code\" type=\"boolean\" default=\"false\"/>\n" % ID_ATTRIBUTES[ "dynamic_code"]
#
# buff += "</attributes>\n"
buff += "<nodes>\n"
for node in self.G.nodes() :
buff += "<node id=\"%d\" label=\"%s\">\n" % (node, escape(self.nodes_id[ node ].label))
buff += self.nodes_id[ node ].get_attributes_gexf()
buff += "</node>\n"
buff += "</nodes>\n"
buff += "<edges>\n"
nb = 0
for edge in self.G.edges() :
buff += "<edge id=\"%d\" source=\"%d\" target=\"%d\"/>\n" % (nb, edge[0], edge[1])
nb += 1
buff += "</edges>\n"
buff += "</graph>\n"
buff += "</gexf>\n"
return buff
def get_current_real_node_count(self):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_REAL) == "True":
count += 1
return count
def get_current_node_count(self):
return len(self.G.nodes())
def get_current_edge_count(self):
return len(self.G.edges())
def get_current_permission_level_node_count(self, permission_level):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) == permission_level:
count += 1
return count
def get_current_protected_node_count(self):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) != None:
count += 1
return count
class Buchi(object):
"""
construct buchi automaton graph
"""
def __init__(self, task):
"""
initialization
:param task: task specified in LTL
"""
# task specified in LTL
self.formula = task.formula
self.subformula = task.subformula
self.number_of_robots = task.number_of_robots
# graph of buchi automaton
self.buchi_graph = DiGraph(type='buchi', init=[], accept=[])
# minimal length (in terms of number of transitions) between a pair of nodes
self.min_length = dict()
def construct_buchi_graph(self):
"""
parse the output of the program ltl2ba and build the buchi automaton
"""
# directory of the program ltl2ba
dirname = os.path.dirname(__file__)
# output of the program ltl2ba
output = subprocess.check_output(dirname + "/ltl2ba -f \"" +
self.formula + "\"",
shell=True).decode("utf-8")
# find all states/nodes in the buchi automaton
state_re = re.compile(r'\n(\w+):\n\t')
state_group = re.findall(state_re, output)
# find initial and accepting states
init = [s for s in state_group if 'init' in s]
print('init', init)
accept = [s for s in state_group if 'accept' in s]
# finish the inilization of the graph of the buchi automaton
self.buchi_graph.graph['init'] = init
self.buchi_graph.graph['accept'] = accept
order_key = list(self.subformula.keys())
order_key.sort(reverse=True)
# for each state/node, find it transition relations
for state in state_group:
# add node
self.buchi_graph.add_node(state)
# loop over all transitions starting from current state
state_if_fi = re.findall(state + r':\n\tif(.*?)fi', output,
re.DOTALL)
if state_if_fi:
relation_group = re.findall(r':: (\(.*?\)) -> goto (\w+)\n\t',
state_if_fi[0])
for symbol, next_state in relation_group:
# delete edges with multiple subformulas
if ' && ' in symbol: continue
# whether the edge is feasible in terms of atomic propositions
for k in order_key:
symbol = symbol.replace('e{0}'.format(k),
self.subformula[k])
# get the trurh assignment
truth_table = self.get_truth_assignment(symbol)
# infeasible transition
if not truth_table: continue
# add edge
self.buchi_graph.add_edge(state,
next_state,
truth=truth_table)
else:
state_skip = re.findall(state + r':\n\tskip\n', output,
re.DOTALL)
if state_skip:
self.buchi_graph.add_edge(state, state, truth='1')
def get_truth_assignment(self, symbol):
"""
get one set of truth assignment that makes the symbol true
:param symbol: logical expression which controls the transition
:return: a set of truth assignment enables the symbol
"""
# empty symbol
if symbol == '(1)':
return '1'
# non-empty symbol
else:
exp = symbol.replace('||', '|').replace('&&',
'&').replace('!', '~')
# add extra constraints: a single robot can reside in at most one region
robot_region = self.robot2region(exp)
for robot, region in robot_region.items():
mutual_execlusion = list(combinations(region, 2))
# single label in the symbol
if not mutual_execlusion: continue
for i in range(len(mutual_execlusion)):
mutual_execlusion[i] = '(~(' + ' & '.join(
list(mutual_execlusion[i])) + '))'
exp = exp + '&' + ' & '.join(mutual_execlusion)
# find one truth assignment that makes symbol true using function satisfiable
truth = satisfiable(sympify(exp), algorithm="dpll")
try:
truth_table = dict()
for key, value in truth.items():
truth_table[key.name] = value
except AttributeError:
return False
else:
return truth_table
def get_minimal_length(self):
"""
search the shortest path from a node to another, i.e., # of transitions in the path
:return:
"""
# loop over pairs of buchi states
for head_node in self.buchi_graph.nodes():
for tail_node in self.buchi_graph.nodes():
# head_node = tail_node, and tail_node is an accepting state
if head_node != tail_node and 'accept' in tail_node:
try:
length, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph,
source=head_node,
target=tail_node)
# couldn't find a path from head_node to tail_node
except nx.exception.NetworkXNoPath:
length = np.inf
self.min_length[(head_node, tail_node)] = length
# head_node != tail_node and tail_node is an accepting state
# move 1 step forward to all reachable states of head_node then calculate the minimal length
elif head_node == tail_node and 'accept' in tail_node:
length = np.inf
for suc in self.buchi_graph.succ[head_node]:
try:
len1, _ = nx.algorithms.single_source_dijkstra(
self.buchi_graph, source=suc, target=tail_node)
except nx.exception.NetworkXNoPath:
len1 = np.inf
if len1 < length:
length = len1 + 1
self.min_length[(head_node, tail_node)] = length
def get_feasible_accepting_state(self):
"""
get feasbile accepting/final state, or check whether an accepting state is feaasible
:return:
"""
accept = self.buchi_graph.graph['accept']
self.buchi_graph.graph['accept'] = []
for ac in accept:
for init in self.buchi_graph.graph['init']:
if self.min_length[(init, ac)] < np.inf and self.min_length[
(ac, ac)] < np.inf:
self.buchi_graph.graph['accept'].append(ac)
break
def robot2region(self, symbol):
"""
pair of robot and corresponding regions in the expression
:param symbol: logical expression
:return: robot index : regions
eg: input: exp = 'l1_1 & l3_1 & l4_1 & l4_6 | l3_4 & l5_6'
output: {1: ['l1_1', 'l3_1', 'l4_1'], 4: ['l3_4'], 6: ['l4_6', 'l5_6']}
"""
robot_region = dict()
for r in range(self.number_of_robots):
findall = re.findall(r'(l\d+?_{0})[^0-9]'.format(r + 1), symbol)
if findall:
robot_region[str(r + 1)] = findall
return robot_region
class buchi_graph(object):
""" construct buchi automaton graph
Parameter:
formula: LTL formula specifying task
"""
def __init__(self, formula, formula_comp, exclusion):
self.formula = formula
self.formula_comp = formula_comp
self.exclusion = exclusion
def formulaParser(self):
"""replace letter with symbol
"""
indicator = 'FG'
if [True for i in indicator if i in self.formula]:
self.formula.replace('F', '<>').replace('G', '[]')
def execLtl2ba(self):
""" given formula, exectute the ltl2ba
Parameter:
buchi_str: output string of program ltl2ba (utf-8 format)
"""
dirname = os.path.dirname(__file__)
self.buchi_str = subprocess.check_output(dirname + "/./ltl2ba -f \"" + self.formula + "\"", shell=True).decode("utf-8")
def buchiGraph(self):
"""parse the output of ltl2ba
Parameter:
buchi_graph: Graph of buchi automaton
"""
# find all states
state_re = re.compile(r'\n(\w+):\n\t')
state_group = re.findall(state_re, self.buchi_str)
# find initial and accepting states
init = [s for s in state_group if 'init' in s]
accep = [s for s in state_group if 'accept' in s]
"""
Format:
buchi_graph.node = NodeView(('T0_init', 'T1_S1', 'accept_S1'))
buchi_graph.edges = OutEdgeView([('T0_init', 'T0_init'), ('T0_init', 'T1_S1'),....])
buchi_graph.succ = AdjacencyView({'T0_init': {'T0_init': {'label': '1'}, 'T1_S1': {'label': 'r3'}}})
"""
self.buchi_graph = DiGraph(type='buchi', init=init, accept=accep)
order_key = list(self.formula_comp.keys())
order_key.sort(reverse=True)
for state in state_group:
# for each state, find transition relation
# add node
self.buchi_graph.add_node(state)
state_if_fi = re.findall(state + r':\n\tif(.*?)fi', self.buchi_str, re.DOTALL)
if state_if_fi:
relation_group = re.findall(r':: (\(.*?\)) -> goto (\w+)\n\t', state_if_fi[0])
for (labell, state_dest) in relation_group:
# whether the edge is feasible in terms of unit atomic proposition
label = self.InitialDelInfesEdge(labell)
if not label or label.isspace():
continue
# add edge
for k in order_key:
if k >= 10:
label = label.replace('e_{0}'.format(k), self.formula_comp[k])
else:
label = label.replace('e{0}'.format(k), self.formula_comp[k])
# if '!' in label:
# label = self.PutNotInside(label)
self.buchi_graph.add_edge(state, state_dest, label=label)
return self.buchi_graph
def ShorestPathBtRg(self, regions):
"""
calculate shoresr path between any two labeled regions
:param regions: regions
:return: dict (region, region) : length
"""
polys = [[vg.Point(0.4, 1.0), vg.Point(0.4, 0.7), vg.Point(0.6, 0.7), vg.Point(0.6, 1.0)],
[vg.Point(0.3, 0.2), vg.Point(0.3, 0.0), vg.Point(0.7, 0.0), vg.Point(0.7, 0.2)]]
g = vg.VisGraph()
g.build(polys, status=False)
min_len_region = dict()
for key1, value1 in regions.items():
for key2, value2 in regions.items():
init = value1[:2]
tg = value2[:2]
# shorest path between init and tg point
shortest = g.shortest_path(vg.Point(init[0], init[1]), vg.Point(tg[0], tg[1]))
# (key2, key1) is already checked
if (key2, key1) in min_len_region.keys():
min_len_region[(key1, key2)] = min_len_region[(key2, key1)]
else:
# different regions
if key1 != key2:
dis = 0
for i in range(len(shortest)-1):
dis = dis + np.linalg.norm(np.subtract((shortest[i].x, shortest[i].y), (shortest[i+1].x, shortest[i+1].y)))
min_len_region[(key1, key2)] = dis
# same region
else:
min_len_region[(key1, key2)] = 0
return min_len_region
def RobotRegion(self, exp, robot):
"""
pair of robot and corresponding regions in the expression
:param exp: logical expression
:param robot: # of robots
:return: dic of robot index : regions
exp = 'l1_1 & l3_1 & l4_1 & l4_6 | l3_4 & l5_6'
{1: ['l1_1', 'l3_1', 'l4_1'], 4: ['l3_4'], 6: ['l4_6', 'l5_6']}
"""
robot_region_dict = dict()
for r in range(robot):
findall = re.findall(r'(l\d+?_{0})[^0-9]'.format(r + 1), exp)
if findall:
robot_region_dict[str(r + 1)] = findall
return robot_region_dict
def FeasTruthTable(self, exp, robot_region):
"""
Find feasible truth table to make exp true
:param exp: expression
:return:
"""
if exp == '(1)':
return '1'
sgl_value = []
for key, value in robot_region.items():
if len(value) == 1:
sgl_value.append(value[0])
# set all to be false
exp1 = to_cnf(exp)
value_in_exp = [value.name for value in exp1.atoms()]
subs = {true_rb_rg: False for true_rb_rg in value_in_exp}
if exp1.subs(subs):
return subs
# set one to be true, the other to be false
for prod in itertools.product(*robot_region.values()):
exp1 = exp
# set one specific item to be true
for true_rb_rg in prod:
# set the other to be false
value_cp = list(robot_region[true_rb_rg.split('_')[1]])
if len(value_cp) > 1:
value_cp.remove(true_rb_rg)
# replace the rest with same robot to be ~
for v_remove in value_cp:
exp1 = exp1.replace(v_remove, '~' + true_rb_rg)
# simplify
exp1 = to_cnf(exp1)
# all value in expression
value_in_exp = [value.name for value in exp1.atoms()]
# all single value in expression
sgl_value_in_exp = [value for value in value_in_exp if value in sgl_value]
# not signle value in expression
not_sgl_value_in_exp = [value for value in value_in_exp if value not in sgl_value]
subs1 = {true_rb_rg: True for true_rb_rg in not_sgl_value_in_exp}
tf = [False, True]
# if type(exp1) == Or:
# tf = [False, True]
if len(sgl_value_in_exp):
for p in itertools.product(*[tf] * len(sgl_value_in_exp)):
subs2 = {sgl_value_in_exp[i]: p[i] for i in range(len(sgl_value_in_exp))}
subs = {**subs1, **subs2}
if exp1.subs(subs):
return subs
else:
if exp1.subs(subs1):
return subs1
return []
def DelInfesEdge(self, robot):
"""
Delete infeasible edge
:param buchi_graph: buchi automaton
:param robot: # robot
"""
TobeDel = []
# print(self.buchi_graph.number_of_edges())
i = 0
for edge in self.buchi_graph.edges():
i = i+1
# print(i)
b_label = self.buchi_graph.edges[edge]['label']
# multiple labels
if ') && (' in b_label:
TobeDel.append(edge)
continue
if b_label != '(1)':
exp = b_label.replace('||', '|').replace('&&', '&').replace('!', '~')
truth = satisfiable(exp, algorithm="dpll")
truth_table = dict()
for key, value in truth.items():
truth_table[key.name] = value
if not truth_table:
TobeDel.append(edge)
else:
self.buchi_graph.edges[edge]['truth'] = truth_table
else:
self.buchi_graph.edges[edge]['truth'] = '1'
for edge in TobeDel:
self.buchi_graph.remove_edge(edge[0], edge[1])
# print(self.buchi_graph.number_of_edges())
def InitialDelInfesEdge(self, orig_label):
div_by_or = orig_label.split(') || (')
for item in div_by_or:
feas = True
for excl in self.exclusion:
# mutual exclusion term exist
if excl[0] in item and excl[1] in item and '!{0}'.format(excl[0]) not in item and '!{0}'.format(excl[1]) not in item:
feas = False
break
if not feas:
item = item.strip('(').strip(')')
item = '(' + item + ')'
orig_label = orig_label.replace(' '+item+' ||', '').replace(item+' || ','').replace(' || '+item,'').replace(item,'')
return orig_label
def MinLen(self):
"""
search the shorest path from a node to another, weight = 1, i.e. # of state in the path
:param buchi_graph:
:return: dict of pairs of node : length of path
"""
min_qb_dict = dict()
for node1 in self.buchi_graph.nodes():
for node2 in self.buchi_graph.nodes():
if node1 != node2 and 'accept' in node2:
try:
l, _ = nx.algorithms.single_source_dijkstra(self.buchi_graph, source=node1, target=node2)
except nx.exception.NetworkXNoPath:
l = np.inf
min_qb_dict[(node1, node2)] = l
elif node1 == node2 and 'accept' in node2:
l = np.inf
for succ in self.buchi_graph.succ[node1]:
try:
l0, _ = nx.algorithms.single_source_dijkstra(self.buchi_graph, source=succ, target=node1)
except nx.exception.NetworkXNoPath:
l0 = np.inf
if l0 < l:
l = l0 + 1
min_qb_dict[(node1, node2)] = l
return min_qb_dict
# def MinLen_Cost(self):
# """
# search the shorest path from a node to another, weight = cost
# :param buchi_graph:
# :return: dict of pairs of node : length of path
# """
# min_qb_dict = dict()
# for node1 in self.buchi_graph.nodes():
# for node2 in self.buchi_graph.nodes():
# c = np.inf
# if node1 != node2:
# try:
# path = nx.all_simple_paths(self.buchi_graph, source=node1, target=node2)
# for i in range(len(path)-2):
# word_init = self.buchi_graph.edges[(path[i], path[i+1])]['label']
# word_tg = self.buchi_graph.edges[(path[i+1], path[i+2])]['label']
# # calculate distance travelled from word_init to word_tg
# t_s_b = True
# # split label with ||
# label_init = word_init.split('||')
# label_tg = word_tg.split('||')
# for label in b_label:
# t_s_b = True
# # spit label with &&
# atomic_label = label.split('&&')
# for a in atomic_label:
# a = a.strip()
# a = a.strip('(')
# a = a.strip(')')
# if a == '1':
# continue
# # whether ! in an atomic proposition
# if '!' in a:
# if a[1:] in x_label:
# t_s_b = False
# break
# else:
# if not a in x_label:
# t_s_b = False
# break
# # either one of || holds
# if t_s_b:
# return t_s_b
# except nx.exception.NetworkXNoPath:
# c = np.inf
# else:
# c = 0
# min_qb_dict[(node1, node2)] = c
#
# return min_qb_dict
def FeasAcpt(self, min_qb):
"""
delte infeasible final state
:param buchi_graph: buchi automaton
:param min_qb: dict of pairs of node : length of path
"""
accept = self.buchi_graph.graph['accept']
for acpt in accept:
if min_qb[(self.buchi_graph.graph['init'][0], acpt)] == np.inf or min_qb[(acpt, acpt)] == np.inf:
self.buchi_graph.graph['accept'].remove(acpt)
def PutNotInside(self, str):
"""
put not inside the parenthesis !(p1 && p2) -> !p1 or !p2
:param str: old
:return: new
"""
substr = re.findall("(!\(.*?\))", str) # ['!(p1 && p2)', '!(p4 && p5)']
for s in substr:
oldstr = s.strip().strip('!').strip('(').strip(')')
nstr = ''
for ss in oldstr.split():
if '&&' in ss:
nstr = nstr + ' or '
elif 'or' in ss:
nstr = nstr + ' && '
else:
nstr = nstr + '!' + ss
str = str.replace(s, nstr)
return str
def label2sat(self):
for edge in self.buchi_graph.edges():
label = self.buchi_graph.edges[edge]['label']
label = label.replace('||', '|').replace('&&', '&').replace('!', '~')
exp1 = to_cnf(label)
self.buchi_graph.edges[edge]['label'] = exp1
class BoardGraph(object):
def __init__(self, board_class=Board):
self.graph = self.start = self.last = self.closest = None
self.min_domino_count = None
self.board_class = board_class
def walk(self, board, size_limit=maxsize):
pending_nodes = []
self.graph = DiGraph()
self.start = board.display(cropped=True)
self.graph.add_node(self.start)
pending_nodes.append(self.start)
self.last = self.start
while pending_nodes:
if len(self.graph) >= size_limit:
raise GraphLimitExceeded(size_limit)
state = pending_nodes.pop()
board = self.board_class.create(state, border=1)
for move, new_state in self.generate_moves(board):
if not self.graph.has_node(new_state):
# new node
self.graph.add_node(new_state)
pending_nodes.append(new_state)
self.graph.add_edge(state, new_state, move=move)
return set(self.graph.nodes())
def generate_moves(self, board):
""" Generate all moves from the board's current state.
:param Board board: the current state
:return: a generator of (state, move_description) tuples
"""
dominoes = set(board.dominoes)
domino_count = len(dominoes)
if self.min_domino_count is None or domino_count < self.min_domino_count:
self.min_domino_count = domino_count
self.last = board.display(cropped=True)
for domino in dominoes:
dx, dy = domino.direction
yield from self.try_move(domino, dx, dy)
yield from self.try_move(domino, -dx, -dy)
def try_move(self, domino, dx, dy):
try:
new_state = self.move(domino, dx, dy)
move = domino.describe_move(dx, dy)
yield move, new_state
except BadPositionError:
pass
def move(self, domino, dx, dy):
""" Move a domino and calculate the new board state.
Afterward, put the board back in its original state.
@return: the new board state
@raise BadPositionError: if the move is illegal
"""
domino.move(dx, dy)
try:
board = domino.head.board
if not board.isConnected():
raise BadPositionError('Board is not connected.')
if board.hasLoner():
raise BadPositionError('Board has a lonely domino.')
return board.display(cropped=True)
finally:
domino.move(-dx, -dy)
def get_solution(self, return_partial=False):
""" Find a solution from the graph of moves.
@param return_partial: If True, a partial solution will be returned if no
solution exists.
@return: a list of strings describing each move. Each string is two
digits describing the domino that moved plus a letter to show the
direction.
"""
solution = []
goal = self.closest if return_partial else self.last or ''
solution_nodes = shortest_path(self.graph, self.start, goal)
for i in range(len(solution_nodes) - 1):
source, target = solution_nodes[i:i + 2]
solution.append(self.graph[source][target]['move'])
return solution
def get_choice_counts(self):
solution_nodes = shortest_path(self.graph, self.start, self.last)
return [len(self.graph[node]) for node in solution_nodes[:-1]]
def get_average_choices(self):
choices = self.get_choice_counts()
return sum(choices) / float(len(choices)) if choices else maxsize
def get_max_choices(self):
choices = self.get_choice_counts()
return max(choices) if choices else maxsize
def load(fname):
def clean_bool(string):
if string == "0":
return None
else:
return string
def to_bool(string):
if string == "1" or string == "True":
return True
elif string == "0" or string == "False":
return False
else:
return string
def to_float(string):
if string == "None":
return None
try:
return float(string)
except:
return string
mode = "node0"
nodes = []
edges = []
volatiles = set()
outputs = None
inputs = None
named_ranges = {}
infile = gzip.GzipFile(fname, 'r')
for line in infile.read().splitlines():
if line == "====":
mode = "node0"
continue
if line == "-----":
cellmap_temp = {n.address(): n for n in nodes}
Range = RangeFactory(cellmap_temp)
mode = "node0"
continue
elif line == "edges":
cellmap = {n.address(): n for n in nodes}
mode = "edges"
continue
elif line == "outputs":
mode = "outputs"
continue
elif line == "inputs":
mode = "inputs"
continue
elif line == "named_ranges":
mode = "named_ranges"
continue
if mode == "node0":
[address, formula, python_expression, is_range, is_named_range, is_volatile, should_eval] = line.split(SEP)
formula = clean_bool(formula)
python_expression = clean_bool(python_expression)
is_range = to_bool(is_range)
is_named_range = to_bool(is_named_range)
is_volatile = to_bool(is_volatile)
should_eval = should_eval
mode = "node1"
elif mode == "node1":
if is_range:
reference = json.loads(line) if is_volatile else line # in order to be able to parse dicts
vv = Range(reference)
if is_volatile:
if not is_named_range:
address = vv.name
volatiles.add(address)
cell = Cell(address, None, vv, formula, is_range, is_named_range, should_eval)
cell.python_expression = python_expression
nodes.append(cell)
else:
value = to_bool(to_float(line))
cell = Cell(address, None, value, formula, is_range, is_named_range, should_eval)
cell.python_expression = python_expression
if formula:
if 'OFFSET' in formula or 'INDEX' in formula:
volatiles.add(address)
cell.compile()
nodes.append(cell)
elif mode == "edges":
source, target = line.split(SEP)
edges.append((cellmap[source], cellmap[target]))
elif mode == "outputs":
outputs = line.split(SEP)
elif mode == "inputs":
inputs = line.split(SEP)
elif mode == "named_ranges":
k,v = line.split(SEP)
named_ranges[k] = v
G = DiGraph(data = edges)
print "Graph loading done, %s nodes, %s edges, %s cellmap entries" % (len(G.nodes()),len(G.edges()),len(cellmap))
return (G, cellmap, named_ranges, volatiles, outputs, inputs)