def _congruence_graph(self): """Form a graph whose vertex set is the QNF basis. The edges store the information which makes two vertices congruent.""" from networkx.classes.digraph import DiGraph basis = self.quasinormal_basis min_exp = basis.minimal_expansion_for(self) terminal, initial = self.semi_infinite_end_points() endpts = sorted(terminal + initial) G = DiGraph() orbit_generators = set(min_exp) orbit_generators.update(endpts) #1. Add an edge for every direct congruence relationship. for gen in orbit_generators: type, images, _ = self.orbit_type(gen, basis) for power, img in images.items(): images[power] = basis.test_above(img) congruent_pairs = permutations(images.items(), 2) for (pow1, (head1, tail1)), (pow2, (head2, tail2)) in congruent_pairs: if head1 == head2: continue data = dict(start_tail = tail1, power = pow2 - pow1, end_tail = tail2) G.add_edge(head1, head2, data) assert self.repeated_image(head1.extend(tail1), pow2 - pow1) == head2.extend(tail2) return G
def remove_low_frequency_label(
community_label_list_for_nodes: multivalued_dict) -> DiGraph:
from sklearn.ensemble import IsolationForest
digraph_of_node_labels_and_frequencies = DiGraph()
for graph_of_node, label_list_of_nodes in community_label_list_for_nodes.items(
):
digraph_of_node_labels_and_frequencies.add_node(graph_of_node)
label_set = set(label_list_of_nodes)
dict_of_frequency_of_label = dict(
sorted([(label_list_of_nodes.count(label_item), label_item)
for label_item in label_set],
key=lambda frequency_and_label: frequency_and_label[0],
reverse=True))
dict_of_sn_and_frequency = dict([
(sequence_number, frequency_of_label)
for sequence_number, frequency_of_label in enumerate(
dict_of_frequency_of_label.keys(), 1)
])
list_of_mapping_points = []
for sequence_number, frequency_of_label in dict_of_sn_and_frequency.items(
):
list_of_mapping_points.extend([[sequence_number]] *
frequency_of_label)
clf = IsolationForest(n_estimators=120, contamination='auto')
clf.fit(list_of_mapping_points)
for sequence_number, frequency_of_label in dict_of_sn_and_frequency.items(
):
if clf.predict([[sequence_number]])[0] == 1:
label_item = dict_of_frequency_of_label.__getitem__(
frequency_of_label)
digraph_of_node_labels_and_frequencies.add_edge(
graph_of_node, label_item, weight=frequency_of_label)
return digraph_of_node_labels_and_frequencies
class MockSimpleGraphDB\
(
Generic[MockVertexData],
PartiallyStatefulDirectedGraphInterface[SimpleGraphKey, MockVertexData]
):
'''
Type for mocking a simple database.
'''
data: DiGraph
def __init__(self) -> None:
self.data = DiGraph()
def write_stateful_vertex_(self, label: SimpleGraphKey,
data: MockVertexData, *args: Any,
**kwargs: Any) -> None:
self.data.add_node(node_for_adding=label,
memento=data) # type: ignore unkonwn
def write_stateless_directed_edge_(self, source: SimpleGraphKey,
destination: SimpleGraphKey, *args: Any,
**kwargs: Any) -> None:
self.data.add_edge(
u_of_edge=source,
v_of_edge=destination) # type: ignore unkonwn member
class buchi_graph(object):
""" construct buchi automaton graph
Parameter:
formula: LTL formula specifying task
"""
def __init__(self, formula):
self.formula = formula
def formulaParser(self):
"""replace letter with symbol
"""
indicator = 'FG'
if [True for i in indicator if i in self.formula]:
self.formula.replace('F', '<>').replace('G', '[]')
def execLtl2ba(self):
""" given formula, exectute the ltl2ba
Parameter:
buchi_str: output string of program ltl2ba (utf-8 format)
"""
dirname = os.path.dirname(__file__)
self.buchi_str = subprocess.check_output(dirname + "/./ltl2ba -f \"" +
self.formula + "\"",
shell=True).decode("utf-8")
def buchiGraph(self):
"""parse the output of ltl2ba
Parameter:
buchi_graph: Graph of buchi automaton
"""
# find all states
state_re = re.compile(r'\n(\w+):\n\t')
state_group = re.findall(state_re, self.buchi_str)
# find initial and accepting states
init = [s for s in state_group if 'init' in s]
accep = [s for s in state_group if 'accept' in s]
"""
Format:
buchi_graph.node = NodeView(('T0_init', 'T1_S1', 'accept_S1'))
buchi_graph.edges = OutEdgeView([('T0_init', 'T0_init'), ('T0_init', 'T1_S1'),....])
buchi_graph.succ = AdjacencyView({'T0_init': {'T0_init': {'label': '1'}, 'T1_S1': {'label': 'r3'}}})
"""
self.buchi_graph = DiGraph(type='buchi', init=init, accept=accep)
for state in state_group:
# for each state, find transition relation
# add node
self.buchi_graph.add_node(state)
state_if_fi = re.findall(state + r':\n\tif(.*?)fi', self.buchi_str,
re.DOTALL)
if state_if_fi:
relation_group = re.findall(r':: \((.*?)\) -> goto (\w+)\n\t',
state_if_fi[0])
for (label, state_dest) in relation_group:
# add edge
self.buchi_graph.add_edge(state, state_dest, label=label)
return self.buchi_graph
def build_crm(roadmap_dir='./build_roadmap/roadmap_2D.p',
wsn_rate_dir='./wsn_routing/rate_map.p',
ws_img_dir='./build_roadmap/ws_img.p'):
'''
build combined roadmap, crm
via load roadmap and merge in wsn_rate info
'''
roadmap_edges = pickle.load(open(roadmap_dir, 'rb'))
wsn_rate, wifis_loc, sink_loc = pickle.load(open(wsn_rate_dir, 'rb'))
img = pickle.load(open(ws_img_dir, 'rb'))
crm = DiGraph(name='combined_model',
ws_img=img,
wifis=wifis_loc,
sink=sink_loc)
for e in roadmap_edges:
(f_node, t_node) = e
near_f_node = min(wsn_rate.keys(), key=lambda p: dist_2D(p, f_node))
f_node_rate = wsn_rate[near_f_node]
crm.add_node(f_node, rate=f_node_rate)
near_t_node = min(wsn_rate.keys(), key=lambda p: dist_2D(p, t_node))
t_node_rate = wsn_rate[near_t_node]
crm.add_node(t_node, rate=t_node_rate)
# interpolation
dist_e = dist_2D(f_node, t_node)
f_t_rate = intp_rate(f_node_rate, t_node_rate)
crm.add_edge(f_node, t_node, rate=f_t_rate, dist=dist_e)
t_f_rate = intp_rate(t_node_rate, f_node_rate)
crm.add_edge(t_node, f_node, rate=t_f_rate, dist=dist_e)
print '---crm constructed from %s and %s, %d nodes, %d edges---' % (
roadmap_dir, wsn_rate_dir, len(crm.nodes()), len(crm.edges()))
return crm
def _digraph(self, index):
g = DiGraph()
g.add_node(index[self])
for _, (fun, _) in self.calls:
g.add_node(index[fun])
g.add_edge(index[fun], index[self])
g = networkx.compose(g, fun._digraph(index))
return g
def load_graph(file_or_path:FileOrPath, ext:str=None) -> DiGraph:
graph = DiGraph()
for adj in load_jsonl(text_file_for(file_or_path)):
src = adj[0]
graph.add_node(src)
for dst in adj[1:]:
graph.add_edge(src, dst)
return graph
def hoftask(init, buchi_graph, regions):
h_task = DiGraph(type='subtask')
try:
label = to_dnf(
buchi_graph.edges[(buchi_graph.graph['init'][0],
buchi_graph.graph['init'][0])]['label'], True)
except KeyError:
# no self loop
label = to_dnf('0')
init_node = State((init, buchi_graph.graph['init'][0]), label)
h_task.add_node(init_node)
open_set = list()
open_set.append(init_node)
explore_set = list()
while open_set:
curr = open_set.pop(0)
# assume co-safe
# if curr.q in buchi_graph.graph['accept']:
# continue
# print(curr)
for q_b in buchi_graph.succ[curr.q]:
try:
label = to_dnf(buchi_graph.edges[(q_b, q_b)]['label'], True)
except KeyError:
# no self loop
label = to_dnf('0')
if q_b != curr.q: # and q_b not in buchi_graph.graph['accept']:
# region corresponding to the label/word
edge_label = (buchi_graph.edges[(curr.q, q_b)]['label'])
if edge_label == '(1)':
x_set = [curr.x]
else:
x_set = target(to_dnf(edge_label), regions)
if x_set:
for x in x_set:
cand = State((x, q_b), label)
# print(cand)
if cand not in open_set and cand not in explore_set:
open_set.append(cand)
# check the match with each subtask in h_task
if not match(h_task, curr, cand):
# print(cand)
h_task.add_node(cand)
h_task.add_edge(curr, cand)
# continue
# roots for accepting states
# if q_b in buchi_graph.graph['accept']:
# h_task.add_edge(curr, cand)
explore_set.append(curr)
# for x in h_task.nodes:
# print(x)
return h_task
def _transition_graph():
expected_graph = DiGraph()
expected_graph.add_node('85',
coords=(-3.8347478, -38.592189),
datetime=['2017-09-02 22:00:27'],
freq_source=1,
freq_target=0)
expected_graph.add_node('673',
coords=(-3.8235834, -38.590389),
datetime=['2017-09-02 22:01:36'],
freq_source=0,
freq_target=0)
expected_graph.add_node('394',
coords=(-3.813889, -38.5904445),
datetime=['2017-09-02 22:03:08'],
freq_source=0,
freq_target=0)
expected_graph.add_node('263',
coords=(-3.9067654, -38.5907723),
datetime=['2017-09-02 22:03:46'],
freq_source=0,
freq_target=0)
expected_graph.add_node('224',
coords=(-3.8857223, -38.5928892),
datetime=['2017-09-02 22:07:19'],
freq_source=0,
freq_target=0)
expected_graph.add_node('623',
coords=(-3.8828723, -38.5929789),
datetime=['2017-09-02 22:07:40'],
freq_source=0,
freq_target=1)
expected_graph.add_edge('85',
'673',
weight=1,
mean_times='0 days 00:01:09')
expected_graph.add_edge('673',
'394',
weight=1,
mean_times='0 days 00:01:32')
expected_graph.add_edge('394',
'263',
weight=1,
mean_times='0 days 00:00:38')
expected_graph.add_edge('263',
'224',
weight=1,
mean_times='0 days 00:03:33')
expected_graph.add_edge('224',
'623',
weight=1,
mean_times='0 days 00:00:21')
return expected_graph
class SimpleGraphDB\
(
Generic[SimpleVertexLabel, VertexData],
PartiallyStatefulDirectedGraphInterface[SimpleVertexLabel, VertexData],
StatefulVertexGraphLoaderInterface[SimpleVertexLabel, VertexData]
):
'''
Class that can write stateful vertices and stateless directed edges into a graph-like structure.
'''
__graph: DiGraph
'''
Property and dunder methods
'''
def __init__(self, *args: Any, **kwargs: Any) -> None:
'''
Sets up a `networkx.DiGraph` structure for writing stateful vertices and stateless direWcted edges.
'''
self.__graph = DiGraph()
return None
@property
def _graph(self) -> DiGraph: return self.__graph
'''
ABC extensions.
'''
def write_stateful_vertex_(self, label: SimpleVertexLabel, data: VertexData, *args: Any, **kwargs: Any) -> None:
'''
Writes a vertex with this `label` associated with this `data` into a `networkx.DiGraph` object.
'''
self.__graph.add_node(node_for_adding = label, data = data)
return None
def write_stateless_directed_edge_(self, source: SimpleVertexLabel, destination: SimpleVertexLabel, *args: Any, **kwargs: Any) -> None:
'''
Writes an unlabelled edge between a vertex with this `source` label and one with this `destination` label into a `networkx.DiGraph` object.
'''
self.__graph.add_edge(u_of_edge = source, v_of_edge = destination)
return None
def load_stateful_vertex(self, label: SimpleVertexLabel, *args: Any, **kwargs: Any) -> VertexData:
'''
Loads the `VertexData` associated with this `label` from a `networkx.DiGraph` object.
'''
return self.__graph.nodes[label].get('data')
def add_edge(self, u, v=None):
if v is None: (u,v)=u # no v given, assume u is an edge tuple
if self.has_edge(u,v): return # no parallel edges
elif u in self and v in self:
raise NetworkXError, "adding edge %s-%s not allowed in tree"%(u,v)
elif u in self or v in self:
DiGraph.add_edge(self,u,v) # u->v
return
elif len(self.adj)==0: # first leaf
DiGraph.add_edge(self,u,v) # u->v
return
else:
raise NetworkXError("adding edge %s-%s not allowed in tree"%(u,v))
def buchi_from_ltl(formula,Type):
promela_string = run_ltl2ba(formula)
symbols = find_symbols(formula)
edges = parse_ltl(promela_string)
(states, initials, accepts) = find_states(edges)
buchi = DiGraph(type=Type, initial=initials, accept=accepts, symbols=symbols)
for state in states:
buchi.add_node(state)
for (ef,et) in edges.keys():
guard_formula = edges[(ef,et)]
guard_expr = parse_guard(guard_formula)
buchi.add_edge(ef, et, guard=guard_expr, guard_formula=guard_formula)
return buchi
def load_from_graphml(path):
"""
:type path: str
:rtype: networkx.classes.graph.Graph
"""
#: :type : networkx.classes.graph.Graph
g1 = nx.read_graphml(path)
#: :type graph: networkx.classes.digraph.DiGraph
g2 = DiGraph()
typetest = re.compile(r"([a-zA-z]+)(\d*)")
max_qualifiers = dict(crossing=0, poi=0, plcs=0)
node_mapping = dict()
for node, data in g1.nodes_iter(data=True):
m = typetest.match(data["label"])
if m is None:
raise(SALMAException("Wrong label format for node {}!".format(node)))
loctype = m.group(1)
if loctype in ["c"]:
loctype = "crossing"
elif loctype in ["p"]:
loctype = "poi"
elif loctype in ["pl"]:
loctype = "plcs"
if loctype not in ["poi", "plcs", "crossing"]:
raise(SALMAException("Wrong loctype for node {}: {}".format(node, loctype)))
qualifier = m.group(2)
if len(qualifier) == 0:
qualifier = max_qualifiers[loctype] + 1
nid = data["label"] + str(qualifier)
else:
nid = data["label"]
max_qualifiers[loctype] = max(max_qualifiers[loctype], qualifier)
pos = (round(float(data["x"])), round(float(data["y"])))
g2.add_node(nid, pos=pos, scaled_pos=pos, loctype=loctype)
node_mapping[node] = nid
for u, v in g1.edges_iter():
n1 = node_mapping[u]
n2 = node_mapping[v]
g2.add_edge(n1, n2)
g2.add_edge(n2, n1)
MapGenerator.__add_roadlengths(g2)
return g2
class BoardGraph(object):
def walk(self, board, size_limit=maxint):
pending_nodes = []
self.graph = DiGraph()
self.start = board.display(cropped=True)
self.graph.add_node(self.start)
pending_nodes.append(self.start)
self.min_domino_count = len(board.dominoes)
while pending_nodes:
if len(self.graph) >= size_limit:
raise GraphLimitExceeded(size_limit)
state = pending_nodes.pop()
board = Board.create(state, border=1)
dominoes = set(board.dominoes)
self.min_domino_count = min(self.min_domino_count, len(dominoes))
for domino in dominoes:
dx, dy = domino.direction
self.try_move(state, domino, dx, dy, pending_nodes)
self.try_move(state, domino, -dx, -dy, pending_nodes)
self.last = state
return set(self.graph.nodes())
def try_move(self, old_state, domino, dx, dy, pending_states):
try:
new_state = self.move(domino, dx, dy)
move = domino.describe_move(dx, dy)
if not self.graph.has_node(new_state):
# new node
self.graph.add_node(new_state)
pending_states.append(new_state)
self.graph.add_edge(old_state, new_state, move=move)
except BoardError:
pass
def move(self, domino, dx, dy):
""" Move a domino and calculate the new board state.
Afterward, put the board back in its original state.
@return: the new board state
@raise BoardError: if the move is illegal
"""
domino.move(dx, dy)
try:
board = domino.head.board
if not board.isConnected():
raise BoardError('Board is not connected.')
if board.hasLoner():
raise BoardError('Board has a lonely domino.')
return board.display(cropped=True)
finally:
domino.move(-dx, -dy)
def add_edge(self, u, v=None):
if v is None: (u, v) = u # no v given, assume u is an edge tuple
if self.has_edge(u, v): return # no parallel edges
elif u in self and v in self:
raise NetworkXError, "adding edge %s-%s not allowed in tree" % (u,
v)
elif u in self or v in self:
DiGraph.add_edge(self, u, v) # u->v
return
elif len(self.adj) == 0: # first leaf
DiGraph.add_edge(self, u, v) # u->v
return
else:
raise NetworkXError("adding edge %s-%s not allowed in tree" %
(u, v))
def construct_arg_crm(self):
arg_crm = DiGraph()
for wp_f in self.crm.nodes_iter():
for d_f in iter(self.data_set):
s_f = (wp_f, d_f)
arg_crm.add_node(s_f)
for wp_t in self.crm.neighbors(wp_f):
d_evolve = d_f - self.crm.edge[wp_f][wp_t][
'rate'] * self.sample_time - 2 * self.quant_size
d_t = [d for d in self.data_set if d >= d_evolve][0]
s_t = (wp_t, d_t)
arg_crm.add_edge(s_f, s_t, weight=1.0)
print '---static argumented crm constructed: %d nodes, %d edges----' % (
len(arg_crm.nodes()), len(arg_crm.edges()))
self.arg_crm = arg_crm
def _get_projected_digraph(bg, matching, top_nodes):
"""
"""
digraph = DiGraph()
digraph.add_nodes_from(top_nodes)
for n in top_nodes:
# Add in-edges
if n in matching:
for t in bg[matching[n]]:
if t != n:
digraph.add_edge(t, n)
# Add out-edges
for b in bg[n]:
if b in matching and matching[b] != n:
digraph.add_edge(n, matching[b])
return digraph
def buchi_from_ltl(formula, Type1):
promela_string = "never { /* G(goods->Xdepot)&&G(depot->Xgoods)&&(G(!b)&&G(!door||open)) */ accept_init : /* init */ if :: (!goods && !depot && !b && !door) || (!goods && !depot && !b && open) -> goto accept_init :: (!goods && !b && !door) || (!goods && !b && open) -> goto accept_S2 :: (!depot && !b && !door) || (!depot && !b && open) -> goto accept_S3 :: (!b && !door) || (!b && open) -> goto accept_S4 fi;accept_S2 : /* 1 */ if :: (goods && !depot && !b && !door) || (goods && !depot && !b && open) -> goto accept_S3 :: (goods && !b && !door) || (goods && !b && open) -> goto accept_S4 fi;accept_S3 : /* 2 */ if :: (!goods && depot && !b && !door) || (!goods && depot && !b && open) -> goto accept_S2 :: (depot && !b && !door) || (depot && !b && open) -> goto accept_S4 fi;accept_S4 : /* 3 */ if :: (goods && depot && !b && !door) || (goods && depot && !b && open) -> goto accept_S4 fi;}"
#promela_string = run_ltl2ba(formula)
symbols = find_symbols(formula)
edges = parse_ltl(promela_string)
(states, initials, accepts) = find_states(edges)
buchi = DiGraph(type=Type,
initial=initials,
accept=accepts,
symbols=symbols)
for state in states:
buchi.add_node(state)
for (ef, et) in edges.keys():
guard_formula = edges[(ef, et)]
guard_expr = parse_guard(guard_formula)
buchi.add_edge(ef, et, guard=guard_expr, guard_formula=guard_formula)
return buchi
def DuoBA_from_ltls(hard_spec, soft_spec):
hard_buchi = buchi_from_ltl(hard_spec, 'hard_buchi')
soft_buchi = buchi_from_ltl(soft_spec, 'soft_buchi')
hard_symbols = hard_buchi.graph['symbols']
soft_symbols = soft_buchi.graph['symbols']
symbols = set(hard_symbols).union(set(soft_symbols))
DuoBA = DiGraph(type='safe_buchi',
hard=hard_buchi,
soft=soft_buchi,
symols=symbols)
initial = set()
accept = set()
for (h_node, s_node, l) in cartesian_product(hard_buchi.nodes(),
soft_buchi.nodes(), [1, 2]):
DuoNode = (h_node, s_node, l)
DuoBA.add_node(DuoNode, hard=h_node, soft=s_node, level=l)
if (h_node in hard_buchi.graph['initial']
and s_node in soft_buchi.graph['initial'] and l == 1):
initial.add(DuoNode)
if (h_node in hard_buchi.graph['accept'] and l == 1):
accept.add(DuoNode)
DuoBA.graph['accept'] = accept
DuoBA.graph['initial'] = initial
for f_duonode in DuoBA.nodes_iter():
for t_duonode in DuoBA.nodes_iter():
f_h_node, f_s_node, f_level = check_duo_attr(DuoBA, f_duonode)
t_h_node, t_s_node, t_level = check_duo_attr(DuoBA, t_duonode)
if (t_h_node not in DuoBA.graph['hard'].neighbors(f_h_node) or
t_s_node not in DuoBA.graph['soft'].neighbors(f_s_node)):
continue
# relaxed because no common input alphabets are enabled
hardguard = DuoBA.graph['hard'].edge[f_h_node][t_h_node]['guard']
softguard = DuoBA.graph['soft'].edge[f_s_node][t_s_node]['guard']
if ((f_h_node not in DuoBA.graph['hard'].graph['accept']
and f_level == 1 and t_level == 1)
or (f_h_node in DuoBA.graph['hard'].graph['accept']
and f_level == 1 and t_level == 2)
or (f_s_node not in DuoBA.graph['soft'].graph['accept']
and f_level == 2 and t_level == 2)
or (f_s_node in DuoBA.graph['soft'].graph['accept']
and f_level == 2 and t_level == 1)):
DuoBA.add_edge(f_duonode,
t_duonode,
hardguard=hardguard,
softguard=softguard)
return DuoBA
def hoftask_no_simplified(init, buchi_graph, regions):
try:
label = to_dnf(
buchi_graph.edges[(buchi_graph.graph['init'][0],
buchi_graph.graph['init'][0])]['label'], True)
except KeyError:
# no self loop
label = to_dnf('0')
init_node = State((init, buchi_graph.graph['init'][0]), label)
h_task = DiGraph(type='subtask', init=init_node)
h_task.add_node(init_node)
open_set = list()
open_set.append(init_node)
explore_set = list()
while open_set:
curr = open_set.pop(0)
for q_b in buchi_graph.succ[curr.q]:
try:
label = to_dnf(buchi_graph.edges[(q_b, q_b)]['label'], True)
except KeyError:
# no self loop
label = to_dnf('0')
if q_b != curr.q:
# region corresponding to the label/word
edge_label = (buchi_graph.edges[(curr.q, q_b)]['label'])
if edge_label == '(1)':
x_set = [curr.x]
elif '~' in edge_label and to_dnf(edge_label).nargs == {
1
}: # ~l3_1
x_set = [curr.x] # !!!!!!!
else:
x_set = target(to_dnf(edge_label), regions)
if x_set:
for x in x_set:
cand = State((x, q_b), label)
# whether cand is already in the h_task
# cand = find_node(cand, h_task)
h_task.add_edge(curr, cand)
if cand not in open_set and cand not in explore_set:
open_set.append(cand)
explore_set.append(curr)
return h_task
def build_txs_graph(datadir: str) -> DiGraph:
"""
read all block and transaction files in the given datadir and construct
a full transaction graph.
Each node represents a transaction. the node's id is the txid and it has
the following attributes:
- "tx" - the full tx json, as returned by bitcoind
- "fee" - the tx fee
- "height" - the block height in which the tx was included
Each edge has the following attributes:
- "value" the value in BTC of the output represented by this edge
"""
blocks = load_blocks(datadir)
txs = load_txs(datadir)
txid_to_fee = {txid: find_tx_fee(txid, txs) for txid in txs.keys()}
txid_to_height = {
txid: block["height"]
for block in blocks.values() for txid in block["tx"]
}
graph = DiGraph()
# add all transactions
for txid in txs.keys():
graph.add_node(txid,
tx=txs[txid],
fee=txid_to_fee[txid],
height=txid_to_height[txid])
# add edges between transactions
for dest_txid, dest_tx in txs.items():
for entry in dest_tx["vin"]:
if "coinbase" in entry:
continue # coinbase transaction. no src
src_txid = entry["txid"]
index = entry["vout"]
value = txs[src_txid]["vout"][index]["value"]
graph.add_edge(src_txid, dest_txid, value=value)
return graph
def find_SCCs(mdp, Sneg):
#----simply find strongly connected components----
print 'Remaining states size', len(Sneg)
SCC = set()
simple_digraph = DiGraph()
A = dict()
for s in mdp.nodes():
A[s] = mdp.node[s]['act'].copy()
for s_f in Sneg:
if s_f not in simple_digraph:
simple_digraph.add_node(s_f)
for s_t in mdp.successors_iter(s_f):
if s_t in Sneg:
simple_digraph.add_edge(s_f,s_t)
print "SubGraph of one Sf: %s states and %s edges" %(str(len(simple_digraph.nodes())), str(len(simple_digraph.edges())))
sccs = strongly_connected_component_subgraphs(simple_digraph)
for scc in sccs:
SCC.add(frozenset(scc.nodes()))
return SCC, A
def build_graph(num_grid, t):
env = DiGraph(name='Environment')
open_set = set()
explore_set = set()
r = round(1 / num_grid, 10)
open_set.add((round(r / 2, 10), round(r / 2, 10)))
while open_set:
point = open_set.pop()
env.add_node(point)
fea_point = direction(point, r, t)
for p in fea_point:
env.add_edge(point, p)
env.add_edge(p, point)
if point not in explore_set:
open_set.add(p)
explore_set.add(point)
return env
def __init__(self,data=None,**kwds):
DiGraph.__init__(self,**kwds)
if data is not None:
try: # build a rooted tree
D=DiGraph()
for (child,parent) in data.par.iteritems():
D.add_edge(parent,child)
except AttributeError:
D=DiGraph(data)
except: # else nothing we can do
raise NetworkXError, "Data %s is not a rooted tree:"%data
if D.order()==D.size()+1:
self.pred=D.pred.copy()
self.succ=D.succ.copy()
self.adj=self.succ
del D
else: # not a tree
raise NetworkXError, "Data %s is not a rooted tree:"%data
def __init__(self, data=None, **kwds):
DiGraph.__init__(self, **kwds)
if data is not None:
try: # build a rooted tree
D = DiGraph()
for (child, parent) in data.par.iteritems():
D.add_edge(parent, child)
except AttributeError:
D = DiGraph(data)
except: # else nothing we can do
raise NetworkXError, "Data %s is not a rooted tree:" % data
if D.order() == D.size() + 1:
self.pred = D.pred.copy()
self.succ = D.succ.copy()
self.adj = self.succ
del D
else: # not a tree
raise NetworkXError, "Data %s is not a rooted tree:" % data
def DuoBA_from_ltls(hard_spec, soft_spec):
hard_buchi = buchi_from_ltl(hard_spec, 'hard_buchi')
soft_buchi = buchi_from_ltl(soft_spec, 'soft_buchi')
hard_symbols = hard_buchi.graph['symbols']
soft_symbols = soft_buchi.graph['symbols']
symbols = set(hard_symbols).union(set(soft_symbols))
DuoBA = DiGraph(type='safe_buchi', hard=hard_buchi, soft=soft_buchi, symols=symbols)
initial = set()
accept = set()
for (h_node, s_node, l) in cartesian_product(hard_buchi.nodes(), soft_buchi.nodes(), [1, 2]):
DuoNode = (h_node, s_node, l)
DuoBA.add_node(DuoNode,hard=h_node, soft=s_node, level=l)
if (h_node in hard_buchi.graph['initial'] and
s_node in soft_buchi.graph['initial'] and l == 1):
initial.add(DuoNode)
if (h_node in hard_buchi.graph['accept'] and l == 1):
accept.add(DuoNode)
DuoBA.graph['accept'] = accept
DuoBA.graph['initial'] = initial
for f_duonode in DuoBA.nodes():
for t_duonode in DuoBA.nodes():
f_h_node, f_s_node, f_level = check_duo_attr(DuoBA, f_duonode)
t_h_node, t_s_node, t_level = check_duo_attr(DuoBA, t_duonode)
if (t_h_node not in DuoBA.graph['hard'].neighbors(f_h_node) or
t_s_node not in DuoBA.graph['soft'].neighbors(f_s_node)):
continue
# relaxed because no common input alphabets are enabled
hardguard = DuoBA.graph['hard'].edges[f_h_node,t_h_node]['guard']
softguard = DuoBA.graph['soft'].edges[f_s_node,t_s_node]['guard']
if ((f_h_node not in DuoBA.graph['hard'].graph['accept'] and
f_level == 1 and t_level == 1) or
(f_h_node in DuoBA.graph['hard'].graph['accept'] and
f_level == 1 and t_level == 2) or
(f_s_node not in DuoBA.graph['soft'].graph['accept'] and
f_level == 2 and t_level == 2) or
(f_s_node in DuoBA.graph['soft'].graph['accept'] and
f_level == 2 and t_level == 1)):
DuoBA.add_edge(f_duonode, t_duonode, hardguard=hardguard, softguard=softguard)
return DuoBA
def buchi_from_ltl(formula, Type):
promela_string = run_ltl2ba(formula)
symbols = find_symbols(formula)
# print "Output from symbols"
# print symbols
edges = parse_ltl(promela_string)
(states, initials, accepts) = find_states(edges)
# print "Output from find_states"
# print states
# print initials
# print accepts
buchi = DiGraph(type=Type,
initial=initials,
accept=accepts,
symbols=symbols)
for state in states:
buchi.add_node(state)
for (ef, et) in edges.keys():
guard_formula = edges[(ef, et)]
guard_expr = parse_guard(guard_formula)
buchi.add_edge(ef, et, guard=guard_expr, guard_formula=guard_formula)
# write_dot(buchi, "./result.dot")
return buchi
def setUp(self):
graph = DiGraph()
graph.add_cycle([1, 2, 3, 4])
graph.add_cycle([5, 6, 7, 8])
graph.add_node(0)
graph.add_edge(9, 10)
self.graph_1 = graph
graph = DiGraph()
graph.add_cycle([1, 2, 3, 4])
graph.add_cycle([5, 6, 7, 8])
graph.add_node(0)
graph.add_edge(9, 10)
graph.add_edge(3, 9)
graph.add_edge(10, 7)
self.graph_2 = graph
def _build_ast(cls, rpn_expression):
"""build an AST from an Excel formula
:param rpn_expression: a string formula or the result of parse_to_rpn()
:return: AST which can be used to generate code
"""
# use a directed graph to store the syntax tree
tree = DiGraph()
# production stack
stack = []
for node in rpn_expression:
# The graph does not maintain the order of adding nodes/edges, so
# add an attribute 'pos' so we can always sort to the correct order
node.ast = tree
if isinstance(node, OperatorNode):
if node.token.type == node.token.OP_IN:
try:
arg2 = stack.pop()
arg1 = stack.pop()
except IndexError:
raise FormulaParserError(
"'{}' operator missing operand".format(
node.token.value))
tree.add_node(arg1, pos=0)
tree.add_node(arg2, pos=1)
tree.add_edge(arg1, node)
tree.add_edge(arg2, node)
else:
try:
arg1 = stack.pop()
except IndexError:
raise FormulaParserError(
"'{}' operator missing operand".format(
node.token.value))
tree.add_node(arg1, pos=1)
tree.add_edge(arg1, node)
elif isinstance(node, FunctionNode):
if node.num_args:
args = stack[-node.num_args:]
del stack[-node.num_args:]
for i, a in enumerate(args):
tree.add_node(a, pos=i)
tree.add_edge(a, node)
else:
tree.add_node(node, pos=0)
stack.append(node)
assert 1 == len(stack)
return stack[0]
def _build_ast(cls, rpn_expression):
"""build an AST from an Excel formula
:param rpn_expression: a string formula or the result of parse_to_rpn()
:return: AST which can be used to generate code
"""
# use a directed graph to store the syntax tree
tree = DiGraph()
# production stack
stack = []
for node in rpn_expression:
# The graph does not maintain the order of adding nodes/edges, so
# add an attribute 'pos' so we can always sort to the correct order
node.ast = tree
if isinstance(node, OperatorNode):
if node.token.type == node.token.OP_IN:
try:
arg2 = stack.pop()
arg1 = stack.pop()
except IndexError:
raise FormulaParserError(
"'{}' operator missing operand".format(
node.token.value))
tree.add_node(arg1, pos=0)
tree.add_node(arg2, pos=1)
tree.add_edge(arg1, node)
tree.add_edge(arg2, node)
else:
try:
arg1 = stack.pop()
except IndexError:
raise FormulaParserError(
"'{}' operator missing operand".format(
node.token.value))
tree.add_node(arg1, pos=1)
tree.add_edge(arg1, node)
elif isinstance(node, FunctionNode):
if node.num_args:
args = stack[-node.num_args:]
del stack[-node.num_args:]
for i, a in enumerate(args):
tree.add_node(a, pos=i)
tree.add_edge(a, node)
else:
tree.add_node(node, pos=0)
stack.append(node)
assert 1 == len(stack)
return stack[0]
def build_ast(expression, debug = False):
"""build an AST from an Excel formula expression in reverse polish notation"""
#use a directed graph to store the tree
G = DiGraph()
stack = []
for n in expression:
# Since the graph does not maintain the order of adding nodes/edges
# add an extra attribute 'pos' so we can always sort to the correct order
if isinstance(n,OperatorNode):
if n.ttype == "operator-infix":
arg2 = stack.pop()
arg1 = stack.pop()
# Hack to write the name of sheet in 2argument address
if(n.tvalue == ':'):
if '!' in arg1.tvalue and arg2.ttype == 'operand' and '!' not in arg2.tvalue:
arg2.tvalue = arg1.tvalue.split('!')[0] + '!' + arg2.tvalue
G.add_node(arg1,{'pos':1})
G.add_node(arg2,{'pos':2})
G.add_edge(arg1, n)
G.add_edge(arg2, n)
else:
arg1 = stack.pop()
G.add_node(arg1,{'pos':1})
G.add_edge(arg1, n)
elif isinstance(n,FunctionNode):
args = []
for _ in range(n.num_args):
try:
args.append(stack.pop())
except:
raise Exception()
#try:
# args = [stack.pop() for _ in range(n.num_args)]
#except:
# print 'STACK', stack, type(n)
# raise Exception('prut')
args.reverse()
for i,a in enumerate(args):
G.add_node(a,{'pos':i})
G.add_edge(a,n)
else:
G.add_node(n,{'pos':0})
stack.append(n)
return G,stack.pop()
def build_ast(expression, debug=False):
"""build an AST from an Excel formula expression in reverse polish notation"""
#use a directed graph to store the tree
G = DiGraph()
stack = []
for n in expression:
# Since the graph does not maintain the order of adding nodes/edges
# add an extra attribute 'pos' so we can always sort to the correct order
if isinstance(n, OperatorNode):
if n.ttype == "operator-infix":
arg2 = stack.pop()
arg1 = stack.pop()
# Hack to write the name of sheet in 2argument address
if (n.tvalue == ':'):
if '!' in arg1.tvalue and arg2.ttype == 'operand' and '!' not in arg2.tvalue:
arg2.tvalue = arg1.tvalue.split(
'!')[0] + '!' + arg2.tvalue
G.add_node(arg1, {'pos': 1})
G.add_node(arg2, {'pos': 2})
G.add_edge(arg1, n)
G.add_edge(arg2, n)
else:
arg1 = stack.pop()
G.add_node(arg1, {'pos': 1})
G.add_edge(arg1, n)
elif isinstance(n, FunctionNode):
args = []
for _ in range(n.num_args):
try:
args.append(stack.pop())
except:
raise Exception()
#try:
# args = [stack.pop() for _ in range(n.num_args)]
#except:
# print 'STACK', stack, type(n)
# raise Exception('prut')
args.reverse()
for i, a in enumerate(args):
G.add_node(a, {'pos': i})
G.add_edge(a, n)
else:
G.add_node(n, {'pos': 0})
stack.append(n)
return G, stack.pop()
def build_ast(expression):
"""build an AST from an Excel formula expression in reverse polish notation"""
#use a directed graph to store the tree
G = DiGraph()
stack = []
for n in expression:
# Since the graph does not maintain the order of adding nodes/edges
# add an extra attribute 'pos' so we can always sort to the correct order
if isinstance(n,OperatorNode):
if n.ttype == "operator-infix":
arg2 = stack.pop()
arg1 = stack.pop()
G.add_node(arg1,{'pos':1})
G.add_node(arg2,{'pos':2})
G.add_edge(arg1, n)
G.add_edge(arg2, n)
else:
arg1 = stack.pop()
G.add_node(arg1,{'pos':1})
G.add_edge(arg1, n)
elif isinstance(n,FunctionNode):
args = [stack.pop() for _ in range(n.num_args)]
args.reverse()
for i,a in enumerate(args):
G.add_node(a,{'pos':i})
G.add_edge(a,n)
#for i in range(n.num_args):
# G.add_edge(stack.pop(),n)
else:
G.add_node(n,{'pos':0})
stack.append(n)
return G,stack.pop()
def build_ast(expression):
"""build an AST from an Excel formula expression in reverse polish notation"""
#use a directed graph to store the tree
G = DiGraph()
stack = []
for n in expression:
# Since the graph does not maintain the order of adding nodes/edges
# add an extra attribute 'pos' so we can always sort to the correct order
if isinstance(n, OperatorNode):
if n.ttype == "operator-infix":
arg2 = stack.pop()
arg1 = stack.pop()
G.add_node(arg1, {'pos': 1})
G.add_node(arg2, {'pos': 2})
G.add_edge(arg1, n)
G.add_edge(arg2, n)
else:
arg1 = stack.pop()
G.add_node(arg1, {'pos': 1})
G.add_edge(arg1, n)
elif isinstance(n, FunctionNode):
args = [stack.pop() for _ in range(n.num_args)]
args.reverse()
for i, a in enumerate(args):
G.add_node(a, {'pos': i})
G.add_edge(a, n)
#for i in range(n.num_args):
# G.add_edge(stack.pop(),n)
else:
G.add_node(n, {'pos': 0})
stack.append(n)
return G, stack.pop()
def dp_dag_general(G, r, U,
cost_func,
node_reward_key='r',
debug=False):
"""
cost_func(node, D table, graph, [(cost at child , child)])
It should return cost as integer type(fixed point is used when appropriate)
"""
ns = G.nodes()
if debug:
print("total #nodes {}".format(len(ns)))
A, D, BP = {}, {}, {}
for n in ns:
A[n] = {} # maximum sum of node u at a cost i
A[n][0] = G.node[n][node_reward_key]
D[n] = {} # set of nodes included corresponding to A[u][i]
D[n][0] = {n}
BP[n] = defaultdict(list) # backpointer corresponding to A[u][i]
for n_i, n in enumerate(
topological_sort(G, reverse=True)): # leaves come first
if debug:
print("#nodes processed {}".format(n_i))
children = G.neighbors(n)
if debug:
print('{}\'s children={}'.format(n, children))
reward = G.node[n][node_reward_key]
if len(children) == 1:
child = children[0]
if debug:
print('child={}'.format(child))
for i in A[child]:
c = cost_func(n, D, G,
[(i, child)])
assert isinstance(c, int)
if c <= U:
A[n][c] = A[child][i] + reward
D[n][c] = D[child][i] | {n}
BP[n][c] = [(child, i)]
elif len(children) > 1:
assert len(children) == 2
lchild, rchild = children
for i in A[lchild]:
c = cost_func(n, D, G,
[(i, lchild)])
assert isinstance(c, int)
if debug:
print('n={}, D={}, cost_child_tuples={}'.format(
n, D, [(i, lchild)])
)
print('c={}'.format(c))
if c <= U:
if A[n].get(c) is None or A[lchild][i] + reward > A[n][c]:
A[n][c] = A[lchild][i] + reward
D[n][c] = D[lchild][i] | {n}
BP[n][c] = [(lchild, i)]
for i in A[rchild]:
c = cost_func(n, D, G,
[(i, rchild)])
assert isinstance(c, int)
if c <= U:
if A[n].get(c) is None or A[rchild][i] + reward > A[n][c]:
A[n][c] = A[rchild][i] + reward
D[n][c] = D[rchild][i] | {n}
BP[n][c] = [(rchild, i)]
for i in A[lchild]:
for j in A[rchild]:
c = cost_func(n, D, G,
[(i, lchild), (j, rchild)])
assert isinstance(c, int)
lset, rset = D[lchild][i], D[rchild][j]
if c <= U:
if (A[n].get(c) is None or
A[lchild][i] + A[rchild][j] + reward > A[n][c]) and \
len(lset & rset) == 0:
A[n][c] = A[lchild][i] + A[rchild][j] + reward
D[n][c] = D[lchild][i] | D[rchild][j] | {n}
BP[n][c] = [(lchild, i), (rchild, j)]
if n == r: # no need to continue once we processed root
break
best_cost = max(xrange(U + 1),
key=lambda i: A[r][i] if i in A[r] else float('-inf'))
tree = DiGraph()
tree.add_node(r)
stack = []
if debug and len(stack) == 0:
print('stack empty')
print(A)
for n, cost in BP[r][best_cost]:
stack.append((r, n, cost))
while len(stack) > 0:
if debug:
print('stack size: {}'.format(len(stack)))
print('stack: {}'.format(stack))
parent, child, cost = stack.pop(0)
tree.add_edge(parent, child)
# copy the attributes
tree[parent][child] = G[parent][child]
tree.node[parent] = G.node[parent]
tree.node[child] = G.node[child]
for grandchild, cost2 in BP[child][cost]:
if debug:
print(grandchild, cost2)
stack.append((child, grandchild, cost2))
return tree
def lst_dag(G, r, U,
node_reward_key='r',
edge_cost_key='c',
edge_weight_decimal_point=None,
fixed_point_func=round,
debug=False):
"""
Param:
-------------
binary_dag: a DAG in networkx format. Each node can have at most 2 child
r: root node in dag
U: the maximum threshold of edge weight sum
Return:
maximum-sum subtree rooted at r whose sum of edge weights <= A
------------
"""
# round edge weight to fixed decimal point if necessary
if edge_weight_decimal_point is not None:
G = G.copy()
G, U = round_edge_weights_by_multiplying(
G,
U,
edge_weight_decimal_point,
edge_cost_key=edge_cost_key,
fixed_point_func=fixed_point_func
)
if debug:
print('U => {}'.format(U))
ns = G.nodes()
if debug:
print("total #nodes {}".format(len(ns)))
A, D, BP = {}, {}, {}
for n in ns:
A[n] = {} # maximum sum of node u at a cost i
A[n][0] = G.node[n][node_reward_key]
D[n] = {} # set of nodes included corresponding to A[u][i]
D[n][0] = {n}
BP[n] = defaultdict(list) # backpointer corresponding to A[u][i]
for n_i, n in enumerate(
topological_sort(G, reverse=True)): # leaves come first
if debug:
print("#nodes processed {}".format(n_i))
children = G.neighbors(n)
reward = G.node[n][node_reward_key]
if len(children) == 1:
child = children[0]
w = G[n][child][edge_cost_key]
for i in xrange(U, w - 1, -1):
if (i-w) in A[child]:
A[n][i] = A[child][i-w] + reward
D[n][i] = D[child][i-w] | {n}
BP[n][i] = [(child, i-w)]
elif len(children) > 1:
lchild, rchild = children
lw = G[n][lchild][edge_cost_key]
rw = G[n][rchild][edge_cost_key]
for i in A[lchild]:
c = lw + i
if debug:
print('n={}, D={}, cost_child_tuples={}'.format(
n, D, [(i, lchild)])
)
print('c={}'.format(c))
if c <= U:
if A[n].get(c) is None or A[lchild][i] + reward > A[n][c]:
A[n][c] = A[lchild][i] + reward
D[n][c] = D[lchild][i] | {n}
BP[n][c] = [(lchild, i)]
for i in A[rchild]:
c = rw + i
if c <= U:
if A[n].get(c) is None or A[rchild][i] + reward > A[n][c]:
A[n][c] = A[rchild][i] + reward
D[n][c] = D[rchild][i] | {n}
BP[n][c] = [(rchild, i)]
for i in A[lchild]:
for j in A[rchild]:
c = lw + rw + i + j
if c <= U:
if (A[n].get(c) is None or
A[lchild][i] + A[rchild][j] + reward > A[n][c]) and \
len(D[lchild][i] & D[rchild][j]) == 0:
A[n][c] = A[lchild][i] + A[rchild][j] + reward
D[n][c] = D[lchild][i] | D[rchild][j] | {n}
BP[n][c] = [(lchild, i), (rchild, j)]
# if n == r: # no need to continue once we processed root
# break
if debug:
print('A[r]', A[r])
best_cost = max(xrange(U + 1),
key=lambda i: A[r][i] if i in A[r] else float('-inf'))
if debug:
print("best_cost", best_cost)
tree = DiGraph()
tree.add_node(r)
stack = []
for n, cost in BP[r][best_cost]:
stack.append((r, n, cost))
while len(stack) > 0:
# if debug:
# print('stack size: {}'.format(len(stack)))
# print('stack: {}'.format(stack))
parent, child, cost = stack.pop(0)
tree.add_edge(parent, child)
# copy the attributes
tree[parent][child] = G[parent][child]
tree.node[parent] = G.node[parent]
tree.node[child] = G.node[child]
for grandchild, cost2 in BP[child][cost]:
# if debug:
# print(grandchild, cost2)
stack.append((child, grandchild, cost2))
return tree
net, _i, _f = run_heuristic(log, label,
'dest_class_func',
parameters=heuristic_b_params)
print(f'Creating graph for petri net...')
graph = DiGraph()
all_nodes = set()
for place in net.places:
graph.add_node(str(place), label=str(place))
all_nodes.add(str(place))
for transition in net.transitions:
graph.add_node(str(transition), label=str(transition))
all_nodes.add(str(transition))
for arc in net.arcs:
graph.add_edge(str(arc.source), str(arc.target))
sp_nodes = set([label for label in nodes_in_shortest_path(graph)
if heuristic_filter(label)])
filtered_nodes = set([label for label in all_nodes
if heuristic_filter(label)])
# print("Filtered Nodes")
# print(filtered_nodes)
# print()
# print("Shortest Path Nodes")
# print(sp_nodes)
# print()
print()
class tree(object):
""" construction of prefix and suffix tree
"""
def __init__(self, ts, buchi_graph, init, step_size, base=1e3):
"""
:param ts: transition system
:param buchi_graph: Buchi graph
:param init: product initial state
"""
self.robot = 1
self.goals = []
self.ts = ts
self.buchi_graph = buchi_graph
self.init = init
self.step_size = step_size
self.dim = len(self.ts['workspace'])
uni_v = np.power(np.pi, self.robot * self.dim /
2) / math.gamma(self.robot * self.dim / 2 + 1)
self.gamma = np.ceil(
4 * np.power(1 / uni_v, 1. /
(self.dim * self.robot))) # unit workspace
self.tree = DiGraph(type='PBA', init=init)
label = self.label(init[0])
if label != '':
label = label + '_' + str(1)
# accepting state before current node
acc = set()
if 'accept' in init[1]:
acc.add(init)
self.tree.add_node(init, cost=0, label=label, acc=acc)
self.search_goal(init, label, acc)
# already used skilles
self.used = set()
self.base = base
def sample(self):
"""
sample point from the workspace
:return: sampled point, tuple
"""
x_rand = []
for i in range(self.dim):
x_rand.append(uniform(0, self.ts['workspace'][i]))
return tuple(x_rand)
def nearest(self, x_rand):
"""
find the nearest class of vertices in the tree
:param: x_rand randomly sampled point form: single point ()
:return: nearest class of vertices form: single point ()
"""
min_dis = math.inf
q_nearest = []
for vertex in self.tree.nodes:
x_vertex = vertex[0]
dis = np.linalg.norm(np.subtract(x_rand, x_vertex))
if dis < min_dis:
q_nearest = list()
q_nearest.append(vertex)
min_dis = dis
elif dis == min_dis:
q_nearest.append(vertex)
return q_nearest
def steer(self, x_rand, x_nearest):
"""
steer
:param: x_rand randomly sampled point form: single point ()
:param: x_nearest nearest point in the tree form: single point ()
:return: new point single point ()
"""
if np.linalg.norm(np.subtract(x_rand, x_nearest)) <= self.step_size:
return x_rand
else:
return tuple(
np.asarray(x_nearest) + self.step_size *
(np.subtract(x_rand, x_nearest)) /
np.linalg.norm(np.subtract(x_rand, x_nearest)))
def acpt_check(self, q_min, q_new):
"""
check the accepting state in the patg leading to q_new
:param q_min:
:param q_new:
:return:
"""
changed = False
acc = set(self.tree.nodes[q_min]['acc']) # copy
if 'accept' in q_new[1]:
acc.add(q_new)
# print(acc)
changed = True
return acc, changed
def search_goal(self, q_new, label_new, acc):
"""
whether q_new can connect to point before acc
:param q_new:
:param label_new:
:param acc:
:return:
"""
for ac in acc:
# connect to path leading to accepting state, including accepting state
path = self.findpath([ac])[0][1]
for point in path:
if list(self.obs_check([q_new], point[0], self.tree.nodes[point]['label']).values())[0] \
and self.checkTranB(q_new[1], label_new, point[1]):
self.goals.append(
(q_new, point,
ac)) # endpoint, middle point, accepting point
def extend(self, q_new, near_v, label, obs_check, succ_list):
"""
:param: q_new: new state form: tuple (mulp, buchi)
:param: near_v: near state form: tuple (mulp, buchi)
:param: obs_check: check obstacle free form: dict { (mulp, mulp): True }
:param: succ: list of successor of the root
:return: extending the tree
"""
added = 0
cost = np.inf
q_min = ()
for near_vertex in near_v:
if near_vertex in succ_list: # do not extend if there is a corresponding root
continue
if q_new != near_vertex and obs_check[(q_new[0], near_vertex[0])] \
and self.checkTranB(near_vertex[1], self.tree.nodes[near_vertex]['label'], q_new[1]):
c = self.tree.nodes[near_vertex]['cost'] + \
np.linalg.norm(np.subtract(q_new[0], near_vertex[0]))
if c < cost:
added = 1
q_min = near_vertex
cost = c
if added == 1:
self.tree.add_node(q_new, cost=cost, label=label)
self.tree.nodes[q_new]['acc'] = set(
self.acpt_check(q_min, q_new)[0])
self.tree.add_edge(q_min, q_new)
# self.search_goal(q_new, label, self.tree.nodes[q_new]['acc'])
return added
def rewire(self, q_new, near_v, obs_check):
"""
:param: q_new: new state form: tuple (mul, buchi)
:param: near_v: near state form: tuple (mul, buchi)
:param: obs_check: check obstacle free form: dict { (mulp, mulp): True }
:return: rewiring the tree
"""
for near_vertex in near_v:
if obs_check[(q_new[0], near_vertex[0])] \
and self.checkTranB(q_new[1], self.tree.nodes[q_new]['label'], near_vertex[1]):
c = self.tree.nodes[q_new]['cost'] \
+ np.linalg.norm(np.subtract(q_new[0], near_vertex[0]))
delta_c = self.tree.nodes[near_vertex]['cost'] - c
# update the cost of node in the subtree rooted at near_vertex
if delta_c > 0:
# self.tree.nodes[near_vertex]['cost'] = c
self.tree.remove_edge(
list(self.tree.pred[near_vertex].keys())[0],
near_vertex)
self.tree.add_edge(q_new, near_vertex)
edges = dfs_labeled_edges(self.tree, source=near_vertex)
acc, changed = self.acpt_check(q_new, near_vertex)
self.tree.nodes[near_vertex]['acc'] = set(acc)
for u, v, d in edges:
if d == 'forward':
self.tree.nodes[v][
'cost'] = self.tree.nodes[v]['cost'] - delta_c
if changed:
self.tree.nodes[v]['acc'] = set(
self.acpt_check(u, v)[0]) # copy
# better to research the goal but abandon the implementation
def near(self, x_new):
"""
find the states in the near ball
:param x_new: new point form: single point
:return: p_near: near state, form: tuple (mulp, buchi)
"""
p_near = []
r = min(
self.gamma * np.power(
np.log(self.tree.number_of_nodes() + 1) /
self.tree.number_of_nodes(), 1. / (self.dim * self.robot)),
self.step_size)
# r = self.step_size
for vertex in self.tree.nodes:
if np.linalg.norm(np.subtract(x_new, vertex[0])) <= r:
p_near.append(vertex)
return p_near
def obs_check(self, q_near, x_new, label):
"""
check whether obstacle free along the line from x_near to x_new
:param q_near: states in the near ball, tuple (mulp, buchi)
:param x_new: new state form: multiple point
:param label: label of x_new
:param stage: regular stage or final stage, deciding whether it's goal state
:return: dict (x_near, x_new): true (obs_free)
"""
obs_check_dict = {}
checked = set()
for x in q_near:
if x[0] in checked:
continue
checked.add(x[0])
obs_check_dict[(x_new, x[0])] = True
# the line connecting two points crosses an obstacle
for (obs, boundary) in iter(self.ts['obs'].items()):
if LineString([Point(x[0]),
Point(x_new)]).intersects(boundary):
obs_check_dict[(x_new, x[0])] = False
break
for (region, boundary) in iter(self.ts['region'].items()):
if LineString([Point(x[0]), Point(x_new)]).intersects(boundary) \
and region + '_' + str(1) != label \
and region + '_' + str(1) != self.tree.nodes[x]['label']:
# if stage == 'reg' or (stage == 'final' and region in self.no):
obs_check_dict[(x_new, x[0])] = False
break
return obs_check_dict
def label(self, x):
"""
generating the label of position state
:param x: position
:return: label
"""
point = Point(x)
# whether x lies within obstacle
for (obs, boundary) in iter(self.ts['obs'].items()):
if point.within(boundary):
return obs
# whether x lies within regions
for (region, boundary) in iter(self.ts['region'].items()):
if point.within(boundary):
return region
# x lies within unlabeled region
return ''
def checkTranB(self, b_state, x_label, q_b_new):
""" decide valid transition, whether b_state --L(x)---> q_b_new
Algorithm2 in Chapter 2 Motion and Task Planning
:param b_state: buchi state
:param x_label: label of x
:param q_b_new buchi state
:return True satisfied
"""
b_state_succ = self.buchi_graph.succ[b_state]
# q_b_new is not the successor of b_state
if q_b_new not in b_state_succ:
return False
truth = self.buchi_graph.edges[(b_state, q_b_new)]['truth']
if self.t_satisfy_b_truth(x_label, truth):
return True
return False
def t_satisfy_b_truth(self, x_label, truth):
"""
check whether transition enabled under current label
:param x_label: current label
:param truth: truth value making transition enabled
:return: true or false
"""
if truth == '1':
return True
true_label = [
truelabel for truelabel in truth.keys() if truth[truelabel]
]
for label in true_label:
if label not in x_label:
return False
false_label = [
falselabel for falselabel in truth.keys() if not truth[falselabel]
]
for label in false_label:
if label in x_label:
return False
return True
def findpath(self, goals):
"""
find the path backwards
:param goals: goal state
:return: dict path : cost
"""
paths = OrderedDict()
for i in range(len(goals)):
goal = goals[i]
path = [goal]
s = goal
while s != self.init:
s = list(self.tree.pred[s].keys())[0]
if s == path[0]:
print("loop")
path.insert(0, s)
paths[i] = [self.tree.nodes[goal]['cost'], path]
return paths
class BoardGraph(object):
def __init__(self, board_class=Board):
self.graph = self.start = self.last = self.closest = None
self.min_domino_count = None
self.board_class = board_class
def walk(self, board, size_limit=maxsize):
pending_nodes = []
self.graph = DiGraph()
self.start = board.display(cropped=True)
self.graph.add_node(self.start)
pending_nodes.append(self.start)
self.last = self.start
while pending_nodes:
if len(self.graph) >= size_limit:
raise GraphLimitExceeded(size_limit)
state = pending_nodes.pop()
board = self.board_class.create(state, border=1)
for move, new_state in self.generate_moves(board):
if not self.graph.has_node(new_state):
# new node
self.graph.add_node(new_state)
pending_nodes.append(new_state)
self.graph.add_edge(state, new_state, move=move)
return set(self.graph.nodes())
def generate_moves(self, board):
""" Generate all moves from the board's current state.
:param Board board: the current state
:return: a generator of (state, move_description) tuples
"""
dominoes = set(board.dominoes)
domino_count = len(dominoes)
if self.min_domino_count is None or domino_count < self.min_domino_count:
self.min_domino_count = domino_count
self.last = board.display(cropped=True)
for domino in dominoes:
dx, dy = domino.direction
yield from self.try_move(domino, dx, dy)
yield from self.try_move(domino, -dx, -dy)
def try_move(self, domino, dx, dy):
try:
new_state = self.move(domino, dx, dy)
move = domino.describe_move(dx, dy)
yield move, new_state
except BadPositionError:
pass
def move(self, domino, dx, dy):
""" Move a domino and calculate the new board state.
Afterward, put the board back in its original state.
@return: the new board state
@raise BadPositionError: if the move is illegal
"""
domino.move(dx, dy)
try:
board = domino.head.board
if not board.isConnected():
raise BadPositionError('Board is not connected.')
if board.hasLoner():
raise BadPositionError('Board has a lonely domino.')
return board.display(cropped=True)
finally:
domino.move(-dx, -dy)
def get_solution(self, return_partial=False):
""" Find a solution from the graph of moves.
@param return_partial: If True, a partial solution will be returned if no
solution exists.
@return: a list of strings describing each move. Each string is two
digits describing the domino that moved plus a letter to show the
direction.
"""
solution = []
goal = self.closest if return_partial else self.last or ''
solution_nodes = shortest_path(self.graph, self.start, goal)
for i in range(len(solution_nodes) - 1):
source, target = solution_nodes[i:i + 2]
solution.append(self.graph[source][target]['move'])
return solution
def get_choice_counts(self):
solution_nodes = shortest_path(self.graph, self.start, self.last)
return [len(self.graph[node]) for node in solution_nodes[:-1]]
def get_average_choices(self):
choices = self.get_choice_counts()
return sum(choices) / float(len(choices)) if choices else maxsize
def get_max_choices(self):
choices = self.get_choice_counts()
return max(choices) if choices else maxsize
class StadynaMcgAnalysis:
def __init__(self):
self.androGuardObjects = []
self.nodes = {}
self.nodes_id = {}
self.entry_nodes = []
self.G = DiGraph()
# self.internal_methods = []
#self.GI = DiGraph()
def analyseFile(self, vmx, apk):
vm = vmx.get_vm()
self.androGuardObjects.append((apk, vm, vmx))
# self.internal_methods.extend(vm.get_methods())
#creating real internal nodes
internal_called_methods = vmx.get_tainted_packages().stadyna_get_internal_called_methods()
for method in internal_called_methods:
class_name, method_name, descriptor = method
nodeType = None
if method_name == "<clinit>":
nodeType = NODE_STATIC_INIT
elif method_name == "<init>":
nodeType = NODE_CONSTRUCTOR
else:
nodeType = NODE_METHOD
n = self._get_node(nodeType, (class_name, method_name, descriptor))
n.set_attribute(ATTR_CLASS_NAME, class_name)
n.set_attribute(ATTR_METHOD_NAME, method_name)
n.set_attribute(ATTR_DESCRIPTOR, descriptor)
self.G.add_node(n.id)
#creating real edges (nodes are already there)
#currently we are working only with internal packages.
for j in vmx.get_tainted_packages().get_internal_packages():
src_class_name, src_method_name, src_descriptor = j.get_src(vm.get_class_manager())
dst_class_name, dst_method_name, dst_descriptor = j.get_dst(vm.get_class_manager())
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
# n1.set_attribute(ATTR_CLASS_NAME, src_class_name)
# n1.set_attribute(ATTR_METHOD_NAME, src_method_name)
# n1.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
n2 = self._get_existed_node((dst_class_name, dst_method_name, dst_descriptor))
# n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
# n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
# n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#adding fake class nodes
for method in internal_called_methods:
src_class_name, src_method_name, src_descriptor = method
if src_method_name == "<init>" or src_method_name == "<clinit>":
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
n2 = self._get_node(NODE_FAKE_CLASS, src_class_name, None, False)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
if src_method_name == "<clinit>":
self.G.add_edge(n1.id, n2.id)
elif src_method_name == "<init>":
self.G.add_edge(n2.id, n1.id)
#real (external) reflection invoke nodes
reflection_invoke_paths = analysis.seccon_get_invoke_method_paths(vmx)
for j in reflection_invoke_paths:
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find the node [%s], where reflection invoke is called!" % (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_INVOKE)
n2 = self._get_node(NODE_REFL_INVOKE, key, LABEL_REFL_INVOKE, True)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge( n1.id, n2.id )
#real (external) reflection new instance nodes
reflection_newInstance_paths = analysis.seccon_get_newInstance_method_paths(vmx)
for j in reflection_newInstance_paths:
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find the node [%s], where reflection new instance is called!" % (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_NEWINSTANCE)
n2 = self._get_node(NODE_REFL_NEWINSTANCE, key, LABEL_REFL_NEWINSTANCE, True)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge( n1.id, n2.id )
#adding fake entry points
if apk != None:
for i in apk.get_activities() :
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node((j, "onCreate", "(Landroid/os/Bundle;)V"))
if n1 != None:
key = "%s %s %s %s" % (j, "onCreate", "(Landroid/os/Bundle;)V", POSTFIX_ACTIVITY)
n2 = self._get_node(NODE_FAKE_ACTIVITY, key, LABEL_ACTIVITY, False)
self.G.add_edge( n2.id, n1.id )
self.entry_nodes.append( n1.id )
for i in apk.get_services() :
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node( (j, "onCreate", "()V") )
if n1 != None :
key = "%s %s %s %s" % (j, "onCreate", "()V", POSTFIX_SERVICE)
n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_SERVICE, False)
self.G.add_edge( n2.id, n1.id )
self.entry_nodes.append( n1.id )
for i in apk.get_receivers() :
j = bytecode.FormatClassToJava(i)
n1 = self._get_existed_node( (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V") )
if n1 != None :
key = "%s %s %s %s" % (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V", POSTFIX_RECEIVER)
n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_RECEIVER, False)
self.G.add_edge( n2.id, n1.id )
self.entry_nodes.append( n1.id )
#fake permissions
list_permissions = vmx.stadyna_get_permissions([])
for x in list_permissions:
for j in list_permissions[x]:
if isinstance(j, PathVar):
continue
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find node [%s %s %s] for permission [%s]!" % (src_class_name, src_method_name, src_descriptor, x))
continue
#SOURCE, DEST, POSTFIX, PERMISSION_NAME
key = "%s %s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, x)
n2 = self._get_node(NODE_FAKE_PERMISSION, key, x, False)
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n2.set_attribute(ATTR_PERM_NAME, x)
n2.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ x ][0])
self.G.add_edge(n1.id, n2.id)
#fake DexClassLoader nodes
dyn_code_loading = analysis.seccon_get_dyncode_loading_paths(vmx)
for j in dyn_code_loading:
src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() )
dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() )
n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor))
if n1 == None:
logger.warning("Cannot find dexload node [%s]!" % (src_class_name, src_method_name, src_descriptor))
continue
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_DEXLOAD)
n2 = self._get_node(NODE_FAKE_DEXLOAD, key, LABEL_DEXLOAD, False)
n2.set_attribute(ATTR_CLASS_NAME, src_class_name)
n2.set_attribute(ATTR_METHOD_NAME, src_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor)
self.G.add_edge( n1.id, n2.id )
# Specific Java/Android library
for c in vm.get_classes():
#if c.get_superclassname() == "Landroid/app/Service;" :
# n1 = self._get_node( c.get_name(), "<init>", "()V" )
# n2 = self._get_node( c.get_name(), "onCreate", "()V" )
# self.G.add_edge( n1.id, n2.id )
if c.get_superclassname() == "Ljava/lang/Thread;" or c.get_superclassname() == "Ljava/util/TimerTask;" :
for i in vm.get_method("run") :
if i.get_class_name() == c.get_name() :
n1 = self._get_node(NODE_METHOD, (i.get_class_name(), i.get_name(), i.get_descriptor()))
n2 = self._get_node(NODE_METHOD, (i.get_class_name(), "start", i.get_descriptor()))
# link from start to run
self.G.add_edge( n2.id, n1.id )
#n2.add_edge( n1, {} )
# link from init to start
for init in vm.get_method("<init>") :
if init.get_class_name() == c.get_name():
#TODO: Leaving _get_existed_node to check if all the nodes are included
#It is possible that internal_packages does not contain this node. Leaving _get_existed_node to check this
n3 = self._get_node(NODE_CONSTRUCTOR, (init.get_class_name(), "<init>", init.get_descriptor()))
self.G.add_edge( n3.id, n2.id )
#n3.add_edge( n2, {} )
def addInvokePath(self, src, through, dst):
src_class_name, src_method_name, src_descriptor = src
dst_class_name, dst_method_name, dst_descriptor = dst
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_INVOKE)
n1 = self._get_existed_node(key)
if n1 == None:
logger.warning("Something wrong has happened! Could not find invoke Node in Graph with key [%s]" % str(key))
return
n2 = self._get_node(NODE_METHOD, (dst_class_name, dst_method_name, dst_descriptor))
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#check if called method calls protected feature
data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor)
if data in DVM_PERMISSIONS_BY_API_CALLS:
logger.info("BINGOOOOOOO! The protected method is called through reflection!")
perm = DVM_PERMISSIONS_BY_API_CALLS[ data ]
key1 = "%s %s %s %s %s %s %s %s" % (through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm)
n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False)
n3.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n3.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n3.set_attribute(ATTR_PERM_NAME, perm)
n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ perm ][0])
self.G.add_edge(n2.id, n3.id)
def addNewInstancePath(self, src, through, dst):
src_class_name, src_method_name, src_descriptor = src
dst_class_name, dst_method_name, dst_descriptor = dst
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_NEWINSTANCE)
n1 = self._get_existed_node(key)
if n1 == None:
logger.error("Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key))
return
n2 = self._get_node(NODE_CONSTRUCTOR, (dst_class_name, dst_method_name, dst_descriptor))
n2.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n2.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
self.G.add_edge(n1.id, n2.id)
#we also need to add link to the class node
#TODO: Think in the future what to do with this
n_class = self._get_node(NODE_FAKE_CLASS, dst_class_name, None, False)
n_class.set_attribute(ATTR_CLASS_NAME, dst_class_name)
self.G.add_edge(n_class.id, n2.id)
#checking if we need to add additional permission nodes
data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor)
if data in DVM_PERMISSIONS_BY_API_CALLS:
logger.info("BINGOOOOOOO! The protected method is called through reflection!")
perm = DVM_PERMISSIONS_BY_API_CALLS[ data ]
key1 = "%s %s %s %s %s %s %s %s" % (through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm)
n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False)
n3.set_attribute(ATTR_CLASS_NAME, dst_class_name)
n3.set_attribute(ATTR_METHOD_NAME, dst_method_name)
n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor)
n3.set_attribute(ATTR_PERM_NAME, perm)
n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ perm ][0])
self.G.add_edge(n2.id, n3.id)
def addDexloadPath(self, src, through, filename):
src_class_name, src_method_name, src_descriptor = src
through_class_name, through_method_name, through_descriptor = through
key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_DEXLOAD)
n1 = self._get_existed_node(key)
if n1 == None:
logger.error("Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key))
return
n2 = self._get_node(NODE_FAKE_DEXLOAD_FILE, filename, filename, False)
n2.set_attribute(ATTR_DEXLOAD_FILENAME, filename)
self.G.add_edge(n1.id, n2.id)
def _get_node(self, nType, key, label=None, real=True):
node_key = None
if isinstance(key, basestring):
node_key = key
elif isinstance(key, tuple):
node_key = "%s %s %s" % key
else:
logger.error("Unknown instance type of key!!!")
if node_key not in self.nodes.keys():
new_node = NodeS(len(self.nodes), nType, node_key, label, real)
self.nodes[node_key] = new_node
self.nodes_id[new_node.id] = new_node
return self.nodes[node_key]
def _get_existed_node(self, key):
node_key = None
if isinstance(key, basestring):
node_key = key
elif isinstance(key, tuple):
node_key = "%s %s %s" % key
else:
logger.error("Unknown instance type of key!!!")
try:
return self.nodes[node_key]
except KeyError:
logger.error("Could not find existed node [%s]!" % node_key)
return None
def export_to_gexf(self) :
buff = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
buff += "<gexf xmlns=\"http://www.gephi.org/gexf\" xmlns:viz=\"http://www.gephi.org/gexf/viz\">\n"
buff += "<graph type=\"static\">\n"
buff += "<attributes class=\"node\" type=\"static\">\n"
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_TYPE, ID_ATTRIBUTES[ ATTR_TYPE ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_CLASS_NAME, ID_ATTRIBUTES[ ATTR_CLASS_NAME ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_METHOD_NAME, ID_ATTRIBUTES[ ATTR_METHOD_NAME ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_DESCRIPTOR, ID_ATTRIBUTES[ ATTR_DESCRIPTOR ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_REAL, ID_ATTRIBUTES[ ATTR_REAL ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_PERM_NAME, ID_ATTRIBUTES[ ATTR_PERM_NAME ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_PERM_LEVEL, ID_ATTRIBUTES[ ATTR_PERM_LEVEL ])
buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_DEXLOAD_FILENAME, ID_ATTRIBUTES[ ATTR_DEXLOAD_FILENAME ])
buff += "</attributes>\n"
# buff += "<attributes class=\"node\" type=\"static\">\n"
# buff += "<attribute id=\"%d\" title=\"type\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "type"]
# buff += "<attribute id=\"%d\" title=\"class_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "class_name"]
# buff += "<attribute id=\"%d\" title=\"method_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "method_name"]
# buff += "<attribute id=\"%d\" title=\"descriptor\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "descriptor"]
#
#
# buff += "<attribute id=\"%d\" title=\"permissions\" type=\"integer\" default=\"0\"/>\n" % ID_ATTRIBUTES[ "permissions"]
# buff += "<attribute id=\"%d\" title=\"permissions_level\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "permissions_level"]
#
# buff += "<attribute id=\"%d\" title=\"dynamic_code\" type=\"boolean\" default=\"false\"/>\n" % ID_ATTRIBUTES[ "dynamic_code"]
#
# buff += "</attributes>\n"
buff += "<nodes>\n"
for node in self.G.nodes() :
buff += "<node id=\"%d\" label=\"%s\">\n" % (node, escape(self.nodes_id[ node ].label))
buff += self.nodes_id[ node ].get_attributes_gexf()
buff += "</node>\n"
buff += "</nodes>\n"
buff += "<edges>\n"
nb = 0
for edge in self.G.edges() :
buff += "<edge id=\"%d\" source=\"%d\" target=\"%d\"/>\n" % (nb, edge[0], edge[1])
nb += 1
buff += "</edges>\n"
buff += "</graph>\n"
buff += "</gexf>\n"
return buff
def get_current_real_node_count(self):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_REAL) == "True":
count += 1
return count
def get_current_node_count(self):
return len(self.G.nodes())
def get_current_edge_count(self):
return len(self.G.edges())
def get_current_permission_level_node_count(self, permission_level):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) == permission_level:
count += 1
return count
def get_current_protected_node_count(self):
count = 0
for node in self.nodes_id.keys():
if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) != None:
count += 1
return count
def find_MECs(mdp, Sneg):
#----implementation of Alg.47 P866 of Baier08----
print 'Remaining states size', len(Sneg)
U = mdp.graph['U']
A = dict()
for s in Sneg:
A[s] = mdp.node[s]['act'].copy()
if not A[s]:
print "Isolated state"
MEC = set()
MECnew = set()
MECnew.add(frozenset(Sneg))
#----
k = 0
while MEC != MECnew:
print "<============iteration %s============>" %k
k +=1
MEC = MECnew
MECnew = set()
print "MEC size: %s" %len(MEC)
print "MECnew size: %s" %len(MECnew)
for T in MEC:
R = set()
T_temp = set(T)
simple_digraph = DiGraph()
for s_f in T_temp:
if s_f not in simple_digraph:
simple_digraph.add_node(s_f)
for s_t in mdp.successors_iter(s_f):
if s_t in T_temp:
simple_digraph.add_edge(s_f,s_t)
print "SubGraph of one MEC: %s states and %s edges" %(str(len(simple_digraph.nodes())), str(len(simple_digraph.edges())))
Sccs = strongly_connected_component_subgraphs(simple_digraph)
i = 0
for Scc in Sccs:
i += 1
if (len(Scc.edges())>=1):
for s in Scc.nodes():
U_to_remove = set()
for u in A[s]:
for t in mdp.successors_iter(s):
if ((u in mdp.edge[s][t]['prop'].keys()) and (t not in Scc.nodes())):
U_to_remove.add(u)
A[s].difference_update(U_to_remove)
if not A[s]:
R.add(s)
while R:
s = R.pop()
T_temp.remove(s)
for f in mdp.predecessors(s):
if f in T_temp:
A[f].difference_update(set(mdp.edge[f][s]['prop'].keys()))
if not A[f]:
R.add(f)
New_Sccs = strongly_connected_component_subgraphs(simple_digraph)
j = 0
for Scc in New_Sccs:
j += 1
if (len(Scc.edges()) >= 1):
common = set(Scc.nodes()).intersection(T_temp)
if common:
MECnew.add(frozenset(common))
#---------------
print 'Final MEC and MECnew size:', len(MEC)
return MEC, A
class Program:
def __init__(self, rules):
self.rules = rules
self.predicate_names = None
self.predicate_deps_graph = None
assert (self.is_stratified())
def get_rules(self):
return self.rules
def get_derivation_rules(self):
return [rule for rule in self.rules if not rule.is_type_rule]
def get_type_rules(self):
return [rule for rule in self.rules if rule.is_type_rule]
def get_predicate_names(self):
if self.predicate_names != None:
return self.predicate_names
self.predicate_names = set()
for rule in self.get_derivation_rules():
self.predicate_names.add(rule.head.name)
for literal in rule.get_literals():
self.predicate_names.add(literal.atom.name)
return self.predicate_names
def get_nonrecursive_idb_predicate_names(self):
return self.extend_nonrecursive_idb_predicate_names(set())
def get_recursive_idb_predicate_names(self):
return self.get_idb_predicate_names(
) - self.get_nonrecursive_idb_predicate_names()
def extend_nonrecursive_idb_predicate_names(self, cur_nonrecursive):
extended_nonrecursive = set()
for predicate_name in self.get_idb_predicate_names():
if predicate_name in cur_nonrecursive:
extended_nonrecursive.add(predicate_name)
continue
recursive = False
for rule in self.get_rules_for_predicate(predicate_name):
for literal in rule.get_literals():
literal_predicate_name = literal.atom.name
if literal_predicate_name not in self.get_edb_predicate_names(
) and literal_predicate_name not in cur_nonrecursive:
recursive = True
if not recursive:
extended_nonrecursive.add(predicate_name)
if cur_nonrecursive == extended_nonrecursive:
return extended_nonrecursive
else:
return self.extend_nonrecursive_idb_predicate_names(
extended_nonrecursive)
def get_idb_predicate_names(self):
idbs = set()
for rule in self.get_derivation_rules():
idbs.add(rule.head.name)
return idbs
def get_edb_predicate_names(self):
return self.get_predicate_names() - self.get_idb_predicate_names()
def predicate_depends_on(self, predicate_name1, predicate_name2):
return has_path(self.get_predicate_deps_graph(), predicate_name2,
predicate_name1)
def get_rules_for_predicate(self, predicate_name):
rules = set()
for rule in self.get_derivation_rules():
if rule.head.name == predicate_name:
rules.add(rule)
return rules
def get_predicate_deps_graph(self):
if self.predicate_deps_graph != None:
return self.predicate_deps_graph
self.predicate_deps_graph = DiGraph()
# add a node for each Predicate
self.predicate_deps_graph.add_nodes_from(self.get_predicate_names())
for predicate_name in self.get_predicate_names():
for rule in self.get_rules_for_predicate(predicate_name):
for literal in rule.get_literals():
self.predicate_deps_graph.add_edge(literal.atom.name,
predicate_name)
return self.predicate_deps_graph
def is_stratified(self):
positivePredicateDeps = {}
negativePredicateDeps = {}
for predicate_name in self.get_predicate_names():
positivePredicateDeps[predicate_name] = set()
negativePredicateDeps[predicate_name] = set()
for rule in self.get_rules_for_predicate(predicate_name):
for literal in rule.get_literals():
if literal.negated:
negativePredicateDeps[predicate_name].add(
literal.atom.name)
else:
positivePredicateDeps[predicate_name].add(
literal.atom.name)
for scc in strongly_connected_components(
self.get_predicate_deps_graph()):
for predicate_name in scc:
if negativePredicateDeps[predicate_name].intersection(
set(scc)):
return False
return True
def is_recursive_predicate(self, predicate_name):
for cyclic_predicates in simple_cycles(
self.get_predicate_deps_graph()):
if predicate_name in cyclic_predicates:
return True
return False
