def get_example_3(): """get a binarized example, whose original graph is more complicated than the above example """ g = DiGraph() g.add_nodes_from(range(1, 10)) g.add_edges_from([(1, 2), (1, 3), (1, 7), (2, 4), (2, 5), (2, 6), (2, 7), (3, 8), (3, 9)]) rewards = range(1, 10) for r, n in zip(rewards, g.nodes()): g.node[n]['r'] = r # all edges have cost 2 except 1 -> 2 and 1 -> 3(cost 1) for s, t in g.edges(): g[s][t]['c'] = 2 g[1][2]['c'] = 1 g[1][3]['c'] = 1 g = binarize_dag(g, vertex_weight_key='r', edge_weight_key='c', dummy_node_name_prefix='d_') # parameters and expected output U = [0, 2, 3, 4, 100] expected_edges_set = [ [], [(1, 7)], [(1, 'd_1'), ('d_1', 3), (3, 9)], [(1, 'd_1'), ('d_1', 3), (3, 9), ('d_1', 2)], # (1, 7) removed to make it a tree list(set(g.edges()) - set([(1, 7)])) ] return (g, U, expected_edges_set)
def build_crm(roadmap_dir='./build_roadmap/roadmap_2D.p', wsn_rate_dir='./wsn_routing/rate_map.p', ws_img_dir='./build_roadmap/ws_img.p'): ''' build combined roadmap, crm via load roadmap and merge in wsn_rate info ''' roadmap_edges = pickle.load(open(roadmap_dir, 'rb')) wsn_rate, wifis_loc, sink_loc = pickle.load(open(wsn_rate_dir, 'rb')) img = pickle.load(open(ws_img_dir, 'rb')) crm = DiGraph(name='combined_model', ws_img=img, wifis=wifis_loc, sink=sink_loc) for e in roadmap_edges: (f_node, t_node) = e near_f_node = min(wsn_rate.keys(), key=lambda p: dist_2D(p, f_node)) f_node_rate = wsn_rate[near_f_node] crm.add_node(f_node, rate=f_node_rate) near_t_node = min(wsn_rate.keys(), key=lambda p: dist_2D(p, t_node)) t_node_rate = wsn_rate[near_t_node] crm.add_node(t_node, rate=t_node_rate) # interpolation dist_e = dist_2D(f_node, t_node) f_t_rate = intp_rate(f_node_rate, t_node_rate) crm.add_edge(f_node, t_node, rate=f_t_rate, dist=dist_e) t_f_rate = intp_rate(t_node_rate, f_node_rate) crm.add_edge(t_node, f_node, rate=t_f_rate, dist=dist_e) print '---crm constructed from %s and %s, %d nodes, %d edges---' % ( roadmap_dir, wsn_rate_dir, len(crm.nodes()), len(crm.edges())) return crm
def get_variance_example_1(): g = DiGraph() g.add_edges_from([(0, 1), (0, 2), (2, 3), (3, 4), (2, 'dummy'), ('dummy', 5)]) g.node['dummy']['dummy'] = True for n in (0, 1, 2, 5): # topic 1 g.node[n]['repr'] = np.array([0, 0]) for n in (3, 4): # topic 2 g.node[n]['repr'] = np.array([1, 1]) for n in g.nodes_iter(): g.node[n]['r'] = 1 # correct is (0, 1, 2, 5) for cost 0 U = [0, 42] expected_edge_set = [set(g.edges()) - {(2, 3), (3, 4)}, set(g.edges())] return (g, U, expected_edge_set)
def setDelta(self, G: DiGraph, T: DiGraph, weight="weight") -> Dict[tuple, float]: T_nodes = T.nodes() G_edges = G.edges() for edge in G_edges: _from, to = edge T_from, T_to = T_nodes[_from], T_nodes[to] G_edge = G_edges[edge] tp = T_from["parent"] # 全てのedge \in G\Tに対してdelta(potential)を計算する if tp is None or tp != to: # tp = to <-> (_from, tp) == (_from, to) tmp = G_edge[weight] + T_to["distance"] - T_from["distance"] G_edge["delta"] = tmp if tmp == tmp else 0
def construct_arg_crm(self): arg_crm = DiGraph() for wp_f in self.crm.nodes_iter(): for d_f in iter(self.data_set): s_f = (wp_f, d_f) arg_crm.add_node(s_f) for wp_t in self.crm.neighbors(wp_f): d_evolve = d_f - self.crm.edge[wp_f][wp_t][ 'rate'] * self.sample_time - 2 * self.quant_size d_t = [d for d in self.data_set if d >= d_evolve][0] s_t = (wp_t, d_t) arg_crm.add_edge(s_f, s_t, weight=1.0) print '---static argumented crm constructed: %d nodes, %d edges----' % ( len(arg_crm.nodes()), len(arg_crm.edges())) self.arg_crm = arg_crm
def get_example_5(): g = DiGraph() g.add_edges_from([(0, 1), (0, 2), (1, 3), (1, 4), (2, 4), (2, 5), (2, 6)]) for s, t in g.edges(): g[s][t]['c'] = 1 g[1][4]['c'] = 0 g[2][4]['c'] = 0 g[2][6]['c'] = 3 for n in g.nodes(): g.node[n]['r'] = 1 g.node[3]['r'] = 10 g.node[4]['r'] = 100 g.node[5]['r'] = 11 U = [10] # sub-optimal answer actually expected_edge_set = [[(0, 2), (2, 4), (2, 5), (2, 6)]] return (g, U, expected_edge_set)
def create_graph_with_new_data(): ''' A DiGraph object holds directed edges - Parallel edges still aren't allowed - For a Digraph, two edges are parallel if they connect the same ordered pair of vertices - Thus, two edges that connect the same vertices but go in different directions are not parallel ''' # Create with edge list g = DiGraph([(1, 5), (5, 1)]) #g = Graph([(1, 5), (5, 1)]) g.add_node(6) print(g.nodes()) # [1, 5, 6] # These are two distinct edges print(g.edges()) # [(1, 5), (5, 1)] print(g.neighbors(1)) # [5] print(g.neighbors(5)) # [1] g.edge[1][5]['foo'] = 'bar' print(g.edge[1][5]) # {'foo': 'bar'} print(g.edge[5][1]) # {}
def get_example_4(): g = DiGraph() g.add_edges_from([ (0, 1), (1, 2), (2, 3), (2, 14), # tree 1 (2, 15), (3, 16), (3, 17), (0, 4), (4, 5), (4, 6), # tree 2 (5, 11), (6, 11), (6, 12), (6, 13), (0, 7), (7, 8), (7, 9), # tree 3 (8, 10), (8, 11), (9, 12), (9, 13) ]) for s, t in g.edges(): g[s][t]['c'] = 1 for n in g.nodes(): g.node[n]['r'] = 1 g.node[10]['r'] = 2 U = [7] expected_edge_set = [[ (0, 7), (7, 8), (7, 9), # tree 3 (8, 10), (8, 11), (9, 12), (9, 13) ]] return (g, U, expected_edge_set)
def find_SCCs(mdp, Sneg): #----simply find strongly connected components---- print 'Remaining states size', len(Sneg) SCC = set() simple_digraph = DiGraph() A = dict() for s in mdp.nodes(): A[s] = mdp.node[s]['act'].copy() for s_f in Sneg: if s_f not in simple_digraph: simple_digraph.add_node(s_f) for s_t in mdp.successors(s_f): if s_t in Sneg: simple_digraph.add_edge(s_f, s_t) print "SubGraph of one Sf: %s states and %s edges" % (str( len(simple_digraph.nodes())), str(len(simple_digraph.edges()))) sccs = strongly_connected_component_subgraphs(simple_digraph) for scc in sccs: SCC.add(frozenset(scc.nodes())) return SCC, A
def get_example_4(): g = DiGraph() g.add_edges_from([(0, 1), (1, 2), (2, 3), (2, 14), # tree 1 (2, 15), (3, 16), (3, 17), (0, 4), (4, 5), (4, 6), # tree 2 (5, 11), (6, 11), (6, 12), (6, 13), (0, 7), (7, 8), (7, 9), # tree 3 (8, 10), (8, 11), (9, 12), (9, 13)]) for s, t in g.edges(): g[s][t]['c'] = 1 for n in g.nodes(): g.node[n]['r'] = 1 g.node[10]['r'] = 2 U = [7] expected_edge_set = [ [(0, 7), (7, 8), (7, 9), # tree 3 (8, 10), (8, 11), (9, 12), (9, 13)] ] return (g, U, expected_edge_set)
def get_example_6(): # IN-OPTIMAL CASE g = DiGraph() g.add_edges_from([(0, 1), (0, 2), (1, 3), (1, 4), (2, 4), (2, 5)]) for s, t in g.edges(): g[s][t]['c'] = 0 g[1][3]['c'] = 4 g[1][4]['c'] = 4 g[2][4]['c'] = 2 g[2][5]['c'] = 1 for n in g.nodes(): g.node[n]['r'] = 0 g.node[3]['r'] = 1 g.node[4]['r'] = 100 g.node[5]['r'] = 1 U = [7] # sub-optimal answer actually expected_edge_set = [[(0, 2), (2, 4), (2, 5)]] return (g, U, expected_edge_set)
def get_variance_example_1(): g = DiGraph() g.add_edges_from([ (0, 1), (0, 2), (2, 3), (3, 4), (2, 'dummy'), ('dummy', 5) ]) g.node['dummy']['dummy'] = True for n in (0, 1, 2, 5): # topic 1 g.node[n]['repr'] = np.array([0, 0]) for n in (3, 4): # topic 2 g.node[n]['repr'] = np.array([1, 1]) for n in g.nodes_iter(): g.node[n]['r'] = 1 # correct is (0, 1, 2, 5) for cost 0 U = [0, 42] expected_edge_set = [ set(g.edges()) - {(2, 3), (3, 4)}, set(g.edges()) ] return (g, U, expected_edge_set)
class StadynaMcgAnalysis: def __init__(self): self.androGuardObjects = [] self.nodes = {} self.nodes_id = {} self.entry_nodes = [] self.G = DiGraph() # self.internal_methods = [] #self.GI = DiGraph() def analyseFile(self, vmx, apk): vm = vmx.get_vm() self.androGuardObjects.append((apk, vm, vmx)) # self.internal_methods.extend(vm.get_methods()) #creating real internal nodes internal_called_methods = vmx.get_tainted_packages( ).stadyna_get_internal_called_methods() for method in internal_called_methods: class_name, method_name, descriptor = method nodeType = None if method_name == "<clinit>": nodeType = NODE_STATIC_INIT elif method_name == "<init>": nodeType = NODE_CONSTRUCTOR else: nodeType = NODE_METHOD n = self._get_node(nodeType, (class_name, method_name, descriptor)) n.set_attribute(ATTR_CLASS_NAME, class_name) n.set_attribute(ATTR_METHOD_NAME, method_name) n.set_attribute(ATTR_DESCRIPTOR, descriptor) self.G.add_node(n.id) #creating real edges (nodes are already there) #currently we are working only with internal packages. for j in vmx.get_tainted_packages().get_internal_packages(): src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager()) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager()) n1 = self._get_existed_node( (src_class_name, src_method_name, src_descriptor)) # n1.set_attribute(ATTR_CLASS_NAME, src_class_name) # n1.set_attribute(ATTR_METHOD_NAME, src_method_name) # n1.set_attribute(ATTR_DESCRIPTOR, src_descriptor) n2 = self._get_existed_node( (dst_class_name, dst_method_name, dst_descriptor)) # n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) # n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) # n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) self.G.add_edge(n1.id, n2.id) #adding fake class nodes for method in internal_called_methods: src_class_name, src_method_name, src_descriptor = method if src_method_name == "<init>" or src_method_name == "<clinit>": n1 = self._get_existed_node( (src_class_name, src_method_name, src_descriptor)) n2 = self._get_node(NODE_FAKE_CLASS, src_class_name, None, False) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) if src_method_name == "<clinit>": self.G.add_edge(n1.id, n2.id) elif src_method_name == "<init>": self.G.add_edge(n2.id, n1.id) #real (external) reflection invoke nodes reflection_invoke_paths = analysis.seccon_get_invoke_method_paths(vmx) for j in reflection_invoke_paths: src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager()) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager()) n1 = self._get_existed_node( (src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning( "Cannot find the node [%s], where reflection invoke is called!" % (src_class_name, src_method_name, src_descriptor)) continue key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_INVOKE) n2 = self._get_node(NODE_REFL_INVOKE, key, LABEL_REFL_INVOKE, True) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) n2.set_attribute(ATTR_METHOD_NAME, src_method_name) n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor) self.G.add_edge(n1.id, n2.id) #real (external) reflection new instance nodes reflection_newInstance_paths = analysis.seccon_get_newInstance_method_paths( vmx) for j in reflection_newInstance_paths: src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager()) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager()) n1 = self._get_existed_node( (src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning( "Cannot find the node [%s], where reflection new instance is called!" % (src_class_name, src_method_name, src_descriptor)) continue key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_NEWINSTANCE) n2 = self._get_node(NODE_REFL_NEWINSTANCE, key, LABEL_REFL_NEWINSTANCE, True) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) n2.set_attribute(ATTR_METHOD_NAME, src_method_name) n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor) self.G.add_edge(n1.id, n2.id) #adding fake entry points if apk != None: for i in apk.get_activities(): j = bytecode.FormatClassToJava(i) n1 = self._get_existed_node( (j, "onCreate", "(Landroid/os/Bundle;)V")) if n1 != None: key = "%s %s %s %s" % (j, "onCreate", "(Landroid/os/Bundle;)V", POSTFIX_ACTIVITY) n2 = self._get_node(NODE_FAKE_ACTIVITY, key, LABEL_ACTIVITY, False) self.G.add_edge(n2.id, n1.id) self.entry_nodes.append(n1.id) for i in apk.get_services(): j = bytecode.FormatClassToJava(i) n1 = self._get_existed_node((j, "onCreate", "()V")) if n1 != None: key = "%s %s %s %s" % (j, "onCreate", "()V", POSTFIX_SERVICE) n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_SERVICE, False) self.G.add_edge(n2.id, n1.id) self.entry_nodes.append(n1.id) for i in apk.get_receivers(): j = bytecode.FormatClassToJava(i) n1 = self._get_existed_node( (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V")) if n1 != None: key = "%s %s %s %s" % ( j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V", POSTFIX_RECEIVER) n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_RECEIVER, False) self.G.add_edge(n2.id, n1.id) self.entry_nodes.append(n1.id) #fake permissions list_permissions = vmx.stadyna_get_permissions([]) for x in list_permissions: for j in list_permissions[x]: if isinstance(j, PathVar): continue src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager()) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager()) n1 = self._get_existed_node( (src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning( "Cannot find node [%s %s %s] for permission [%s]!" % (src_class_name, src_method_name, src_descriptor, x)) continue #SOURCE, DEST, POSTFIX, PERMISSION_NAME key = "%s %s %s %s %s %s %s %s" % ( src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, x) n2 = self._get_node(NODE_FAKE_PERMISSION, key, x, False) n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) n2.set_attribute(ATTR_PERM_NAME, x) n2.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[x][0]) self.G.add_edge(n1.id, n2.id) #fake DexClassLoader nodes dyn_code_loading = analysis.seccon_get_dyncode_loading_paths(vmx) for j in dyn_code_loading: src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager()) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager()) n1 = self._get_existed_node( (src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning( "Cannot find dexload node [%s]!" % (src_class_name, src_method_name, src_descriptor)) continue key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_DEXLOAD) n2 = self._get_node(NODE_FAKE_DEXLOAD, key, LABEL_DEXLOAD, False) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) n2.set_attribute(ATTR_METHOD_NAME, src_method_name) n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor) self.G.add_edge(n1.id, n2.id) # Specific Java/Android library for c in vm.get_classes(): #if c.get_superclassname() == "Landroid/app/Service;" : # n1 = self._get_node( c.get_name(), "<init>", "()V" ) # n2 = self._get_node( c.get_name(), "onCreate", "()V" ) # self.G.add_edge( n1.id, n2.id ) if c.get_superclassname( ) == "Ljava/lang/Thread;" or c.get_superclassname( ) == "Ljava/util/TimerTask;": for i in vm.get_method("run"): if i.get_class_name() == c.get_name(): n1 = self._get_node(NODE_METHOD, (i.get_class_name(), i.get_name(), i.get_descriptor())) n2 = self._get_node( NODE_METHOD, (i.get_class_name(), "start", i.get_descriptor())) # link from start to run self.G.add_edge(n2.id, n1.id) #n2.add_edge( n1, {} ) # link from init to start for init in vm.get_method("<init>"): if init.get_class_name() == c.get_name(): #TODO: Leaving _get_existed_node to check if all the nodes are included #It is possible that internal_packages does not contain this node. Leaving _get_existed_node to check this n3 = self._get_node( NODE_CONSTRUCTOR, (init.get_class_name(), "<init>", init.get_descriptor())) self.G.add_edge(n3.id, n2.id) #n3.add_edge( n2, {} ) def addInvokePath(self, src, through, dst): src_class_name, src_method_name, src_descriptor = src dst_class_name, dst_method_name, dst_descriptor = dst through_class_name, through_method_name, through_descriptor = through key = "%s %s %s %s %s %s %s" % ( src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_INVOKE) n1 = self._get_existed_node(key) if n1 == None: logger.warning( "Something wrong has happened! Could not find invoke Node in Graph with key [%s]" % str(key)) return n2 = self._get_node(NODE_METHOD, (dst_class_name, dst_method_name, dst_descriptor)) n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) self.G.add_edge(n1.id, n2.id) #check if called method calls protected feature data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor) if data in DVM_PERMISSIONS_BY_API_CALLS: logger.info( "BINGOOOOOOO! The protected method is called through reflection!" ) perm = DVM_PERMISSIONS_BY_API_CALLS[data] key1 = "%s %s %s %s %s %s %s %s" % ( through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm) n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False) n3.set_attribute(ATTR_CLASS_NAME, dst_class_name) n3.set_attribute(ATTR_METHOD_NAME, dst_method_name) n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) n3.set_attribute(ATTR_PERM_NAME, perm) n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[perm][0]) self.G.add_edge(n2.id, n3.id) def addNewInstancePath(self, src, through, dst): src_class_name, src_method_name, src_descriptor = src dst_class_name, dst_method_name, dst_descriptor = dst through_class_name, through_method_name, through_descriptor = through key = "%s %s %s %s %s %s %s" % ( src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_NEWINSTANCE) n1 = self._get_existed_node(key) if n1 == None: logger.error( "Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key)) return n2 = self._get_node(NODE_CONSTRUCTOR, (dst_class_name, dst_method_name, dst_descriptor)) n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) self.G.add_edge(n1.id, n2.id) #we also need to add link to the class node #TODO: Think in the future what to do with this n_class = self._get_node(NODE_FAKE_CLASS, dst_class_name, None, False) n_class.set_attribute(ATTR_CLASS_NAME, dst_class_name) self.G.add_edge(n_class.id, n2.id) #checking if we need to add additional permission nodes data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor) if data in DVM_PERMISSIONS_BY_API_CALLS: logger.info( "BINGOOOOOOO! The protected method is called through reflection!" ) perm = DVM_PERMISSIONS_BY_API_CALLS[data] key1 = "%s %s %s %s %s %s %s %s" % ( through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm) n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False) n3.set_attribute(ATTR_CLASS_NAME, dst_class_name) n3.set_attribute(ATTR_METHOD_NAME, dst_method_name) n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) n3.set_attribute(ATTR_PERM_NAME, perm) n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[perm][0]) self.G.add_edge(n2.id, n3.id) def addDexloadPath(self, src, through, filename): src_class_name, src_method_name, src_descriptor = src through_class_name, through_method_name, through_descriptor = through key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_DEXLOAD) n1 = self._get_existed_node(key) if n1 == None: logger.error( "Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key)) return n2 = self._get_node(NODE_FAKE_DEXLOAD_FILE, filename, filename, False) n2.set_attribute(ATTR_DEXLOAD_FILENAME, filename) self.G.add_edge(n1.id, n2.id) def _get_node(self, nType, key, label=None, real=True): node_key = None if isinstance(key, basestring): node_key = key elif isinstance(key, tuple): node_key = "%s %s %s" % key else: logger.error("Unknown instance type of key!!!") if node_key not in self.nodes.keys(): new_node = NodeS(len(self.nodes), nType, node_key, label, real) self.nodes[node_key] = new_node self.nodes_id[new_node.id] = new_node return self.nodes[node_key] def _get_existed_node(self, key): node_key = None if isinstance(key, basestring): node_key = key elif isinstance(key, tuple): node_key = "%s %s %s" % key else: logger.error("Unknown instance type of key!!!") try: return self.nodes[node_key] except KeyError: logger.error("Could not find existed node [%s]!" % node_key) return None def export_to_gexf(self): buff = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" buff += "<gexf xmlns=\"http://www.gephi.org/gexf\" xmlns:viz=\"http://www.gephi.org/gexf/viz\">\n" buff += "<graph type=\"static\">\n" buff += "<attributes class=\"node\" type=\"static\">\n" buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_TYPE, ID_ATTRIBUTES[ATTR_TYPE]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_CLASS_NAME, ID_ATTRIBUTES[ATTR_CLASS_NAME]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_METHOD_NAME, ID_ATTRIBUTES[ATTR_METHOD_NAME]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_DESCRIPTOR, ID_ATTRIBUTES[ATTR_DESCRIPTOR]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_REAL, ID_ATTRIBUTES[ATTR_REAL]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_PERM_NAME, ID_ATTRIBUTES[ATTR_PERM_NAME]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_PERM_LEVEL, ID_ATTRIBUTES[ATTR_PERM_LEVEL]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % ( ATTR_DEXLOAD_FILENAME, ID_ATTRIBUTES[ATTR_DEXLOAD_FILENAME]) buff += "</attributes>\n" # buff += "<attributes class=\"node\" type=\"static\">\n" # buff += "<attribute id=\"%d\" title=\"type\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "type"] # buff += "<attribute id=\"%d\" title=\"class_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "class_name"] # buff += "<attribute id=\"%d\" title=\"method_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "method_name"] # buff += "<attribute id=\"%d\" title=\"descriptor\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "descriptor"] # # # buff += "<attribute id=\"%d\" title=\"permissions\" type=\"integer\" default=\"0\"/>\n" % ID_ATTRIBUTES[ "permissions"] # buff += "<attribute id=\"%d\" title=\"permissions_level\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "permissions_level"] # # buff += "<attribute id=\"%d\" title=\"dynamic_code\" type=\"boolean\" default=\"false\"/>\n" % ID_ATTRIBUTES[ "dynamic_code"] # # buff += "</attributes>\n" buff += "<nodes>\n" for node in self.G.nodes(): buff += "<node id=\"%d\" label=\"%s\">\n" % ( node, escape(self.nodes_id[node].label)) buff += self.nodes_id[node].get_attributes_gexf() buff += "</node>\n" buff += "</nodes>\n" buff += "<edges>\n" nb = 0 for edge in self.G.edges(): buff += "<edge id=\"%d\" source=\"%d\" target=\"%d\"/>\n" % ( nb, edge[0], edge[1]) nb += 1 buff += "</edges>\n" buff += "</graph>\n" buff += "</gexf>\n" return buff def get_current_real_node_count(self): count = 0 for node in self.nodes_id.keys(): if self.nodes_id[node].get_attribute(ATTR_REAL) == "True": count += 1 return count def get_current_node_count(self): return len(self.G.nodes()) def get_current_edge_count(self): return len(self.G.edges()) def get_current_permission_level_node_count(self, permission_level): count = 0 for node in self.nodes_id.keys(): if self.nodes_id[node].get_attribute( ATTR_PERM_LEVEL) == permission_level: count += 1 return count def get_current_protected_node_count(self): count = 0 for node in self.nodes_id.keys(): if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) != None: count += 1 return count
def load(fname): def clean_bool(string): if string == "0": return None else: return string def to_bool(string): if string == "1" or string == "True": return True elif string == "0" or string == "False": return False else: return string def to_float(string): if string == "None": return None try: return float(string) except: return string mode = "node0" nodes = [] edges = [] pointers = set() outputs = None inputs = None named_ranges = {} infile = gzip.GzipFile(fname, 'r') for line in infile.read().splitlines(): if line == "====": mode = "node0" continue if line == "-----": cellmap_temp = {n.address(): n for n in nodes} Range = RangeFactory(cellmap_temp) mode = "node0" continue elif line == "edges": cellmap = {n.address(): n for n in nodes} mode = "edges" continue elif line == "outputs": mode = "outputs" continue elif line == "inputs": mode = "inputs" continue elif line == "named_ranges": mode = "named_ranges" continue if mode == "node0": [ address, formula, python_expression, is_range, is_named_range, is_pointer, should_eval ] = line.split(SEP) formula = clean_bool(formula) python_expression = clean_bool(python_expression) is_range = to_bool(is_range) is_named_range = to_bool(is_named_range) is_pointer = to_bool(is_pointer) should_eval = should_eval mode = "node1" elif mode == "node1": if is_range: reference = json.loads( line ) if is_pointer else line # in order to be able to parse dicts vv = Range(reference) if is_pointer: if not is_named_range: address = vv.name pointers.add(address) cell = Cell(address, None, vv, formula, is_range, is_named_range, should_eval) cell.python_expression = python_expression nodes.append(cell) else: value = to_bool(to_float(line)) cell = Cell(address, None, value, formula, is_range, is_named_range, should_eval) cell.python_expression = python_expression if formula: if 'OFFSET' in formula or 'INDEX' in formula: pointers.add(address) cell.compile() nodes.append(cell) elif mode == "edges": source, target = line.split(SEP) edges.append((cellmap[source], cellmap[target])) elif mode == "outputs": outputs = line.split(SEP) elif mode == "inputs": inputs = line.split(SEP) elif mode == "named_ranges": k, v = line.split(SEP) named_ranges[k] = v G = DiGraph(data=edges) print "Graph loading done, %s nodes, %s edges, %s cellmap entries" % (len( G.nodes()), len(G.edges()), len(cellmap)) return (G, cellmap, named_ranges, pointers, outputs, inputs)
class StadynaMcgAnalysis: def __init__(self): self.androGuardObjects = [] self.nodes = {} self.nodes_id = {} self.entry_nodes = [] self.G = DiGraph() # self.internal_methods = [] #self.GI = DiGraph() def analyseFile(self, vmx, apk): vm = vmx.get_vm() self.androGuardObjects.append((apk, vm, vmx)) # self.internal_methods.extend(vm.get_methods()) #creating real internal nodes internal_called_methods = vmx.get_tainted_packages().stadyna_get_internal_called_methods() for method in internal_called_methods: class_name, method_name, descriptor = method nodeType = None if method_name == "<clinit>": nodeType = NODE_STATIC_INIT elif method_name == "<init>": nodeType = NODE_CONSTRUCTOR else: nodeType = NODE_METHOD n = self._get_node(nodeType, (class_name, method_name, descriptor)) n.set_attribute(ATTR_CLASS_NAME, class_name) n.set_attribute(ATTR_METHOD_NAME, method_name) n.set_attribute(ATTR_DESCRIPTOR, descriptor) self.G.add_node(n.id) #creating real edges (nodes are already there) #currently we are working only with internal packages. for j in vmx.get_tainted_packages().get_internal_packages(): src_class_name, src_method_name, src_descriptor = j.get_src(vm.get_class_manager()) dst_class_name, dst_method_name, dst_descriptor = j.get_dst(vm.get_class_manager()) n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor)) # n1.set_attribute(ATTR_CLASS_NAME, src_class_name) # n1.set_attribute(ATTR_METHOD_NAME, src_method_name) # n1.set_attribute(ATTR_DESCRIPTOR, src_descriptor) n2 = self._get_existed_node((dst_class_name, dst_method_name, dst_descriptor)) # n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) # n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) # n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) self.G.add_edge(n1.id, n2.id) #adding fake class nodes for method in internal_called_methods: src_class_name, src_method_name, src_descriptor = method if src_method_name == "<init>" or src_method_name == "<clinit>": n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor)) n2 = self._get_node(NODE_FAKE_CLASS, src_class_name, None, False) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) if src_method_name == "<clinit>": self.G.add_edge(n1.id, n2.id) elif src_method_name == "<init>": self.G.add_edge(n2.id, n1.id) #real (external) reflection invoke nodes reflection_invoke_paths = analysis.seccon_get_invoke_method_paths(vmx) for j in reflection_invoke_paths: src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() ) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() ) n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning("Cannot find the node [%s], where reflection invoke is called!" % (src_class_name, src_method_name, src_descriptor)) continue key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_INVOKE) n2 = self._get_node(NODE_REFL_INVOKE, key, LABEL_REFL_INVOKE, True) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) n2.set_attribute(ATTR_METHOD_NAME, src_method_name) n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor) self.G.add_edge( n1.id, n2.id ) #real (external) reflection new instance nodes reflection_newInstance_paths = analysis.seccon_get_newInstance_method_paths(vmx) for j in reflection_newInstance_paths: src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() ) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() ) n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning("Cannot find the node [%s], where reflection new instance is called!" % (src_class_name, src_method_name, src_descriptor)) continue key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_REFL_NEWINSTANCE) n2 = self._get_node(NODE_REFL_NEWINSTANCE, key, LABEL_REFL_NEWINSTANCE, True) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) n2.set_attribute(ATTR_METHOD_NAME, src_method_name) n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor) self.G.add_edge( n1.id, n2.id ) #adding fake entry points if apk != None: for i in apk.get_activities() : j = bytecode.FormatClassToJava(i) n1 = self._get_existed_node((j, "onCreate", "(Landroid/os/Bundle;)V")) if n1 != None: key = "%s %s %s %s" % (j, "onCreate", "(Landroid/os/Bundle;)V", POSTFIX_ACTIVITY) n2 = self._get_node(NODE_FAKE_ACTIVITY, key, LABEL_ACTIVITY, False) self.G.add_edge( n2.id, n1.id ) self.entry_nodes.append( n1.id ) for i in apk.get_services() : j = bytecode.FormatClassToJava(i) n1 = self._get_existed_node( (j, "onCreate", "()V") ) if n1 != None : key = "%s %s %s %s" % (j, "onCreate", "()V", POSTFIX_SERVICE) n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_SERVICE, False) self.G.add_edge( n2.id, n1.id ) self.entry_nodes.append( n1.id ) for i in apk.get_receivers() : j = bytecode.FormatClassToJava(i) n1 = self._get_existed_node( (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V") ) if n1 != None : key = "%s %s %s %s" % (j, "onReceive", "(Landroid/content/Context;Landroid/content/Intent;)V", POSTFIX_RECEIVER) n2 = self._get_node(NODE_FAKE_SERVICE, key, LABEL_RECEIVER, False) self.G.add_edge( n2.id, n1.id ) self.entry_nodes.append( n1.id ) #fake permissions list_permissions = vmx.stadyna_get_permissions([]) for x in list_permissions: for j in list_permissions[x]: if isinstance(j, PathVar): continue src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() ) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() ) n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning("Cannot find node [%s %s %s] for permission [%s]!" % (src_class_name, src_method_name, src_descriptor, x)) continue #SOURCE, DEST, POSTFIX, PERMISSION_NAME key = "%s %s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, x) n2 = self._get_node(NODE_FAKE_PERMISSION, key, x, False) n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) n2.set_attribute(ATTR_PERM_NAME, x) n2.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ x ][0]) self.G.add_edge(n1.id, n2.id) #fake DexClassLoader nodes dyn_code_loading = analysis.seccon_get_dyncode_loading_paths(vmx) for j in dyn_code_loading: src_class_name, src_method_name, src_descriptor = j.get_src( vm.get_class_manager() ) dst_class_name, dst_method_name, dst_descriptor = j.get_dst( vm.get_class_manager() ) n1 = self._get_existed_node((src_class_name, src_method_name, src_descriptor)) if n1 == None: logger.warning("Cannot find dexload node [%s]!" % (src_class_name, src_method_name, src_descriptor)) continue key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_DEXLOAD) n2 = self._get_node(NODE_FAKE_DEXLOAD, key, LABEL_DEXLOAD, False) n2.set_attribute(ATTR_CLASS_NAME, src_class_name) n2.set_attribute(ATTR_METHOD_NAME, src_method_name) n2.set_attribute(ATTR_DESCRIPTOR, src_descriptor) self.G.add_edge( n1.id, n2.id ) # Specific Java/Android library for c in vm.get_classes(): #if c.get_superclassname() == "Landroid/app/Service;" : # n1 = self._get_node( c.get_name(), "<init>", "()V" ) # n2 = self._get_node( c.get_name(), "onCreate", "()V" ) # self.G.add_edge( n1.id, n2.id ) if c.get_superclassname() == "Ljava/lang/Thread;" or c.get_superclassname() == "Ljava/util/TimerTask;" : for i in vm.get_method("run") : if i.get_class_name() == c.get_name() : n1 = self._get_node(NODE_METHOD, (i.get_class_name(), i.get_name(), i.get_descriptor())) n2 = self._get_node(NODE_METHOD, (i.get_class_name(), "start", i.get_descriptor())) # link from start to run self.G.add_edge( n2.id, n1.id ) #n2.add_edge( n1, {} ) # link from init to start for init in vm.get_method("<init>") : if init.get_class_name() == c.get_name(): #TODO: Leaving _get_existed_node to check if all the nodes are included #It is possible that internal_packages does not contain this node. Leaving _get_existed_node to check this n3 = self._get_node(NODE_CONSTRUCTOR, (init.get_class_name(), "<init>", init.get_descriptor())) self.G.add_edge( n3.id, n2.id ) #n3.add_edge( n2, {} ) def addInvokePath(self, src, through, dst): src_class_name, src_method_name, src_descriptor = src dst_class_name, dst_method_name, dst_descriptor = dst through_class_name, through_method_name, through_descriptor = through key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_INVOKE) n1 = self._get_existed_node(key) if n1 == None: logger.warning("Something wrong has happened! Could not find invoke Node in Graph with key [%s]" % str(key)) return n2 = self._get_node(NODE_METHOD, (dst_class_name, dst_method_name, dst_descriptor)) n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) self.G.add_edge(n1.id, n2.id) #check if called method calls protected feature data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor) if data in DVM_PERMISSIONS_BY_API_CALLS: logger.info("BINGOOOOOOO! The protected method is called through reflection!") perm = DVM_PERMISSIONS_BY_API_CALLS[ data ] key1 = "%s %s %s %s %s %s %s %s" % (through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm) n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False) n3.set_attribute(ATTR_CLASS_NAME, dst_class_name) n3.set_attribute(ATTR_METHOD_NAME, dst_method_name) n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) n3.set_attribute(ATTR_PERM_NAME, perm) n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ perm ][0]) self.G.add_edge(n2.id, n3.id) def addNewInstancePath(self, src, through, dst): src_class_name, src_method_name, src_descriptor = src dst_class_name, dst_method_name, dst_descriptor = dst through_class_name, through_method_name, through_descriptor = through key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_REFL_NEWINSTANCE) n1 = self._get_existed_node(key) if n1 == None: logger.error("Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key)) return n2 = self._get_node(NODE_CONSTRUCTOR, (dst_class_name, dst_method_name, dst_descriptor)) n2.set_attribute(ATTR_CLASS_NAME, dst_class_name) n2.set_attribute(ATTR_METHOD_NAME, dst_method_name) n2.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) self.G.add_edge(n1.id, n2.id) #we also need to add link to the class node #TODO: Think in the future what to do with this n_class = self._get_node(NODE_FAKE_CLASS, dst_class_name, None, False) n_class.set_attribute(ATTR_CLASS_NAME, dst_class_name) self.G.add_edge(n_class.id, n2.id) #checking if we need to add additional permission nodes data = "%s-%s-%s" % (dst_class_name, dst_method_name, dst_descriptor) if data in DVM_PERMISSIONS_BY_API_CALLS: logger.info("BINGOOOOOOO! The protected method is called through reflection!") perm = DVM_PERMISSIONS_BY_API_CALLS[ data ] key1 = "%s %s %s %s %s %s %s %s" % (through_class_name, through_method_name, through_descriptor, dst_class_name, dst_method_name, dst_descriptor, POSTFIX_PERM, perm) n3 = self._get_node(NODE_FAKE_PERMISSION, key1, perm, False) n3.set_attribute(ATTR_CLASS_NAME, dst_class_name) n3.set_attribute(ATTR_METHOD_NAME, dst_method_name) n3.set_attribute(ATTR_DESCRIPTOR, dst_descriptor) n3.set_attribute(ATTR_PERM_NAME, perm) n3.set_attribute(ATTR_PERM_LEVEL, MANIFEST_PERMISSIONS[ perm ][0]) self.G.add_edge(n2.id, n3.id) def addDexloadPath(self, src, through, filename): src_class_name, src_method_name, src_descriptor = src through_class_name, through_method_name, through_descriptor = through key = "%s %s %s %s %s %s %s" % (src_class_name, src_method_name, src_descriptor, through_class_name, through_method_name, through_descriptor, POSTFIX_DEXLOAD) n1 = self._get_existed_node(key) if n1 == None: logger.error("Something wrong has happened! Could not find Node in Graph with key [%s]" % str(key)) return n2 = self._get_node(NODE_FAKE_DEXLOAD_FILE, filename, filename, False) n2.set_attribute(ATTR_DEXLOAD_FILENAME, filename) self.G.add_edge(n1.id, n2.id) def _get_node(self, nType, key, label=None, real=True): node_key = None if isinstance(key, basestring): node_key = key elif isinstance(key, tuple): node_key = "%s %s %s" % key else: logger.error("Unknown instance type of key!!!") if node_key not in self.nodes.keys(): new_node = NodeS(len(self.nodes), nType, node_key, label, real) self.nodes[node_key] = new_node self.nodes_id[new_node.id] = new_node return self.nodes[node_key] def _get_existed_node(self, key): node_key = None if isinstance(key, basestring): node_key = key elif isinstance(key, tuple): node_key = "%s %s %s" % key else: logger.error("Unknown instance type of key!!!") try: return self.nodes[node_key] except KeyError: logger.error("Could not find existed node [%s]!" % node_key) return None def export_to_gexf(self) : buff = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" buff += "<gexf xmlns=\"http://www.gephi.org/gexf\" xmlns:viz=\"http://www.gephi.org/gexf/viz\">\n" buff += "<graph type=\"static\">\n" buff += "<attributes class=\"node\" type=\"static\">\n" buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_TYPE, ID_ATTRIBUTES[ ATTR_TYPE ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_CLASS_NAME, ID_ATTRIBUTES[ ATTR_CLASS_NAME ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_METHOD_NAME, ID_ATTRIBUTES[ ATTR_METHOD_NAME ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_DESCRIPTOR, ID_ATTRIBUTES[ ATTR_DESCRIPTOR ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_REAL, ID_ATTRIBUTES[ ATTR_REAL ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_PERM_NAME, ID_ATTRIBUTES[ ATTR_PERM_NAME ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_PERM_LEVEL, ID_ATTRIBUTES[ ATTR_PERM_LEVEL ]) buff += "<attribute title=\"%s\" id=\"%d\" type=\"string\"/>\n" % (ATTR_DEXLOAD_FILENAME, ID_ATTRIBUTES[ ATTR_DEXLOAD_FILENAME ]) buff += "</attributes>\n" # buff += "<attributes class=\"node\" type=\"static\">\n" # buff += "<attribute id=\"%d\" title=\"type\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "type"] # buff += "<attribute id=\"%d\" title=\"class_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "class_name"] # buff += "<attribute id=\"%d\" title=\"method_name\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "method_name"] # buff += "<attribute id=\"%d\" title=\"descriptor\" type=\"string\"/>\n" % ID_ATTRIBUTES[ "descriptor"] # # # buff += "<attribute id=\"%d\" title=\"permissions\" type=\"integer\" default=\"0\"/>\n" % ID_ATTRIBUTES[ "permissions"] # buff += "<attribute id=\"%d\" title=\"permissions_level\" type=\"string\" default=\"normal\"/>\n" % ID_ATTRIBUTES[ "permissions_level"] # # buff += "<attribute id=\"%d\" title=\"dynamic_code\" type=\"boolean\" default=\"false\"/>\n" % ID_ATTRIBUTES[ "dynamic_code"] # # buff += "</attributes>\n" buff += "<nodes>\n" for node in self.G.nodes() : buff += "<node id=\"%d\" label=\"%s\">\n" % (node, escape(self.nodes_id[ node ].label)) buff += self.nodes_id[ node ].get_attributes_gexf() buff += "</node>\n" buff += "</nodes>\n" buff += "<edges>\n" nb = 0 for edge in self.G.edges() : buff += "<edge id=\"%d\" source=\"%d\" target=\"%d\"/>\n" % (nb, edge[0], edge[1]) nb += 1 buff += "</edges>\n" buff += "</graph>\n" buff += "</gexf>\n" return buff def get_current_real_node_count(self): count = 0 for node in self.nodes_id.keys(): if self.nodes_id[node].get_attribute(ATTR_REAL) == "True": count += 1 return count def get_current_node_count(self): return len(self.G.nodes()) def get_current_edge_count(self): return len(self.G.edges()) def get_current_permission_level_node_count(self, permission_level): count = 0 for node in self.nodes_id.keys(): if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) == permission_level: count += 1 return count def get_current_protected_node_count(self): count = 0 for node in self.nodes_id.keys(): if self.nodes_id[node].get_attribute(ATTR_PERM_LEVEL) != None: count += 1 return count
else: #no collative task if len(colla_ap[other_agent]) == 0: continue else: for ap_item in colla_ap[other_agent]: print(ap_item) #zai region zhong zhao dao suo you yu ap dui ying de qu yu dian cor_pos_list = find_region(ap_item, region_array[other_agent]) print(cor_pos_list) #zai fts zhao dao mei ge qu yu dian de qian ji yi ji cost, qu zui xiao de cun xia lai cost_static = float('inf') fts_digraph = DiGraph() fts_digraph.add_edges_from(fts_array[other_agent].edges()) print(fts_digraph.edges()) for pos in cor_pos_list: if len(pos) != 3: continue for pre_pos in fts_digraph.predecessors(pos): if pre_pos == pos: #pai chu yuan di bu dong de qing kuang continue cost_buf = distance(pos, pre_pos) if cost_buf < cost_static: cost_static = cost_buf #zai ben ji qi ren de product zhong jia bian ,ju ti jia fa an guomeng zhi qian gou jian product zi dong ji de fang fa print(cost_static) for static_edge in static_list[ agent_index]: #shu ju jie guo de tong yi if ap_item in static_edge[2]: print(static_edge[0], static_edge[1],
class buchi_graph(object): """ construct buchi automaton graph Parameter: formula: LTL formula specifying task """ def __init__(self, formula, formula_comp, exclusion): self.formula = formula self.formula_comp = formula_comp self.exclusion = exclusion def formulaParser(self): """replace letter with symbol """ indicator = 'FG' if [True for i in indicator if i in self.formula]: self.formula.replace('F', '<>').replace('G', '[]') def execLtl2ba(self): """ given formula, exectute the ltl2ba Parameter: buchi_str: output string of program ltl2ba (utf-8 format) """ dirname = os.path.dirname(__file__) self.buchi_str = subprocess.check_output(dirname + "/./ltl2ba -f \"" + self.formula + "\"", shell=True).decode("utf-8") def buchiGraph(self): """parse the output of ltl2ba Parameter: buchi_graph: Graph of buchi automaton """ # find all states state_re = re.compile(r'\n(\w+):\n\t') state_group = re.findall(state_re, self.buchi_str) # find initial and accepting states init = [s for s in state_group if 'init' in s] accep = [s for s in state_group if 'accept' in s] """ Format: buchi_graph.node = NodeView(('T0_init', 'T1_S1', 'accept_S1')) buchi_graph.edges = OutEdgeView([('T0_init', 'T0_init'), ('T0_init', 'T1_S1'),....]) buchi_graph.succ = AdjacencyView({'T0_init': {'T0_init': {'label': '1'}, 'T1_S1': {'label': 'r3'}}}) """ self.buchi_graph = DiGraph(type='buchi', init=init, accept=accep) order_key = list(self.formula_comp.keys()) order_key.sort(reverse=True) for state in state_group: # for each state, find transition relation # add node self.buchi_graph.add_node(state) state_if_fi = re.findall(state + r':\n\tif(.*?)fi', self.buchi_str, re.DOTALL) if state_if_fi: relation_group = re.findall(r':: (\(.*?\)) -> goto (\w+)\n\t', state_if_fi[0]) for (labell, state_dest) in relation_group: # whether the edge is feasible in terms of unit atomic proposition label = self.InitialDelInfesEdge(labell) if not label or label.isspace(): continue # add edge for k in order_key: if k >= 10: label = label.replace('e_{0}'.format(k), self.formula_comp[k]) else: label = label.replace('e{0}'.format(k), self.formula_comp[k]) # if '!' in label: # label = self.PutNotInside(label) self.buchi_graph.add_edge(state, state_dest, label=label) return self.buchi_graph def ShorestPathBtRg(self, regions): """ calculate shoresr path between any two labeled regions :param regions: regions :return: dict (region, region) : length """ polys = [[vg.Point(0.4, 1.0), vg.Point(0.4, 0.7), vg.Point(0.6, 0.7), vg.Point(0.6, 1.0)], [vg.Point(0.3, 0.2), vg.Point(0.3, 0.0), vg.Point(0.7, 0.0), vg.Point(0.7, 0.2)]] g = vg.VisGraph() g.build(polys, status=False) min_len_region = dict() for key1, value1 in regions.items(): for key2, value2 in regions.items(): init = value1[:2] tg = value2[:2] # shorest path between init and tg point shortest = g.shortest_path(vg.Point(init[0], init[1]), vg.Point(tg[0], tg[1])) # (key2, key1) is already checked if (key2, key1) in min_len_region.keys(): min_len_region[(key1, key2)] = min_len_region[(key2, key1)] else: # different regions if key1 != key2: dis = 0 for i in range(len(shortest)-1): dis = dis + np.linalg.norm(np.subtract((shortest[i].x, shortest[i].y), (shortest[i+1].x, shortest[i+1].y))) min_len_region[(key1, key2)] = dis # same region else: min_len_region[(key1, key2)] = 0 return min_len_region def RobotRegion(self, exp, robot): """ pair of robot and corresponding regions in the expression :param exp: logical expression :param robot: # of robots :return: dic of robot index : regions exp = 'l1_1 & l3_1 & l4_1 & l4_6 | l3_4 & l5_6' {1: ['l1_1', 'l3_1', 'l4_1'], 4: ['l3_4'], 6: ['l4_6', 'l5_6']} """ robot_region_dict = dict() for r in range(robot): findall = re.findall(r'(l\d+?_{0})[^0-9]'.format(r + 1), exp) if findall: robot_region_dict[str(r + 1)] = findall return robot_region_dict def FeasTruthTable(self, exp, robot_region): """ Find feasible truth table to make exp true :param exp: expression :return: """ if exp == '(1)': return '1' sgl_value = [] for key, value in robot_region.items(): if len(value) == 1: sgl_value.append(value[0]) # set all to be false exp1 = to_cnf(exp) value_in_exp = [value.name for value in exp1.atoms()] subs = {true_rb_rg: False for true_rb_rg in value_in_exp} if exp1.subs(subs): return subs # set one to be true, the other to be false for prod in itertools.product(*robot_region.values()): exp1 = exp # set one specific item to be true for true_rb_rg in prod: # set the other to be false value_cp = list(robot_region[true_rb_rg.split('_')[1]]) if len(value_cp) > 1: value_cp.remove(true_rb_rg) # replace the rest with same robot to be ~ for v_remove in value_cp: exp1 = exp1.replace(v_remove, '~' + true_rb_rg) # simplify exp1 = to_cnf(exp1) # all value in expression value_in_exp = [value.name for value in exp1.atoms()] # all single value in expression sgl_value_in_exp = [value for value in value_in_exp if value in sgl_value] # not signle value in expression not_sgl_value_in_exp = [value for value in value_in_exp if value not in sgl_value] subs1 = {true_rb_rg: True for true_rb_rg in not_sgl_value_in_exp} tf = [False, True] # if type(exp1) == Or: # tf = [False, True] if len(sgl_value_in_exp): for p in itertools.product(*[tf] * len(sgl_value_in_exp)): subs2 = {sgl_value_in_exp[i]: p[i] for i in range(len(sgl_value_in_exp))} subs = {**subs1, **subs2} if exp1.subs(subs): return subs else: if exp1.subs(subs1): return subs1 return [] def DelInfesEdge(self, robot): """ Delete infeasible edge :param buchi_graph: buchi automaton :param robot: # robot """ TobeDel = [] # print(self.buchi_graph.number_of_edges()) i = 0 for edge in self.buchi_graph.edges(): i = i+1 # print(i) b_label = self.buchi_graph.edges[edge]['label'] # multiple labels if ') && (' in b_label: TobeDel.append(edge) continue if b_label != '(1)': exp = b_label.replace('||', '|').replace('&&', '&').replace('!', '~') truth = satisfiable(exp, algorithm="dpll") truth_table = dict() for key, value in truth.items(): truth_table[key.name] = value if not truth_table: TobeDel.append(edge) else: self.buchi_graph.edges[edge]['truth'] = truth_table else: self.buchi_graph.edges[edge]['truth'] = '1' for edge in TobeDel: self.buchi_graph.remove_edge(edge[0], edge[1]) # print(self.buchi_graph.number_of_edges()) def InitialDelInfesEdge(self, orig_label): div_by_or = orig_label.split(') || (') for item in div_by_or: feas = True for excl in self.exclusion: # mutual exclusion term exist if excl[0] in item and excl[1] in item and '!{0}'.format(excl[0]) not in item and '!{0}'.format(excl[1]) not in item: feas = False break if not feas: item = item.strip('(').strip(')') item = '(' + item + ')' orig_label = orig_label.replace(' '+item+' ||', '').replace(item+' || ','').replace(' || '+item,'').replace(item,'') return orig_label def MinLen(self): """ search the shorest path from a node to another, weight = 1, i.e. # of state in the path :param buchi_graph: :return: dict of pairs of node : length of path """ min_qb_dict = dict() for node1 in self.buchi_graph.nodes(): for node2 in self.buchi_graph.nodes(): if node1 != node2 and 'accept' in node2: try: l, _ = nx.algorithms.single_source_dijkstra(self.buchi_graph, source=node1, target=node2) except nx.exception.NetworkXNoPath: l = np.inf min_qb_dict[(node1, node2)] = l elif node1 == node2 and 'accept' in node2: l = np.inf for succ in self.buchi_graph.succ[node1]: try: l0, _ = nx.algorithms.single_source_dijkstra(self.buchi_graph, source=succ, target=node1) except nx.exception.NetworkXNoPath: l0 = np.inf if l0 < l: l = l0 + 1 min_qb_dict[(node1, node2)] = l return min_qb_dict # def MinLen_Cost(self): # """ # search the shorest path from a node to another, weight = cost # :param buchi_graph: # :return: dict of pairs of node : length of path # """ # min_qb_dict = dict() # for node1 in self.buchi_graph.nodes(): # for node2 in self.buchi_graph.nodes(): # c = np.inf # if node1 != node2: # try: # path = nx.all_simple_paths(self.buchi_graph, source=node1, target=node2) # for i in range(len(path)-2): # word_init = self.buchi_graph.edges[(path[i], path[i+1])]['label'] # word_tg = self.buchi_graph.edges[(path[i+1], path[i+2])]['label'] # # calculate distance travelled from word_init to word_tg # t_s_b = True # # split label with || # label_init = word_init.split('||') # label_tg = word_tg.split('||') # for label in b_label: # t_s_b = True # # spit label with && # atomic_label = label.split('&&') # for a in atomic_label: # a = a.strip() # a = a.strip('(') # a = a.strip(')') # if a == '1': # continue # # whether ! in an atomic proposition # if '!' in a: # if a[1:] in x_label: # t_s_b = False # break # else: # if not a in x_label: # t_s_b = False # break # # either one of || holds # if t_s_b: # return t_s_b # except nx.exception.NetworkXNoPath: # c = np.inf # else: # c = 0 # min_qb_dict[(node1, node2)] = c # # return min_qb_dict def FeasAcpt(self, min_qb): """ delte infeasible final state :param buchi_graph: buchi automaton :param min_qb: dict of pairs of node : length of path """ accept = self.buchi_graph.graph['accept'] for acpt in accept: if min_qb[(self.buchi_graph.graph['init'][0], acpt)] == np.inf or min_qb[(acpt, acpt)] == np.inf: self.buchi_graph.graph['accept'].remove(acpt) def PutNotInside(self, str): """ put not inside the parenthesis !(p1 && p2) -> !p1 or !p2 :param str: old :return: new """ substr = re.findall("(!\(.*?\))", str) # ['!(p1 && p2)', '!(p4 && p5)'] for s in substr: oldstr = s.strip().strip('!').strip('(').strip(')') nstr = '' for ss in oldstr.split(): if '&&' in ss: nstr = nstr + ' or ' elif 'or' in ss: nstr = nstr + ' && ' else: nstr = nstr + '!' + ss str = str.replace(s, nstr) return str def label2sat(self): for edge in self.buchi_graph.edges(): label = self.buchi_graph.edges[edge]['label'] label = label.replace('||', '|').replace('&&', '&').replace('!', '~') exp1 = to_cnf(label) self.buchi_graph.edges[edge]['label'] = exp1
def plotly_lookml(G: DiGraph, color_map: list[str], plot_layout: str = "fdp") -> go.Figure: """Create an interactive plotly figure for the input `DiGraph` network Code source: https://plotly.com/python/network-graphs/ Args: G: the DiGraph() network for LookML nodes color_map: color names for each node in `G` plot_layout: layout of the nodes positions using Pydot and Graphviz (options: 'dot', 'twopi', 'fdp', 'sfdp', 'circo') Returns: the interactive plotly figure with the LookML network rendered """ # build layout (i.e. node coordinates / positions) pos = graphviz_layout(G, prog=plot_layout) edge_x = [] edge_y = [] for edge in G.edges(): x0, y0 = pos[edge[0]] x1, y1 = pos[edge[1]] edge_x.append(x0) edge_x.append(x1) edge_x.append(None) edge_y.append(y0) edge_y.append(y1) edge_y.append(None) edge_trace = go.Scatter( x=edge_x, y=edge_y, line=dict(width=0.5, color="#888"), hoverinfo="none", mode="lines", ) node_x = [] node_y = [] for node in G.nodes(): x, y = pos[node] node_x.append(x) node_y.append(y) node_trace = go.Scatter( x=node_x, y=node_y, mode="markers", hoverinfo="text", marker=dict( # colorscale options # 'Greys' | 'YlGnBu' | 'Greens' | 'YlOrRd' | 'Bluered' | 'RdBu' | # 'Reds' | 'Blues' | 'Picnic' | 'Rainbow' | 'Portland' | 'Jet' | # 'Hot' | 'Blackbody' | 'Earth' | 'Electric' | 'Viridis' | colorscale="YlGnBu", color=[], size=10, line_width=2, ), ) # Color node point node_text = [] for node, adjacencies in G.adjacency(): node_text.append(node) node_trace.marker.color = color_map node_trace.text = node_text def layout(): layout = go.Layout( title="LookML Content Relationships Network", titlefont_size=16, showlegend=False, hovermode="closest", margin=dict(b=20, l=5, r=5, t=40), xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), ) return layout # Create Network Graph fig = go.Figure(data=[edge_trace, node_trace], layout=layout()) fig.show() return fig
def load(fname): def clean_bool(string): if string == "0": return None else: return string def to_bool(string): if string == "1" or string == "True": return True elif string == "0" or string == "False": return False else: return string def to_float(string): if string == "None": return None try: return float(string) except: return string mode = "node0" nodes = [] edges = [] volatiles = set() outputs = None inputs = None named_ranges = {} infile = gzip.GzipFile(fname, 'r') for line in infile.read().splitlines(): if line == "====": mode = "node0" continue if line == "-----": cellmap_temp = {n.address(): n for n in nodes} Range = RangeFactory(cellmap_temp) mode = "node0" continue elif line == "edges": cellmap = {n.address(): n for n in nodes} mode = "edges" continue elif line == "outputs": mode = "outputs" continue elif line == "inputs": mode = "inputs" continue elif line == "named_ranges": mode = "named_ranges" continue if mode == "node0": [address, formula, python_expression, is_range, is_named_range, is_volatile, should_eval] = line.split(SEP) formula = clean_bool(formula) python_expression = clean_bool(python_expression) is_range = to_bool(is_range) is_named_range = to_bool(is_named_range) is_volatile = to_bool(is_volatile) should_eval = should_eval mode = "node1" elif mode == "node1": if is_range: reference = json.loads(line) if is_volatile else line # in order to be able to parse dicts vv = Range(reference) if is_volatile: if not is_named_range: address = vv.name volatiles.add(address) cell = Cell(address, None, vv, formula, is_range, is_named_range, should_eval) cell.python_expression = python_expression nodes.append(cell) else: value = to_bool(to_float(line)) cell = Cell(address, None, value, formula, is_range, is_named_range, should_eval) cell.python_expression = python_expression if formula: if 'OFFSET' in formula or 'INDEX' in formula: volatiles.add(address) cell.compile() nodes.append(cell) elif mode == "edges": source, target = line.split(SEP) edges.append((cellmap[source], cellmap[target])) elif mode == "outputs": outputs = line.split(SEP) elif mode == "inputs": inputs = line.split(SEP) elif mode == "named_ranges": k,v = line.split(SEP) named_ranges[k] = v G = DiGraph(data = edges) print "Graph loading done, %s nodes, %s edges, %s cellmap entries" % (len(G.nodes()),len(G.edges()),len(cellmap)) return (G, cellmap, named_ranges, volatiles, outputs, inputs)
def find_MECs(mdp, Sneg): #----implementation of Alg.47 P866 of Baier08---- print 'Remaining states size', len(Sneg) U = mdp.graph['U'] A = dict() for s in Sneg: A[s] = mdp.node[s]['act'].copy() if not A[s]: print "Isolated state" MEC = set() MECnew = set() MECnew.add(frozenset(Sneg)) #---- k = 0 while MEC != MECnew: print "<============iteration %s============>" %k k +=1 MEC = MECnew MECnew = set() print "MEC size: %s" %len(MEC) print "MECnew size: %s" %len(MECnew) for T in MEC: R = set() T_temp = set(T) simple_digraph = DiGraph() for s_f in T_temp: if s_f not in simple_digraph: simple_digraph.add_node(s_f) for s_t in mdp.successors_iter(s_f): if s_t in T_temp: simple_digraph.add_edge(s_f,s_t) print "SubGraph of one MEC: %s states and %s edges" %(str(len(simple_digraph.nodes())), str(len(simple_digraph.edges()))) Sccs = strongly_connected_component_subgraphs(simple_digraph) i = 0 for Scc in Sccs: i += 1 if (len(Scc.edges())>=1): for s in Scc.nodes(): U_to_remove = set() for u in A[s]: for t in mdp.successors_iter(s): if ((u in mdp.edge[s][t]['prop'].keys()) and (t not in Scc.nodes())): U_to_remove.add(u) A[s].difference_update(U_to_remove) if not A[s]: R.add(s) while R: s = R.pop() T_temp.remove(s) for f in mdp.predecessors(s): if f in T_temp: A[f].difference_update(set(mdp.edge[f][s]['prop'].keys())) if not A[f]: R.add(f) New_Sccs = strongly_connected_component_subgraphs(simple_digraph) j = 0 for Scc in New_Sccs: j += 1 if (len(Scc.edges()) >= 1): common = set(Scc.nodes()).intersection(T_temp) if common: MECnew.add(frozenset(common)) #--------------- print 'Final MEC and MECnew size:', len(MEC) return MEC, A
def find_SCCs(mdp, Sneg): #----simply find strongly connected components---- print 'Remaining states size', len(Sneg) SCC = set() simple_digraph = DiGraph() A = dict() for s in mdp.nodes(): A[s] = mdp.node[s]['act'].copy() for s_f in Sneg: if s_f not in simple_digraph: simple_digraph.add_node(s_f) for s_t in mdp.successors_iter(s_f): if s_t in Sneg: simple_digraph.add_edge(s_f,s_t) print "SubGraph of one Sf: %s states and %s edges" %(str(len(simple_digraph.nodes())), str(len(simple_digraph.edges()))) sccs = strongly_connected_component_subgraphs(simple_digraph) for scc in sccs: SCC.add(frozenset(scc.nodes())) return SCC, A