def target(): print 'Thread started' try: print self.cmd retargetPath = "/home/wyang/workspace/CommandLine/epicc/RetargetDir/Jifake/9440bb3da5e1ad862f357248b5da0c59dc7fc96b.apk/" outputfile = os.path.join(retargetPath,"output.log") errfile = os.path.join(retargetPath,"erroutput.log") open(outputfile, 'w').close() open(errfile, 'w').close() self.process = subprocess.Popen(self.cmd, shell=True, stdout = file(outputfile, 'w+'), stderr = file(errfile, 'w+')) # (self.output, self.error) = self.process.communicate() # self.status = self.process.returncode print self.output #"Out:'%s'" % print self.error #"Err:'%s'" % print 'Thread finished' if(os.stat(errfile)[6] == 0): a=open(outputfile,'rb') lines = a.readlines() last = lines[-1] print last if "The following ICC values were found:" not in last: jsonfile = os.path.join(retargetPath,"intent.json") ret = epicc_parser.parse_epicc(outputfile) ret_json = json.dumps(ret, sort_keys=True, indent=4, separators=(',', ': ')) jsonfo = open(jsonfile,"wb") jsonfo.write(ret_json) except: self.error = traceback.format_exc() self.status = -1 print self.error
a=open(outputfile,'rb') lines = a.readlines() flag = False if (command.status == 0): for line in lines: if "The following ICC values were found:" in line: flag = True if flag == False: myfile.write(dir+"_"+appName+": "+str(elapsed)+' Error! \n') else: last = lines[-1] print last if "The following ICC values were found:" not in last: myfile.write(dir+"_"+appName+": "+str(elapsed)+' Have ICC! \n') jsonfile = os.path.join(retargetPath,"intent.json") ret = epicc_parser.parse_epicc(outputfile) ret_json = json.dumps(ret, sort_keys=True, indent=4, separators=(',', ': ')) jsonfo = open(jsonfile,"wb") jsonfo.write(ret_json) #jsonfo.write(ret) else: myfile.write(dir+"_"+appName+": "+str(elapsed)+' No ICC! \n') else: myfile.write(dir+ "_" + appName+": "+str(elapsed)+ ' timeout!!'+ '\n') else: myfile.write(dir+ "_" + appName+": "+str(elapsed)+ ' no outputfile!!'+'\n') print 'finish ICC of '+ appName
def __init__(self, arguments): self.unsound = False self.internalflows = [] self.flowMapping = dict() self.sources = set() self.intentResultIds = dict() self.errorcount = 0 #Graph data structures self.graph = Graph() self.graph.load() epiccDataDict = dict() manifestDataDict = dict() flowdroidDataDict = dict() if not(sys.version_info[0] == 2 and sys.version_info[1] >= 7): self.die("Incompatible version of Python! This script needs Python 2.7.") if len(arguments[1:]) == 0: sys.stderr.write( ("Usage: %s [OPTIONS] [FILES]\n" % (sys.argv[0],)) + "Files: For each app, should include manifest, epicc, and flowdroid output.\n" + #" To override package name, use 'pkg:filename' (UNTESTED).\n" + "Options: \n" + " --unsound select the precision of the intent matching\n" ) self.die("") fileNames = [] args = iter(arguments[1:]) # parse arguments to a list of file names, and set the parameters into fields for arg in args: try: if arg == "--unsound": self.unsound = True else: fileNames.append(arg) except StopIteration: self.die("Option '%s' expects an argument." % (arg,)) fileCount = len(fileNames) currentCount = 0 # Get the package name of all apps, and parse the file data into dictionaries for filename in fileNames: possible_package_name = filename.replace(".epicc", "") possible_package_name = possible_package_name.replace(".fd.xml", "") possible_package_name = possible_package_name.replace(".manifest.xml", "") if possible_package_name in self.graph.processedApps: continue currentCount += 1 sys.stdout.write("\rLoading file " + str(currentCount) + " of " + str(fileCount)) sys.stdout.flush() # if is epicc file try: if filename.endswith(".epicc"): (pkg_name, epicc) = parse_epicc(filename, as_dict=True) if pkg_name in self.graph.processedApps: continue epiccDataDict[pkg_name] = epicc #if is flowdroid file elif filename.endswith(".fd.xml"): tree = ET.parse(filename) root = tree.getroot() pkg_name = root.attrib['package'] if pkg_name in self.graph.processedApps: continue flowdroidDataDict[pkg_name] = root #if is manifest file elif filename.endswith(".manifest.xml"): xml = ET.parse(filename) pkg_name = xml.find('.').attrib['package'] if pkg_name in self.graph.processedApps: continue manifestDataDict[pkg_name] = xml else: self.die("wrong file format :-(") except: continue print("\n") keyCount = len(manifestDataDict.keys()) currentKey = 0 # combine the different data streams into a usable data structure for packageName in manifestDataDict.keys(): currentKey += 1 flowString = "\rCreating flows for " + str(currentKey) + " of " + str(keyCount) + " in package " + packageName sys.stdout.write(flowString + "(1/3) ") #hacks, so we know that we always overwrite the previous text sys.stdout.flush() try: flowDroidFlows = self.get_flows(flowdroidDataDict[packageName]) epiccData = epiccDataDict[packageName] manifestData = manifestDataDict[packageName] self.graph.processedApps.add(packageName) except KeyError: #Error on parsing data in package print "\nError on parsing: " + packageName self.errorcount += 1 manifestDataDict.pop(packageName, None) flowdroidDataDict.pop(packageName, None) epiccDataDict.pop(packageName, None) continue #lets get the intent filters for the app: intentFilters = self.read_intent_filters_from_manifest(manifestData, packageName) flowCount = len(flowDroidFlows) currentFlow = 0 # match internal flows with intent info from Epicc for flow in flowDroidFlows[:]: #To Alex: the [:] syntax returns a copy of the list. currentFlow += 1 sys.stdout.write(flowString + " (2/3) (Flow " + str(currentFlow) + " of " + str(flowCount) + ")") sys.stdout.flush() if isinstance(flow.source, FlowdroidElement) and flow.source.type == INTENT_RESULT and flow.source.intentId == None: try: epicc = epiccData[self.intentResultIds[flow.app]][0] except Exception: flowDroidFlows.remove(flow) epicc = None if not epicc: continue action = [] category = [] mimeTypes = [] data = "" matchAll = False matchingClass = None #epicc is a dictionary that contains a key for each of the important properties. These entries in epicc is being converted to proper IntentDefinition objects. if 'Action' in epicc: actionString = epicc["Action"] if actionString.startswith("["): action = actionString[1:-1].split(",") else: action = [actionString] if 'Type' in epicc: typeString = epicc["Type"] if typeString.startswith("["): mimeTypes = typeString[1:-1].split(",") else: mimeTypes = [typeString] if 'Categories' in epicc: categoryString = epicc["Categories"] #if categoryString.startswith("["): #category = categoryString[1:-1].split(",") #else: category = categoryString if 'Data' in epicc: data = epicc["Data"] if 'Top' in epicc and epicc["Top"]: matchAll = True if 'Class' in epicc: matchingClass = epicc["Class"].replace("/", ".") intentDefinition = IntentDefinition(action, category, mimeTypes, data, "Activity") flow.source = IntentResult(intentDefinition=intentDefinition,app=flow.app) if isinstance(flow.sink, FlowdroidElement) and (flow.sink.intentId == None or flow.sink.intentId.startswith("newField_")): epicc=None # reinitialize if flow.sink.intentId == None: #flow.sink = IntentResult(IntentDefinition([],[],[], "", ""), flow.app) flowDroidFlows.remove(flow) # I have a case where epiccData is not None, flow.sink.intentID is None continue else: try: epicc = epiccData[flow.sink.intentId][0] except Exception: flowDroidFlows.remove(flow) continue if epicc==None: epicc=dict() action = [] category = [] mimeTypes = [] data = "" matchAll = False matchingClass = None #epicc is a dictionary that contains a key for each of the important properties. These entries in epicc is being converted to proper IntentDefinition objects. if 'Action' in epicc: actionString = epicc["Action"] if actionString.startswith("["): action = actionString[1:-1].split(",") else: action = [actionString] if 'Type' in epicc: typeString = epicc["Type"] if typeString.startswith("["): mimeTypes = typeString[1:-1].split(",") else: mimeTypes = [typeString] if 'Categories' in epicc: categoryString = epicc["Categories"] #if categoryString.startswith("["): #category = categoryString[1:-1].split(",") #else: category = categoryString if 'Data' in epicc: data = epicc["Data"] if 'Top' in epicc and epicc["Top"]: matchAll = True if 'Class' in epicc: matchingClass = epicc["Class"].replace("/", ".") intentDefinition = IntentDefinition(action, category, mimeTypes, data, flow.sink.componentType) intentDefinition.matchAll = matchAll intentDefinition.matchingClass = matchingClass if flow.sink.type == INTENT: flow.sink = Intent(intentDefinition, flow.app) else: flow.sink = IntentResult(intentDefinition, flow.app) #Match internal flows with intent filters from the manifest appflows = set() for flow in flowDroidFlows: _flow = flow.toFlow() if isinstance(_flow.source, FlowdroidElement) and _flow.source.intentId == None: component = _flow.source.component componentType = _flow.source.componentType componentIntentFilters = intentFilters.get(component, []) if(len(componentIntentFilters) == 0): definition = IntentDefinition([], [], [], "", componentType) definition.matchingClass = component _flow.source = IntentFilter(definition, _flow.app, 0) appflows.add(_flow) for intentFilter in componentIntentFilters: newFlow = copy.deepcopy(_flow) newFlow.source = intentFilter appflows.add(newFlow) else: appflows.add(_flow) ''' The following matches all flows within a single app to connect explicit intents together to single flows. We are not interested in explicit intents as these only happens within an app, but we are interested in linking together sources with implicit intents, which might happen across several internal components. TODO: Check for Intent Results ''' stillWorking = True permutations = 0 tracker = set() oldFlows = set() for e in appflows: str_e = str(e) if str_e not in tracker: tracker.add(str_e) oldFlows.add(e) seenbefore = set() while stillWorking: permutations += 1 stillWorking = False newFlows = set() flowCount = 0 for flow1 in oldFlows: flowCount += 1 if flowCount % 100 == 0: sys.stdout.write(flowString + "(3/3) (Matching internal intents, permutation: " + str(permutations) + ", Flows: " + str(flowCount) + ", seenBefore: " + str(len(seenbefore)) + ")") sys.stdout.flush() added = False if (isinstance(flow1.sink, Intent) or isinstance(flow1.sink, IntentResult)) and flow1.sink.intentDefinition.matchingClass != None: for flow2 in oldFlows: if isinstance(flow2.source, IntentFilter): if(flow1.sink.intentDefinition.matchingClass == flow2.source.intentDefinition.matchingClass): newFlow = Flow(flow1.source, flow2.sink, flow1.app) if str(newFlow) not in seenbefore: newFlows.add(newFlow) seenbefore.add(str(newFlow)) stillWorking = True added = True if not added: newFlows.add(flow1) if stillWorking: oldFlows = newFlows appflows = oldFlows ''' This part of the algorithm runs through all flows and puts the data in several datastructures used to build the graph. All elements are converted to hash strings, and put in hashToObjectMapping, with the hash as a key and the object as a value. Intents are added to the intents dictionary, with the intent hash as the key and a set of app names as the result. IntentFilters are added to filterToSinkMapping with the filter hash as the key and a set of sinks as the value. IntentResults are added to the onResult, with the IntentResult hash as the key and a set of app names as the value. ''' for flow in appflows: if "Explicit Intent (" in str(flow.source) or ("Explicit Intent (" in str(flow.sink) and not isinstance(flow.sink, IntentResult)): continue if "FlowdroidElement" in str(flow.source) or "FlowdroidElement" in str(flow.sink): continue sourceHash = flow.source.get_md5hash() sinkHash = flow.sink.get_md5hash() if sourceHash not in self.graph.hashToObjectMapping: self.graph.hashToObjectMapping[sourceHash] = flow.source if sinkHash not in self.graph.hashToObjectMapping: self.graph.hashToObjectMapping[sinkHash] = flow.sink if isinstance(flow.sink, IntentResult) and not isinstance(flow.source, IntentFilter): intent = Intent(intentDefinition = flow.sink.intentDefinition, app = flow.sink.app) intentHash = intent.get_md5hash() if "android.intent.action.MAIN" not in flow.sink.intentDefinition.actions: if intentHash not in self.graph.intents: self.graph.intents[intentHash] = set() self.graph.intents[intentHash].add(flow.source.app) self.graph.hashToObjectMapping[intentHash] = intent src = flow.source internalflow = Flow(source = src, sink = intent, app=flow.app) self.internalflows.append(internalflow) continue if isinstance(flow.sink, Intent): intentHash = flow.sink.get_md5hash() if "android.intent.action.MAIN" not in flow.sink.intentDefinition.actions: if intentHash not in self.graph.intents: self.graph.intents[sinkHash] = set() self.graph.intents[sinkHash].add(flow.source.app) if not isinstance(flow.source, IntentFilter): self.internalflows.append(flow) if isinstance(flow.source, IntentFilter): if sourceHash not in self.graph.filterToSinkMapping: self.graph.filterToSinkMapping[sourceHash] = set() self.graph.filterToSinkMapping[sourceHash].add(sinkHash) elif isinstance(flow.source, IntentResult): if sourceHash not in self.graph.onResult: self.graph.onResult[sourceHash] = set() self.graph.onResult[sourceHash].add(flow.source.app) self.internalflows.append(flow) elif isinstance(flow.source, Source) and isinstance(flow.sink, Sink): self.internalflows.append(flow) print("\n")
def main(): if not(sys.version_info[0] == 2 and sys.version_info[1] >= 7): die("Incompatible version of Python! This script needs Python 2.7.") glo.manifest = OrderedDict() # diccionario ordenado vacio flow_lists= [] flow_files = [] all_filenames = [] arg_iter = iter(sys.argv[1:]) gv_base = None gv_out = None gv_legend = None cl_out = None cl_out_from_file = True glo.is_quiet = False if len(sys.argv[1:]) == 0: sys.stderr.write( ("Usage: %s [OPTIONS] [FILES]\n" % (sys.argv[0],)) + "Files: For each app, should include manifest, epicc, and flowdroid output.\n" + #" To override package name, use 'pkg:filename' (UNTESTED).\n" + "Options: \n" + " --gv graphfile: Generates graphfile.{gv,txt,pdf}\n" + " --cl jsonfile: Writes the Flows and taint solution in JSON format and checks that flows comply with security levels\n" " --quiet: Don't write as much to stdout\n" ) sys.exit(1) while True: try: arg = arg_iter.next() except StopIteration: break try: if arg == "--unsound": glo.unsound = True elif arg == "--quiet": glo.is_quiet = True elif arg == "--gv": gv_base = arg_iter.next() if not(re.match("^[A-Za-z0-9_.-/]+$", gv_base)): die("gv filename contains bad characters") assert(not (gv_base.endswith(".gv"))) gv_out = open(gv_base + ".gv", "w") gv_legend = open(gv_base + ".txt", "w") elif arg == "--check_levels_file": # Added cl_out_filename = arg_iter.next() cl_out = open(cl_out_filename, "w") # Writes the Flows and taint solution in JSON format elif arg == "--check_levels_gui": # Added cl_out_from_file = False else: all_filenames.append(arg) except StopIteration: die("Option '%s' expects an argument." % (arg,)) def check_epicc(filename, pkg_name, epicc): def die_epicc(): sys.stderr.write(traceback.format_exc()) die("Aborted due to error in parsing " + filename) try: assert(len(pkg_name) > 0) assert(isinstance(epicc, dict)) warned_unknown = False for (intent_id, v) in epicc.iteritems(): assert(isinstance(intent_id, str)) assert(isinstance(v, list)) if intent_id == "*": sys.stderr.write("Warning: Missing IntentID in %s\n" % (filename,)) warned_unknown = True else: if (intent_id == BCAST_ID): continue else: assert(intent_id.startswith("newField_")) for x in v: try: assert(isinstance(x, dict)) except AssertionError as e: sys.stderr.write("\nx: %r\n" % (x,)) die_epicc() except AssertionError as e: die_epicc() for filename in all_filenames: pkg_rename = None if ":" in filename: [pkg_rename, filename] = filename.split(":") root = try_read_manifest_file(filename) if root != None: pkg_name = pkg_rename or root.find('.').attrib['package'] glo.manifest[pkg_name] = root glo.filter[pkg_name] = read_intent_filters_from_manifest(root) if len(all_filenames) == 1: pprint(dict(glo.filter[pkg_name])) elif filename.endswith(".xml"): flow_files.append(filename) elif filename.endswith(".epicc"): (pkg_name, epicc) = parse_epicc(filename, as_dict=True) if pkg_rename: pkg_name = pkg_rename check_epicc(filename, pkg_name, epicc) glo.epicc[pkg_name] = epicc if len(all_filenames) == 1: print "EPICC info:" pprint(epicc) else: print("Unknown file type: " + filename) # Creates levels chequer check_levels = Check_levels(cl_out_from_file) for filename in flow_files: pkg_rename = None if ":" in filename: [pkg_rename, filename] = filename.split(":") tree = ET.parse(filename) root = tree.getroot() pkg_name = pkg_rename or root.attrib['package'] flows = find_flows(root, check_levels) # Solve Flows def analize(): def num_intents_in_flow(flow): return sum((type(s) in [Intent, IntentResult]) for s in [flow.src, flow.sink]) for filename in flow_files: pkg_rename = None if ":" in filename: [pkg_rename, filename] = filename.split(":") tree = ET.parse(filename) root = tree.getroot() pkg_name = pkg_rename or root.attrib['package'] flows = find_flows(root, check_levels) glo.flows[pkg_name] = flows flow_lists.append(flows) flows = flatten(flow_lists) flows = match_flows(flows) solution = solve_flows(flows) sol_src = {} for (entity, taint_set) in solution.iteritems(): sol_src[entity] = OrderedSet(x for x in taint_set if (type(x)==Sink or type(x)==Src)) if not glo.is_quiet: for num_intents in [0,1,2]: print("---- Flows with %i intent(s) ---------------------------------" % num_intents) pprint(filter(lambda x: num_intents_in_flow(x)==num_intents, flows)) print("--------------------------------------------------------------") print("--------------------------------------------------------------") sols = [[],[],[]] for (entity, taint_set) in sol_src.iteritems(): if type(entity) == Intent: ix = 0 elif type(entity) == IntentResult: ix = 1 elif type(entity)==Sink: ix = 2 else: assert(type(entity)==Src) continue sols[ix].append((entity, taint_set)) for ix in [0,1,2]: print("--------------------") for (entity, taint_set) in sols[ix]: sys.stdout.write("### '%s': ###\n" % (entity,)) pprint(list(taint_set)) print("--------------------") return sol_src def stringize_intents(obj): #?? t = type(obj) if t in [str, int, type(None), Sink, Src]: return str(obj) elif t in [set, OrderedSet, list]: return list(stringize_intents(x) for x in obj) elif t in [dict]: return t((stringize_intents(k), stringize_intents(v)) for (k,v) in obj.iteritems()) elif t in [Intent, IntentResult]: return str(obj) elif t == Flow: narf = list(stringize_intents(x) for x in obj) return Flow(*narf) else: assert(0) return obj def main_window(levels, entities, analize_leves, add_exception): import gettext import wx gettext.install("app") app = wx.PySimpleApp(0) wx.InitAllImageHandlers() frame_1 = MyFrame(None, wx.ID_ANY, "") entities.remove('BUNDLE') frame_1.create(levels, entities, analize_leves, add_exception) app.SetTopWindow(frame_1) frame_1.Show() app.MainLoop() def out_window(security_probl): import wx wx.InitAllImageHandlers() dialog_1 = MyDialog(None, wx.ID_ANY, "") dialog_1.create(security_probl) dialog_1.Show() def analize_leves(tuples): if tuples != None: check_levels.selection_assign_levels(tuples) sol_src = analize() import json cl_dict = stringize_intents({'Taints': sol_src}) cl_str = json.dumps(cl_dict, sort_keys=True, indent=4, separators=(',', ': ')) security_probl = check_levels.check_levels(cl_dict) if tuples != None: out_window(security_probl) else: cl_out.write(security_probl) if cl_out_from_file: analize_leves(None) else: main_window(check_levels.get_levels_order(), check_levels.get_vars_order(), analize_leves, check_levels.add_exception)