def target():
print 'Thread started'
try:
print self.cmd
retargetPath = "/home/wyang/workspace/CommandLine/epicc/RetargetDir/Jifake/9440bb3da5e1ad862f357248b5da0c59dc7fc96b.apk/"
outputfile = os.path.join(retargetPath,"output.log")
errfile = os.path.join(retargetPath,"erroutput.log")
open(outputfile, 'w').close()
open(errfile, 'w').close()
self.process = subprocess.Popen(self.cmd, shell=True, stdout = file(outputfile, 'w+'), stderr = file(errfile, 'w+')) #
(self.output, self.error) = self.process.communicate() #
self.status = self.process.returncode
print self.output #"Out:'%s'" %
print self.error #"Err:'%s'" %
print 'Thread finished'
if(os.stat(errfile)[6] == 0):
a=open(outputfile,'rb')
lines = a.readlines()
last = lines[-1]
print last
if "The following ICC values were found:" not in last:
jsonfile = os.path.join(retargetPath,"intent.json")
ret = epicc_parser.parse_epicc(outputfile)
ret_json = json.dumps(ret, sort_keys=True, indent=4, separators=(',', ': '))
jsonfo = open(jsonfile,"wb")
jsonfo.write(ret_json)
except:
self.error = traceback.format_exc()
self.status = -1
print self.error
a=open(outputfile,'rb')
lines = a.readlines()
flag = False
if (command.status == 0):
for line in lines:
if "The following ICC values were found:" in line:
flag = True
if flag == False:
myfile.write(dir+"_"+appName+": "+str(elapsed)+' Error! \n')
else:
last = lines[-1]
print last
if "The following ICC values were found:" not in last:
myfile.write(dir+"_"+appName+": "+str(elapsed)+' Have ICC! \n')
jsonfile = os.path.join(retargetPath,"intent.json")
ret = epicc_parser.parse_epicc(outputfile)
ret_json = json.dumps(ret, sort_keys=True, indent=4, separators=(',', ': '))
jsonfo = open(jsonfile,"wb")
jsonfo.write(ret_json)
#jsonfo.write(ret)
else:
myfile.write(dir+"_"+appName+": "+str(elapsed)+' No ICC! \n')
else:
myfile.write(dir+ "_" + appName+": "+str(elapsed)+ ' timeout!!'+ '\n')
else:
myfile.write(dir+ "_" + appName+": "+str(elapsed)+ ' no outputfile!!'+'\n')
print 'finish ICC of '+ appName
def __init__(self, arguments):
self.unsound = False
self.internalflows = []
self.flowMapping = dict()
self.sources = set()
self.intentResultIds = dict()
self.errorcount = 0
#Graph data structures
self.graph = Graph()
self.graph.load()
epiccDataDict = dict()
manifestDataDict = dict()
flowdroidDataDict = dict()
if not(sys.version_info[0] == 2 and sys.version_info[1] >= 7):
self.die("Incompatible version of Python! This script needs Python 2.7.")
if len(arguments[1:]) == 0:
sys.stderr.write(
("Usage: %s [OPTIONS] [FILES]\n" % (sys.argv[0],)) +
"Files: For each app, should include manifest, epicc, and flowdroid output.\n" +
#" To override package name, use 'pkg:filename' (UNTESTED).\n" +
"Options: \n" +
" --unsound select the precision of the intent matching\n"
)
self.die("")
fileNames = []
args = iter(arguments[1:])
# parse arguments to a list of file names, and set the parameters into fields
for arg in args:
try:
if arg == "--unsound":
self.unsound = True
else:
fileNames.append(arg)
except StopIteration:
self.die("Option '%s' expects an argument." % (arg,))
fileCount = len(fileNames)
currentCount = 0
# Get the package name of all apps, and parse the file data into dictionaries
for filename in fileNames:
possible_package_name = filename.replace(".epicc", "")
possible_package_name = possible_package_name.replace(".fd.xml", "")
possible_package_name = possible_package_name.replace(".manifest.xml", "")
if possible_package_name in self.graph.processedApps:
continue
currentCount += 1
sys.stdout.write("\rLoading file " + str(currentCount) + " of " + str(fileCount))
sys.stdout.flush()
# if is epicc file
try:
if filename.endswith(".epicc"):
(pkg_name, epicc) = parse_epicc(filename, as_dict=True)
if pkg_name in self.graph.processedApps:
continue
epiccDataDict[pkg_name] = epicc
#if is flowdroid file
elif filename.endswith(".fd.xml"):
tree = ET.parse(filename)
root = tree.getroot()
pkg_name = root.attrib['package']
if pkg_name in self.graph.processedApps:
continue
flowdroidDataDict[pkg_name] = root
#if is manifest file
elif filename.endswith(".manifest.xml"):
xml = ET.parse(filename)
pkg_name = xml.find('.').attrib['package']
if pkg_name in self.graph.processedApps:
continue
manifestDataDict[pkg_name] = xml
else:
self.die("wrong file format :-(")
except:
continue
print("\n")
keyCount = len(manifestDataDict.keys())
currentKey = 0
# combine the different data streams into a usable data structure
for packageName in manifestDataDict.keys():
currentKey += 1
flowString = "\rCreating flows for " + str(currentKey) + " of " + str(keyCount) + " in package " + packageName
sys.stdout.write(flowString + "(1/3) ") #hacks, so we know that we always overwrite the previous text
sys.stdout.flush()
try:
flowDroidFlows = self.get_flows(flowdroidDataDict[packageName])
epiccData = epiccDataDict[packageName]
manifestData = manifestDataDict[packageName]
self.graph.processedApps.add(packageName)
except KeyError:
#Error on parsing data in package
print "\nError on parsing: " + packageName
self.errorcount += 1
manifestDataDict.pop(packageName, None)
flowdroidDataDict.pop(packageName, None)
epiccDataDict.pop(packageName, None)
continue
#lets get the intent filters for the app:
intentFilters = self.read_intent_filters_from_manifest(manifestData, packageName)
flowCount = len(flowDroidFlows)
currentFlow = 0
# match internal flows with intent info from Epicc
for flow in flowDroidFlows[:]: #To Alex: the [:] syntax returns a copy of the list.
currentFlow += 1
sys.stdout.write(flowString + " (2/3) (Flow " + str(currentFlow) + " of " + str(flowCount) + ")")
sys.stdout.flush()
if isinstance(flow.source, FlowdroidElement) and flow.source.type == INTENT_RESULT and flow.source.intentId == None:
try:
epicc = epiccData[self.intentResultIds[flow.app]][0]
except Exception:
flowDroidFlows.remove(flow)
epicc = None
if not epicc:
continue
action = []
category = []
mimeTypes = []
data = ""
matchAll = False
matchingClass = None
#epicc is a dictionary that contains a key for each of the important properties. These entries in epicc is being converted to proper IntentDefinition objects.
if 'Action' in epicc:
actionString = epicc["Action"]
if actionString.startswith("["):
action = actionString[1:-1].split(",")
else:
action = [actionString]
if 'Type' in epicc:
typeString = epicc["Type"]
if typeString.startswith("["):
mimeTypes = typeString[1:-1].split(",")
else:
mimeTypes = [typeString]
if 'Categories' in epicc:
categoryString = epicc["Categories"]
#if categoryString.startswith("["):
#category = categoryString[1:-1].split(",")
#else:
category = categoryString
if 'Data' in epicc:
data = epicc["Data"]
if 'Top' in epicc and epicc["Top"]:
matchAll = True
if 'Class' in epicc:
matchingClass = epicc["Class"].replace("/", ".")
intentDefinition = IntentDefinition(action, category, mimeTypes, data, "Activity")
flow.source = IntentResult(intentDefinition=intentDefinition,app=flow.app)
if isinstance(flow.sink, FlowdroidElement) and (flow.sink.intentId == None or flow.sink.intentId.startswith("newField_")):
epicc=None # reinitialize
if flow.sink.intentId == None:
#flow.sink = IntentResult(IntentDefinition([],[],[], "", ""), flow.app)
flowDroidFlows.remove(flow) # I have a case where epiccData is not None, flow.sink.intentID is None
continue
else:
try:
epicc = epiccData[flow.sink.intentId][0]
except Exception:
flowDroidFlows.remove(flow)
continue
if epicc==None: epicc=dict()
action = []
category = []
mimeTypes = []
data = ""
matchAll = False
matchingClass = None
#epicc is a dictionary that contains a key for each of the important properties. These entries in epicc is being converted to proper IntentDefinition objects.
if 'Action' in epicc:
actionString = epicc["Action"]
if actionString.startswith("["):
action = actionString[1:-1].split(",")
else:
action = [actionString]
if 'Type' in epicc:
typeString = epicc["Type"]
if typeString.startswith("["):
mimeTypes = typeString[1:-1].split(",")
else:
mimeTypes = [typeString]
if 'Categories' in epicc:
categoryString = epicc["Categories"]
#if categoryString.startswith("["):
#category = categoryString[1:-1].split(",")
#else:
category = categoryString
if 'Data' in epicc:
data = epicc["Data"]
if 'Top' in epicc and epicc["Top"]:
matchAll = True
if 'Class' in epicc:
matchingClass = epicc["Class"].replace("/", ".")
intentDefinition = IntentDefinition(action, category, mimeTypes, data, flow.sink.componentType)
intentDefinition.matchAll = matchAll
intentDefinition.matchingClass = matchingClass
if flow.sink.type == INTENT:
flow.sink = Intent(intentDefinition, flow.app)
else:
flow.sink = IntentResult(intentDefinition, flow.app)
#Match internal flows with intent filters from the manifest
appflows = set()
for flow in flowDroidFlows:
_flow = flow.toFlow()
if isinstance(_flow.source, FlowdroidElement) and _flow.source.intentId == None:
component = _flow.source.component
componentType = _flow.source.componentType
componentIntentFilters = intentFilters.get(component, [])
if(len(componentIntentFilters) == 0):
definition = IntentDefinition([], [], [], "", componentType)
definition.matchingClass = component
_flow.source = IntentFilter(definition, _flow.app, 0)
appflows.add(_flow)
for intentFilter in componentIntentFilters:
newFlow = copy.deepcopy(_flow)
newFlow.source = intentFilter
appflows.add(newFlow)
else:
appflows.add(_flow)
'''
The following matches all flows within a single app to connect explicit intents together to single flows.
We are not interested in explicit intents as these only happens within an app, but we are interested in linking
together sources with implicit intents, which might happen across several internal components.
TODO: Check for Intent Results
'''
stillWorking = True
permutations = 0
tracker = set()
oldFlows = set()
for e in appflows:
str_e = str(e)
if str_e not in tracker:
tracker.add(str_e)
oldFlows.add(e)
seenbefore = set()
while stillWorking:
permutations += 1
stillWorking = False
newFlows = set()
flowCount = 0
for flow1 in oldFlows:
flowCount += 1
if flowCount % 100 == 0:
sys.stdout.write(flowString + "(3/3) (Matching internal intents, permutation: " + str(permutations) + ", Flows: " + str(flowCount) + ", seenBefore: " + str(len(seenbefore)) + ")")
sys.stdout.flush()
added = False
if (isinstance(flow1.sink, Intent) or isinstance(flow1.sink, IntentResult)) and flow1.sink.intentDefinition.matchingClass != None:
for flow2 in oldFlows:
if isinstance(flow2.source, IntentFilter):
if(flow1.sink.intentDefinition.matchingClass == flow2.source.intentDefinition.matchingClass):
newFlow = Flow(flow1.source, flow2.sink, flow1.app)
if str(newFlow) not in seenbefore:
newFlows.add(newFlow)
seenbefore.add(str(newFlow))
stillWorking = True
added = True
if not added:
newFlows.add(flow1)
if stillWorking:
oldFlows = newFlows
appflows = oldFlows
'''
This part of the algorithm runs through all flows and puts the data in several datastructures used to build the graph.
All elements are converted to hash strings, and put in hashToObjectMapping, with the hash as a key and the object as a value.
Intents are added to the intents dictionary, with the intent hash as the key and a set of app names as the result.
IntentFilters are added to filterToSinkMapping with the filter hash as the key and a set of sinks as the value.
IntentResults are added to the onResult, with the IntentResult hash as the key and a set of app names as the value.
'''
for flow in appflows:
if "Explicit Intent (" in str(flow.source) or ("Explicit Intent (" in str(flow.sink) and not isinstance(flow.sink, IntentResult)):
continue
if "FlowdroidElement" in str(flow.source) or "FlowdroidElement" in str(flow.sink):
continue
sourceHash = flow.source.get_md5hash()
sinkHash = flow.sink.get_md5hash()
if sourceHash not in self.graph.hashToObjectMapping:
self.graph.hashToObjectMapping[sourceHash] = flow.source
if sinkHash not in self.graph.hashToObjectMapping:
self.graph.hashToObjectMapping[sinkHash] = flow.sink
if isinstance(flow.sink, IntentResult) and not isinstance(flow.source, IntentFilter):
intent = Intent(intentDefinition = flow.sink.intentDefinition, app = flow.sink.app)
intentHash = intent.get_md5hash()
if "android.intent.action.MAIN" not in flow.sink.intentDefinition.actions:
if intentHash not in self.graph.intents:
self.graph.intents[intentHash] = set()
self.graph.intents[intentHash].add(flow.source.app)
self.graph.hashToObjectMapping[intentHash] = intent
src = flow.source
internalflow = Flow(source = src, sink = intent, app=flow.app)
self.internalflows.append(internalflow)
continue
if isinstance(flow.sink, Intent):
intentHash = flow.sink.get_md5hash()
if "android.intent.action.MAIN" not in flow.sink.intentDefinition.actions:
if intentHash not in self.graph.intents:
self.graph.intents[sinkHash] = set()
self.graph.intents[sinkHash].add(flow.source.app)
if not isinstance(flow.source, IntentFilter):
self.internalflows.append(flow)
if isinstance(flow.source, IntentFilter):
if sourceHash not in self.graph.filterToSinkMapping:
self.graph.filterToSinkMapping[sourceHash] = set()
self.graph.filterToSinkMapping[sourceHash].add(sinkHash)
elif isinstance(flow.source, IntentResult):
if sourceHash not in self.graph.onResult:
self.graph.onResult[sourceHash] = set()
self.graph.onResult[sourceHash].add(flow.source.app)
self.internalflows.append(flow)
elif isinstance(flow.source, Source) and isinstance(flow.sink, Sink):
self.internalflows.append(flow)
print("\n")
def main():
if not(sys.version_info[0] == 2 and sys.version_info[1] >= 7):
die("Incompatible version of Python! This script needs Python 2.7.")
glo.manifest = OrderedDict() # diccionario ordenado vacio
flow_lists= []
flow_files = []
all_filenames = []
arg_iter = iter(sys.argv[1:])
gv_base = None
gv_out = None
gv_legend = None
cl_out = None
cl_out_from_file = True
glo.is_quiet = False
if len(sys.argv[1:]) == 0:
sys.stderr.write(
("Usage: %s [OPTIONS] [FILES]\n" % (sys.argv[0],)) +
"Files: For each app, should include manifest, epicc, and flowdroid output.\n" +
#" To override package name, use 'pkg:filename' (UNTESTED).\n" +
"Options: \n" +
" --gv graphfile: Generates graphfile.{gv,txt,pdf}\n" +
" --cl jsonfile: Writes the Flows and taint solution in JSON format and checks that flows comply with security levels\n"
" --quiet: Don't write as much to stdout\n"
)
sys.exit(1)
while True:
try:
arg = arg_iter.next()
except StopIteration:
break
try:
if arg == "--unsound":
glo.unsound = True
elif arg == "--quiet":
glo.is_quiet = True
elif arg == "--gv":
gv_base = arg_iter.next()
if not(re.match("^[A-Za-z0-9_.-/]+$", gv_base)):
die("gv filename contains bad characters")
assert(not (gv_base.endswith(".gv")))
gv_out = open(gv_base + ".gv", "w")
gv_legend = open(gv_base + ".txt", "w")
elif arg == "--check_levels_file": # Added
cl_out_filename = arg_iter.next()
cl_out = open(cl_out_filename, "w") # Writes the Flows and taint solution in JSON format
elif arg == "--check_levels_gui": # Added
cl_out_from_file = False
else:
all_filenames.append(arg)
except StopIteration:
die("Option '%s' expects an argument." % (arg,))
def check_epicc(filename, pkg_name, epicc):
def die_epicc():
sys.stderr.write(traceback.format_exc())
die("Aborted due to error in parsing " + filename)
try:
assert(len(pkg_name) > 0)
assert(isinstance(epicc, dict))
warned_unknown = False
for (intent_id, v) in epicc.iteritems():
assert(isinstance(intent_id, str))
assert(isinstance(v, list))
if intent_id == "*":
sys.stderr.write("Warning: Missing IntentID in %s\n" % (filename,))
warned_unknown = True
else:
if (intent_id == BCAST_ID):
continue
else:
assert(intent_id.startswith("newField_"))
for x in v:
try:
assert(isinstance(x, dict))
except AssertionError as e:
sys.stderr.write("\nx: %r\n" % (x,))
die_epicc()
except AssertionError as e:
die_epicc()
for filename in all_filenames:
pkg_rename = None
if ":" in filename:
[pkg_rename, filename] = filename.split(":")
root = try_read_manifest_file(filename)
if root != None:
pkg_name = pkg_rename or root.find('.').attrib['package']
glo.manifest[pkg_name] = root
glo.filter[pkg_name] = read_intent_filters_from_manifest(root)
if len(all_filenames) == 1:
pprint(dict(glo.filter[pkg_name]))
elif filename.endswith(".xml"):
flow_files.append(filename)
elif filename.endswith(".epicc"):
(pkg_name, epicc) = parse_epicc(filename, as_dict=True)
if pkg_rename:
pkg_name = pkg_rename
check_epicc(filename, pkg_name, epicc)
glo.epicc[pkg_name] = epicc
if len(all_filenames) == 1:
print "EPICC info:"
pprint(epicc)
else:
print("Unknown file type: " + filename)
# Creates levels chequer
check_levels = Check_levels(cl_out_from_file)
for filename in flow_files:
pkg_rename = None
if ":" in filename:
[pkg_rename, filename] = filename.split(":")
tree = ET.parse(filename)
root = tree.getroot()
pkg_name = pkg_rename or root.attrib['package']
flows = find_flows(root, check_levels)
# Solve Flows
def analize():
def num_intents_in_flow(flow):
return sum((type(s) in [Intent, IntentResult]) for s in [flow.src, flow.sink])
for filename in flow_files:
pkg_rename = None
if ":" in filename:
[pkg_rename, filename] = filename.split(":")
tree = ET.parse(filename)
root = tree.getroot()
pkg_name = pkg_rename or root.attrib['package']
flows = find_flows(root, check_levels)
glo.flows[pkg_name] = flows
flow_lists.append(flows)
flows = flatten(flow_lists)
flows = match_flows(flows)
solution = solve_flows(flows)
sol_src = {}
for (entity, taint_set) in solution.iteritems():
sol_src[entity] = OrderedSet(x for x in taint_set if (type(x)==Sink or type(x)==Src))
if not glo.is_quiet:
for num_intents in [0,1,2]:
print("---- Flows with %i intent(s) ---------------------------------" % num_intents)
pprint(filter(lambda x: num_intents_in_flow(x)==num_intents, flows))
print("--------------------------------------------------------------")
print("--------------------------------------------------------------")
sols = [[],[],[]]
for (entity, taint_set) in sol_src.iteritems():
if type(entity) == Intent:
ix = 0
elif type(entity) == IntentResult:
ix = 1
elif type(entity)==Sink:
ix = 2
else:
assert(type(entity)==Src)
continue
sols[ix].append((entity, taint_set))
for ix in [0,1,2]:
print("--------------------")
for (entity, taint_set) in sols[ix]:
sys.stdout.write("### '%s': ###\n" % (entity,))
pprint(list(taint_set))
print("--------------------")
return sol_src
def stringize_intents(obj): #??
t = type(obj)
if t in [str, int, type(None), Sink, Src]:
return str(obj)
elif t in [set, OrderedSet, list]:
return list(stringize_intents(x) for x in obj)
elif t in [dict]:
return t((stringize_intents(k), stringize_intents(v)) for (k,v) in obj.iteritems())
elif t in [Intent, IntentResult]:
return str(obj)
elif t == Flow:
narf = list(stringize_intents(x) for x in obj)
return Flow(*narf)
else:
assert(0)
return obj
def main_window(levels, entities, analize_leves, add_exception):
import gettext
import wx
gettext.install("app")
app = wx.PySimpleApp(0)
wx.InitAllImageHandlers()
frame_1 = MyFrame(None, wx.ID_ANY, "")
entities.remove('BUNDLE')
frame_1.create(levels, entities, analize_leves, add_exception)
app.SetTopWindow(frame_1)
frame_1.Show()
app.MainLoop()
def out_window(security_probl):
import wx
wx.InitAllImageHandlers()
dialog_1 = MyDialog(None, wx.ID_ANY, "")
dialog_1.create(security_probl)
dialog_1.Show()
def analize_leves(tuples):
if tuples != None:
check_levels.selection_assign_levels(tuples)
sol_src = analize()
import json
cl_dict = stringize_intents({'Taints': sol_src})
cl_str = json.dumps(cl_dict, sort_keys=True, indent=4, separators=(',', ': '))
security_probl = check_levels.check_levels(cl_dict)
if tuples != None:
out_window(security_probl)
else:
cl_out.write(security_probl)
if cl_out_from_file:
analize_leves(None)
else:
main_window(check_levels.get_levels_order(), check_levels.get_vars_order(), analize_leves, check_levels.add_exception)
