def getPackageUid(self, package_name):
logger.debug("Getting UID of the package [%s]..." % package_name)
uid = -1
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return uid
if package_name == "":
logger.warning("The name of the package is not specified!")
return uid
uid_lines = subprocess.check_output([
'adb', '-s', self.device_name, 'shell', 'cat',
'/data/system/packages.list'
])
lines = uid_lines.splitlines()
for i in range(
0, len(lines)): #first line just announces the list of devices
words = lines[i].split(' ')
if words[0].strip() == package_name:
uid = int(words[1].strip())
break
logger.debug("The UID of the package [%s] is [%d]" %
(package_name, uid))
return uid
def processNewInstance(self, cls, method, prototype, stack):
logger.debug("Processing new instance. DST: [%s%s%s]..." %
(cls, method, prototype))
#test code
# print "\n"
# for stackEntry in stack:
# print stackEntry
# print "\n"
#end of test code
newInstanceDstFromClient = (cls, method, prototype)
newInstancePosInStack = self._findFirstNewInstancePos(stack)
throughMethod = stack[newInstancePosInStack]
newInstanceSrcFromStack = stack[newInstancePosInStack + 1]
if newInstancePosInStack == -1:
self._addSuspiciousNewInstance(throughMethod,
newInstanceDstFromClient, stack)
return
for newInstancePathFromSources in self._sources_newInstance:
newInstanceSrcFromSources = newInstancePathFromSources[0]
if newInstanceSrcFromSources != newInstanceSrcFromStack:
continue
self._addNewInstancePathToMCG(newInstanceSrcFromSources,
throughMethod,
newInstanceDstFromClient)
if newInstancePathFromSources in self._uncovered_newInstance:
self._uncovered_newInstance.remove(newInstancePathFromSources)
return
self._addSuspiciousNewInstance(throughMethod, newInstanceDstFromClient,
stack)
def processDexLoad(self, fileName, source, output, stack):
logger.debug("Processing dex load message...")
file_hash = getSha256(fileName)
file_load_count = self._loaded_files_count.setdefault(file_hash, 0) + 1
newFilePath = self._rename_source_file(fileName, file_hash,
str(file_load_count))
self._loaded_files_count[file_hash] = file_load_count
self._codeFiles[newFilePath] = file_hash
dexloadPathFromStack = self._getDexLoadPathFromStack(stack)
if dexloadPathFromStack:
srcFromStack = dexloadPathFromStack[0]
throughMethod = dexloadPathFromStack[1]
if dexloadPathFromStack in self._uncovered_dexload:
self._uncovered_dexload.remove(dexloadPathFromStack)
self._addDexloadPathToMCG(srcFromStack, throughMethod, newFilePath)
#we do analyse files if appropriate dex load calls have found in sources of application
#and if we have not analysed file yet
if file_load_count > 1:
logger.info(
"File [%s] with hash [%s] is loaded for the [%d]th time! Skipping its analysis!"
% (newFilePath, file_hash, file_load_count))
else:
self.makeFileAnalysis(newFilePath)
return
#if the stack does not contain a dexload method detected in the sources
self._addSuspiciousDexload(newFilePath, stack)
logger.debug("Dex load message processed!")
def processInvoke(self, cls, method, prototype, stack):
logger.debug("Processing method invoke: [%s %s %s]..."% (cls, method, prototype))
#test code
# print "\n"
# for stackEntry in stack:
# print stackEntry
# print "\n"
#end of test code
invokeDstFromClient = (cls, method, prototype)
invokePosInStack = self._findFirstInvokePos(stack)
throughMethod = stack[invokePosInStack]
invokeSrcFromStack = stack[invokePosInStack + 1]
if invokePosInStack == -1:
logger.info("Cannot find the first occurrence of invoke method in the stack! Adding method to suspicious list!")
self._addSuspiciousInvoke(throughMethod, invokeDstFromClient, stack)
return
for invokePathFromSources in self._sources_invoke:
invokeSrcFromSources = invokePathFromSources[0]
if invokeSrcFromSources != invokeSrcFromStack:
continue
self._addInvokePathToMCG(invokeSrcFromSources, throughMethod, invokeDstFromClient)
if invokePathFromSources in self._uncovered_invoke:
self._uncovered_invoke.remove(invokePathFromSources)
return
self._addSuspiciousInvoke(throughMethod, invokeDstFromClient, stack)
def printMoiLists(self, toLogger=True):
printStr = None
if self._uncovered_invoke or self._uncovered_newInstance or self._uncovered_dexload:
printStr = "Printing still uncovered methods of interest:\n\n"
if self._uncovered_invoke:
printStr += "REFLECTION INVOKE:\n"
for (src, dst) in self._uncovered_invoke:
printStr += "SRC: [%s %s %s]\n" % src
printStr += "DST: [%s %s %s]\n" % dst
if self._uncovered_newInstance:
printStr += "REFLECTION NEW_INSTANCE:\n"
for (src, dst) in self._uncovered_newInstance:
printStr += "SRC: [%s %s %s]\n" % src
printStr += "DST: [%s %s %s]\n" % dst
if self._uncovered_dexload:
printStr += "DYNAMIC LOAD:\n"
for (src, dst) in self._uncovered_dexload:
printStr += "SRC: [%s %s %s]\n" % src
printStr += "DST: [%s %s %s]\n" % dst
else:
printStr = "All methods of interest are covered at least one time!!!\n"
if toLogger:
logger.debug("%s" % printStr)
else:
print printStr
def Run():
global error_occurred
if return_output:
output_dest = subprocess.PIPE
else:
# None means direct to stdout
output_dest = None
if stdin_input:
stdin_dest = subprocess.PIPE
else:
stdin_dest = None
pipe = subprocess.Popen(cmd,
executable='/bin/bash',
stdin=stdin_dest,
stdout=output_dest,
stderr=subprocess.STDOUT,
shell=True)
pid.append(pipe.pid)
try:
output = pipe.communicate(input=stdin_input)[0]
if output is not None and len(output) > 0:
so.append(output)
except OSError, e:
logger.debug("failed to retrieve stdout from: %s" % cmd)
logger.error(e)
so.append("ERROR")
error_occurred = True
def main_process(self):
"""main process function."""
while True:
# 接收4G和NTC的消息,message是字符,address是消息的发送地址
message, address = self.sock.recvfrom(Ubc.BUF_SIZE)
logger.debug('receive from client:' + str(len(message)) + 'bytes data')
self._process_message(message, address)
def printMoiLists(self, toLogger=True):
printStr = None
if self._uncovered_invoke or self._uncovered_newInstance or self._uncovered_dexload:
printStr = "Printing still uncovered methods of interest:\n\n"
if self._uncovered_invoke:
printStr +="REFLECTION INVOKE:\n"
for (src, dst) in self._uncovered_invoke:
printStr += "SRC: [%s %s %s]\n" % src
printStr += "DST: [%s %s %s]\n" % dst
if self._uncovered_newInstance:
printStr += "REFLECTION NEW_INSTANCE:\n"
for (src, dst) in self._uncovered_newInstance:
printStr += "SRC: [%s %s %s]\n" % src
printStr += "DST: [%s %s %s]\n" % dst
if self._uncovered_dexload:
printStr += "DYNAMIC LOAD:\n"
for (src, dst) in self._uncovered_dexload:
printStr += "SRC: [%s %s %s]\n" % src
printStr += "DST: [%s %s %s]\n" % dst
else:
printStr = "All methods of interest are covered at least one time!!!\n"
if toLogger:
logger.debug("%s" % printStr)
else:
print printStr
def processInvoke(self, cls, method, prototype, stack):
logger.debug("Processing method invoke: [%s %s %s]..." %
(cls, method, prototype))
#test code
# print "\n"
# for stackEntry in stack:
# print stackEntry
# print "\n"
#end of test code
invokeDstFromClient = (cls, method, prototype)
invokePosInStack = self._findFirstInvokePos(stack)
throughMethod = stack[invokePosInStack]
invokeSrcFromStack = stack[invokePosInStack + 1]
if invokePosInStack == -1:
logger.info(
"Cannot find the first occurrence of invoke method in the stack! Adding method to suspicious list!"
)
self._addSuspiciousInvoke(throughMethod, invokeDstFromClient,
stack)
return
for invokePathFromSources in self._sources_invoke:
invokeSrcFromSources = invokePathFromSources[0]
if invokeSrcFromSources != invokeSrcFromStack:
continue
self._addInvokePathToMCG(invokeSrcFromSources, throughMethod,
invokeDstFromClient)
if invokePathFromSources in self._uncovered_invoke:
self._uncovered_invoke.remove(invokePathFromSources)
return
self._addSuspiciousInvoke(throughMethod, invokeDstFromClient, stack)
def start_activity(self, package_name, activity_name):
#adb shell am start -n com.package.name/com.package.name.ActivityName
logger.debug("Starting activity [%s] of the package [%s]..." %
(package_name, activity_name))
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return
if not package_name:
logger.warning("The name of the package is not specified!")
return
if not activity_name:
logger.warning("The name of the activity is not specified!")
return
run_string = package_name + '/' + activity_name
try:
with open(os.devnull, 'w') as f_null:
subprocess.check_call([
'adb', '-s', self.device_name, 'shell', 'am start', '-n',
run_string
],
stderr=f_null)
except subprocess.CalledProcessError:
logger.error("Could not run activity!")
return
def Run():
global error_occurred
if return_output:
output_dest = subprocess.PIPE
else:
# None means direct to stdout
output_dest = None
if stdin_input:
stdin_dest = subprocess.PIPE
else:
stdin_dest = None
pipe = subprocess.Popen(
cmd,
executable='/bin/bash',
stdin=stdin_dest,
stdout=output_dest,
stderr=subprocess.STDOUT,
shell=True)
pid.append(pipe.pid)
try:
output = pipe.communicate(input=stdin_input)[0]
if output is not None and len(output) > 0:
so.append(output)
except OSError, e:
logger.debug("failed to retrieve stdout from: %s" % cmd)
logger.error(e)
so.append("ERROR")
error_occurred = True
def processDexLoad(self, fileName, source, output, stack):
logger.debug("Processing dex load message...")
file_hash = getSha256(fileName)
file_load_count = self._loaded_files_count.setdefault(file_hash, 0) + 1
newFilePath = self._rename_source_file(fileName, file_hash, str(file_load_count))
self._loaded_files_count[file_hash] = file_load_count
self._codeFiles[newFilePath] = file_hash
dexloadPathFromStack = self._getDexLoadPathFromStack(stack)
if dexloadPathFromStack:
srcFromStack = dexloadPathFromStack[0]
throughMethod = dexloadPathFromStack[1]
if dexloadPathFromStack in self._uncovered_dexload:
self._uncovered_dexload.remove(dexloadPathFromStack)
self._addDexloadPathToMCG(srcFromStack, throughMethod, newFilePath)
#we do analyse files if appropriate dex load calls have found in sources of application
#and if we have not analysed file yet
if file_load_count > 1:
logger.info("File [%s] with hash [%s] is loaded for the [%d]th time! Skipping its analysis!" % (newFilePath, file_hash, file_load_count))
else:
self.makeFileAnalysis(newFilePath)
return
#if the stack does not contain a dexload method detected in the sources
self._addSuspiciousDexload(newFilePath, stack)
logger.debug("Dex load message processed!")
def processNewInstance(self, cls, method, prototype, stack):
logger.debug("Processing new instance. DST: [%s%s%s]..." % (cls, method, prototype))
#test code
# print "\n"
# for stackEntry in stack:
# print stackEntry
# print "\n"
#end of test code
newInstanceDstFromClient = (cls, method, prototype)
newInstancePosInStack = self._findFirstNewInstancePos(stack)
throughMethod = stack[newInstancePosInStack]
newInstanceSrcFromStack = stack[newInstancePosInStack + 1]
if newInstancePosInStack == -1:
self._addSuspiciousNewInstance(throughMethod, newInstanceDstFromClient, stack)
return
for newInstancePathFromSources in self._sources_newInstance:
newInstanceSrcFromSources = newInstancePathFromSources[0]
if newInstanceSrcFromSources != newInstanceSrcFromStack:
continue
self._addNewInstancePathToMCG(newInstanceSrcFromSources, throughMethod, newInstanceDstFromClient)
if newInstancePathFromSources in self._uncovered_newInstance:
self._uncovered_newInstance.remove(newInstancePathFromSources)
return
self._addSuspiciousNewInstance(throughMethod, newInstanceDstFromClient, stack)
def _process_ubc_heartbeat(self, message, address):
"""Processing the heartbeat packet that are reported from the UBC module.
message:type:dict
address:type:tuple
"""
logger.debug('message:' + str(message) + 'address:' + str(address))
self._response(address, id=RspMsgIdValue.ubc_heartbeat_ack, module=MsgKey.ubc)
def mount_volume(path, password):
logger.info("mounting %s with pw %s" % (path, "****"))
if password == None:
logger.error("cannot mount volume, missing password")
return False
command = ["truecrypt", "-t", "--non-interactive", "--fs-options=user,umask=000", "--mount", path, "-p", password]
logger.debug(" ".join(command))
return subprocess.call(command) == 0
def findDestDirectory(base, entries):
logger.info("finding dest-directory for " + base)
logger.debug(entries)
for name, e in entries.items():
match = e.search(base)
if match is not None:
logger.info("found match for " + base + ": " + name)
return name
def parse_imsi(cls, imsi, database_info):
mcc = imsi[:3]
mnc = imsi[3:5]
msin = imsi[5:]
logger.debug('mcc:' + mcc + ' mnc:' + mnc + ' msin:' + msin)
if mcc != database_info.get('mcc_of_china') or mnc not in database_info:
return database_info.get('other'), imsi
return database_info.get(str(mnc)), str(msin)
def clsToDalvikCls(className):
logger.debug("Converting [%s] class name to Dalvik format class name..." % className)
res = None
if className:
res = "L%s" % className
res = res.replace('.', '/')
logger.debug("Dalvik format of the class name [%s] is [%s]!" % (className, res))
return res
def process_IN_DELETE(self, event):
logger.debug("DELETED: %s" % os.path.join(event.path, event.name))
fullname = os.path.join(event.path, event.name)
if fullname == self.fullname:
self.handler.on_delete(fullname)
elif event.name == self.filename:
logger.info("matched file-DELETED: %s" % os.path.join(event.path, event.name))
self.recalcWatchDir()
def set_feature(self):
logger.debug("set_feature of node pairs")
for item in self.four_G.nodes(data=True):
for item1 in self.tw_G.nodes(data=True):
f_real_nei = list(self.four_G.neighbors(item[0]))
f_nei = []
com_nei = []
for i in f_real_nei:
if self.four_G.node[i][
"in_test"] == 1: # foursquare中哪些是已知的
f_nei.append(self.four_G.node[i]["twitterID"])
com_nei.append(i)
t_nei = list(self.tw_G.neighbors(item1[0]))
result = [v for v in t_nei if v in f_nei] # 共同的已知的neighbor的多少
length = len(com_nei)
res_four = []
for i in range(0, length):
if f_nei[i] in result:
res_four.append(com_nei[i])
CN = len(result)
if (len(f_real_nei) + len(t_nei) - CN) > 0:
JC = CN / (len(f_real_nei) + len(t_nei) - CN)
else:
JC = 0
AA = 0
for i in res_four:
jh = len(list(self.four_G.neighbors(i))) + len(
list(
self.tw_G.neighbors(
self.four_G.node[i]["twitterID"])))
if jh != 2:
AA = AA + 1 / math.log(jh / 2)
social_feature = (CN + JC + AA) / 3
label = 0
if item1[0] == item[1]["twitterID"]:
label = 1
# s = item[0] + ',' + item1[0] + ',' + str(social_feature) + ',' + str(label) + '\n'
# if label == 1:
# # self.positive.append(s)
# # else:
# # self.negative.append(s)
if label == 1:
if item[0] in self.socialnetwork.f_anchor_known_list:
self.y.append(1)
self.x.append([social_feature])
else:
self.xt.append([social_feature])
self.name.append(item[0] + ';' + item1[0])
else:
if item[0] in self.socialnetwork.f_anchor_known_list or item1[
0] in self.socialnetwork.t_anchor_known_list:
self.y.append(0)
self.x.append([social_feature])
else:
self.xt.append([social_feature])
self.name.append(item[0] + ';' + item1[0])
def recalcWatchDir(self, arg = None):
if self.wd != None:
logger.debug("removing current watch %s" % self.wd)
self.wm.rm_watch(self.wd)
path, filename = find_first_existing_dir_in_path(self.fullname)
self.filename = filename
wdd = self.wm.add_watch(path, mask)
self.wd = wdd[path]
logger.debug("_currently watching %s / %s" % (path, filename))
def transformStack(stack):
logger.debug("Transforming stack data into internal representation...")
transformedStack = []
for i in stack:
t = tuple(str(v.strip()) for v in i.split(","))
# t = tuple(v.strip() for v in i.split(","))
transformedStack.append(t)
logger.debug("Stack transformed successfully!")
return transformedStack
def transformStack(stack):
logger.debug("Transforming stack data into internal representation...")
transformedStack = []
for i in stack:
t = tuple(str(v.strip()) for v in i.split(","))
# t = tuple(v.strip() for v in i.split(","))
transformedStack.append(t)
logger.debug("Stack transformed successfully!")
return transformedStack
def _addNewInstancePathToMCG(self, src, through, dst):
logger.debug("Adding newInstance method path to our graph...")
tupl = (src, through, dst)
if tupl not in self._covered_newInstance:
self._covered_newInstance.append(tupl)
self._stadynaMcg.addNewInstancePath(src, through, dst)
logger.info("The path [%s] -- [%s] through [%s] for newInstance method is added to our graph!" % (str(src), str(dst), str(through)))
else:
logger.info("The path [%s] -- [%s] through [%s] for newInstance method is already in our graph!" % (str(src), str(dst), str(through)))
def _addDexloadPathToMCG(self, src, through, filename):
logger.debug("Adding dexload method path to our graph...")
tupl = (src, through, filename)
if tupl not in self._covered_dexload:
self._covered_dexload.append(tupl)
self._stadynaMcg.addDexloadPath(src, through, filename)
logger.info("The path [%s] -- [%s] through [%s] for dexload is added to our graph!" % (str(src), str(filename), str(through)))
else:
logger.info("The path [%s] -- [%s] through [%s] for dexload is already in our graph!" % (str(src), str(filename), str(through)))
def clsToDalvikCls(className):
logger.debug("Converting [%s] class name to Dalvik format class name..." %
className)
res = None
if className:
res = "L%s" % className
res = res.replace('.', '/')
logger.debug("Dalvik format of the class name [%s] is [%s]!" %
(className, res))
return res
def makeFileAnalysis(self, file_path):
logger.debug("Performing analysis of file [%s]..." % file_path)
a = None
d = None
dx = None
ret_type = androconf.is_android(file_path)
if ret_type == "APK":
a = apk.APK(file_path)
d = dvm.DalvikVMFormat(a.get_dex())
elif ret_type == "DEX" :
try :
d = dvm.DalvikVMFormat(open(file_path, "rb").read())
except Exception as e :
logger.error("[%s] is not valid dex file!" % file_path, e)
return
dx = analysis.VMAnalysis(d)
invokeMethodPaths = analysis.seccon_get_invoke_method_paths(dx)
newInstanceMethodPaths = analysis.seccon_get_newInstance_method_paths(dx)
dynamicMethodPaths = analysis.seccon_get_dyncode_loading_paths(dx)
if invokeMethodPaths:
t = None
for path in invokeMethodPaths:
src = path.get_src(d.get_class_manager())
dst = path.get_dst(d.get_class_manager())
t = (src, dst)
self._sources_invoke.append(t)
self._uncovered_invoke.append(t)
if newInstanceMethodPaths:
t = None
for path in newInstanceMethodPaths:
src = path.get_src(d.get_class_manager())
dst = path.get_dst(d.get_class_manager())
t = (src, dst)
self._sources_newInstance.append(t)
self._uncovered_newInstance.append(t)
if dynamicMethodPaths:
t = None
for path in dynamicMethodPaths:
src = path.get_src(d.get_class_manager())
dst = path.get_dst(d.get_class_manager())
t = (src, dst)
self._sources_dexload.append(t)
self._uncovered_dexload.append(t)
#building MFG for the file
self._stadynaMcg.analyseFile(dx, a)
def makeFileAnalysis(self, file_path):
logger.debug("Performing analysis of file [%s]..." % file_path)
a = None
d = None
dx = None
ret_type = androconf.is_android(file_path)
if ret_type == "APK":
a = apk.APK(file_path)
d = dvm.DalvikVMFormat(a.get_dex())
elif ret_type == "DEX":
try:
d = dvm.DalvikVMFormat(open(file_path, "rb").read())
except Exception as e:
logger.error("[%s] is not valid dex file!" % file_path, e)
return
dx = analysis.VMAnalysis(d)
invokeMethodPaths = analysis.seccon_get_invoke_method_paths(dx)
newInstanceMethodPaths = analysis.seccon_get_newInstance_method_paths(
dx)
dynamicMethodPaths = analysis.seccon_get_dyncode_loading_paths(dx)
if invokeMethodPaths:
t = None
for path in invokeMethodPaths:
src = path.get_src(d.get_class_manager())
dst = path.get_dst(d.get_class_manager())
t = (src, dst)
self._sources_invoke.append(t)
self._uncovered_invoke.append(t)
if newInstanceMethodPaths:
t = None
for path in newInstanceMethodPaths:
src = path.get_src(d.get_class_manager())
dst = path.get_dst(d.get_class_manager())
t = (src, dst)
self._sources_newInstance.append(t)
self._uncovered_newInstance.append(t)
if dynamicMethodPaths:
t = None
for path in dynamicMethodPaths:
src = path.get_src(d.get_class_manager())
dst = path.get_dst(d.get_class_manager())
t = (src, dst)
self._sources_dexload.append(t)
self._uncovered_dexload.append(t)
#building MFG for the file
self._stadynaMcg.analyseFile(dx, a)
def runOnce(cmd, timeout_time=None, return_output=True, stdin_input=None):
"""Spawns a subprocess to run the given shell command.
Args:
cmd: shell command to run
timeout_time: time in seconds to wait for command to run before aborting.
return_output: if True return output of command as string. Otherwise,
direct output of command to stdout.
stdin_input: data to feed to stdin
Returns:
output of command
Raises:
adb_interface_errors.WaitForResponseTimedOutError if command did not complete within
timeout_time seconds.
adb_interface_errors.AbortError is command returned error code and setAbortOnError is on.
"""
start_time = time.time()
so = []
pid = []
global _abort_on_error, error_occurred
error_occurred = False
def Run():
global error_occurred
if return_output:
output_dest = subprocess.PIPE
else:
# None means direct to stdout
output_dest = None
if stdin_input:
stdin_dest = subprocess.PIPE
else:
stdin_dest = None
pipe = subprocess.Popen(
cmd,
executable='/bin/bash',
stdin=stdin_dest,
stdout=output_dest,
stderr=subprocess.STDOUT,
shell=True)
pid.append(pipe.pid)
try:
output = pipe.communicate(input=stdin_input)[0]
if output is not None and len(output) > 0:
so.append(output)
except OSError, e:
logger.debug("failed to retrieve stdout from: %s" % cmd)
logger.error(e)
so.append("ERROR")
error_occurred = True
if pipe.returncode:
logger.debug("Error: %s returned %d error code" %(cmd,
pipe.returncode))
error_occurred = True
def processInvokeMsg(stadynaAnalyser, stadynaMsg):
logger.debug("Processing OP_METHOD_INVOKE message...")
#call to str is required to omit 'u' symbol before the strings
cls = str(stadynaMsg.get(consts.JSON_CLASS))
method = str(stadynaMsg.get(consts.JSON_METHOD))
prototype = str(stadynaMsg.get(consts.JSON_PROTO))
# cls = stadynaMsg.get(consts.JSON_CLASS)
# method = stadynaMsg.get(consts.JSON_METHOD)
# prototype = stadynaMsg.get(consts.JSON_PROTO)
stack = utils.transformStack(stadynaMsg.get(consts.JSON_STACK))
stadynaAnalyser.processInvoke(cls, method, prototype, stack)
logger.debug("OP_METHOD_INVOKE message processed!")
def run_classifier(self):
logger.debug("random forest")
# 随机森林训练
clf = RandomForestClassifier(n_estimators=100, max_depth=None, min_samples_split=2)
clf = clf.fit(self.feature.x, self.feature.y)
print(clf.feature_importances_) # 每个特征的重要性
s = clf.predict_proba(self.feature.xt)
print(len(self.feature.xt))
classify_result = []
for i in range(0, len(s)):
classify_result.append(self.feature.name[i] + ';' + str([s[i][0], s[i][1]]))
return classify_result
def runOnce(cmd, timeout_time=None, return_output=True, stdin_input=None):
"""Spawns a subprocess to run the given shell command.
Args:
cmd: shell command to run
timeout_time: time in seconds to wait for command to run before aborting.
return_output: if True return output of command as string. Otherwise,
direct output of command to stdout.
stdin_input: data to feed to stdin
Returns:
output of command
Raises:
adb_interface_errors.WaitForResponseTimedOutError if command did not complete within
timeout_time seconds.
adb_interface_errors.AbortError is command returned error code and setAbortOnError is on.
"""
start_time = time.time()
so = []
pid = []
global _abort_on_error, error_occurred
error_occurred = False
def Run():
global error_occurred
if return_output:
output_dest = subprocess.PIPE
else:
# None means direct to stdout
output_dest = None
if stdin_input:
stdin_dest = subprocess.PIPE
else:
stdin_dest = None
pipe = subprocess.Popen(cmd,
executable='/bin/bash',
stdin=stdin_dest,
stdout=output_dest,
stderr=subprocess.STDOUT,
shell=True)
pid.append(pipe.pid)
try:
output = pipe.communicate(input=stdin_input)[0]
if output is not None and len(output) > 0:
so.append(output)
except OSError, e:
logger.debug("failed to retrieve stdout from: %s" % cmd)
logger.error(e)
so.append("ERROR")
error_occurred = True
if pipe.returncode:
logger.debug("Error: %s returned %d error code" %
(cmd, pipe.returncode))
error_occurred = True
def processInvokeMsg(stadynaAnalyser, stadynaMsg):
logger.debug("Processing OP_METHOD_INVOKE message...")
#call to str is required to omit 'u' symbol before the strings
cls = str(stadynaMsg.get(consts.JSON_CLASS))
method = str(stadynaMsg.get(consts.JSON_METHOD))
prototype = str(stadynaMsg.get(consts.JSON_PROTO))
# cls = stadynaMsg.get(consts.JSON_CLASS)
# method = stadynaMsg.get(consts.JSON_METHOD)
# prototype = stadynaMsg.get(consts.JSON_PROTO)
stack = utils.transformStack(stadynaMsg.get(consts.JSON_STACK))
stadynaAnalyser.processInvoke(cls, method, prototype, stack)
logger.debug("OP_METHOD_INVOKE message processed!")
def performFinalInfoSave(self, where, resultsFileName, executionTime=-1):
logger.debug("Saving information...")
self.calculateFinalNumbers()
#saving gexf
finalGexfFileName = "%s%s" % (resultsFileName, "_final")
self.saveGexf(where, finalGexfFileName)
#saving log file
logFileName = '%s%s' % (resultsFileName, '_log.txt')
logSavePath = os.path.join(where, logFileName)
self._save_log_file(logSavePath, executionTime)
logger.debug("Final results are saved!")
def _addDexloadPathToMCG(self, src, through, filename):
logger.debug("Adding dexload method path to our graph...")
tupl = (src, through, filename)
if tupl not in self._covered_dexload:
self._covered_dexload.append(tupl)
self._stadynaMcg.addDexloadPath(src, through, filename)
logger.info(
"The path [%s] -- [%s] through [%s] for dexload is added to our graph!"
% (str(src), str(filename), str(through)))
else:
logger.info(
"The path [%s] -- [%s] through [%s] for dexload is already in our graph!"
% (str(src), str(filename), str(through)))
def _addNewInstancePathToMCG(self, src, through, dst):
logger.debug("Adding newInstance method path to our graph...")
tupl = (src, through, dst)
if tupl not in self._covered_newInstance:
self._covered_newInstance.append(tupl)
self._stadynaMcg.addNewInstancePath(src, through, dst)
logger.info(
"The path [%s] -- [%s] through [%s] for newInstance method is added to our graph!"
% (str(src), str(dst), str(through)))
else:
logger.info(
"The path [%s] -- [%s] through [%s] for newInstance method is already in our graph!"
% (str(src), str(dst), str(through)))
def _response(cls, address, **kwargs):
"""send data to client.
address:type:tuple
"""
_send = cls.sock.sendto
packet = {}
for key,value in kwargs.items():
packet[key] = value
try:
_send(json.dumps(packet), address)
logger.debug('send' + str(json.dumps(packet)) + str(address))
except socket.timeout as e:
logger.error('send ' + str(json.dumps(packet)) + 'falied ' + e)
def clean_logcat(self):
logger.debug("Cleaning logcat...")
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return
try:
with open(os.devnull, 'w') as f_null:
subprocess.check_call(['adb', '-s', self.device_name, 'logcat', '-c'], stderr=f_null)
except subprocess.CalledProcessError:
logger.error("Could not clean logcat!")
return
logger.debug("Logcat is cleaned!")
def performFinalInfoSave(self, where, resultsFileName, executionTime = -1):
logger.debug("Saving information...")
self.calculateFinalNumbers()
#saving gexf
finalGexfFileName = "%s%s" % (resultsFileName, "_final")
self.saveGexf(where, finalGexfFileName)
#saving log file
logFileName = '%s%s' % (resultsFileName, '_log.txt')
logSavePath = os.path.join(where, logFileName)
self._save_log_file(logSavePath, executionTime)
logger.debug("Final results are saved!")
def startInstrumentationNoResultsMod(
self, package_name, runner_name, no_window_animation=False,
raw_mode=False, wait=True, instrumentation_args={}):
"""Runs instrumentation and dumps output to stdout.
Equivalent to startInstrumentation, but will dump instrumentation
'normal' output to stdout, instead of parsing return results. Command will
never timeout.
"""
adb_command_string = self.previewInstrumentationCommandMod(
package_name, runner_name, no_window_animation=no_window_animation,
raw_mode=raw_mode, wait=wait, instrumentation_args=instrumentation_args)
logger.debug(adb_command_string)
runCommand(adb_command_string, return_output=False)
def startServiceExplicitly(self, package_name, service_name):
#adb shell am start -n com.package.name/com.package.name.ActivityName
logger.debug("Starting service [%s] from the package [%s]..." % (package_name, service_name))
if not package_name:
logger.error("The name of the package is not specified!")
return
if not service_name:
logger.error("The name of the activity is not specified!")
return
run_string = package_name + '/' + service_name
cmd = "am startservice -n %s" % run_string
self.sendShellCommand(cmd)
def get_logcat(self, tag = None, level = None):
logger.debug("Attaching to a logcat pipe...")
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return None
if tag == None: tag = '*'
if level == None: level = 'V'
if level.upper() not in Device.LOG_LEVELS:
logger.error("The log level %s is not specified correctly!" % str(level))
command = ['adb', '-s', self.device_name, 'logcat', '%s:%s' % (tag, level)]
if tag != '*':
command.append('*:S')
return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=None)
def clean_logcat(self):
logger.debug("Cleaning logcat...")
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return
try:
with open(os.devnull, 'w') as f_null:
subprocess.check_call(
['adb', '-s', self.device_name, 'logcat', '-c'],
stderr=f_null)
except subprocess.CalledProcessError:
logger.error("Could not clean logcat!")
return
logger.debug("Logcat is cleaned!")
def startServiceExplicitly(self, package_name, service_name):
#adb shell am start -n com.package.name/com.package.name.ActivityName
logger.debug("Starting service [%s] from the package [%s]..." %
(package_name, service_name))
if not package_name:
logger.error("The name of the package is not specified!")
return
if not service_name:
logger.error("The name of the activity is not specified!")
return
run_string = package_name + '/' + service_name
cmd = "am startservice -n %s" % run_string
self.sendShellCommand(cmd)
def cut_k_list(self, k):
logger.debug("k is %s" % (str(k)))
# 产生配对的dict，设置k为3 four,tw 1,0，这里设置k
four_result_list = dict()
tw_result_list = dict()
logger.info(len(self.f_anchor_known_list))
for line in self.f:
x_list = line.split(';')
if len(x_list) == 3:
rate = eval(x_list[2])[0]
if x_list[0] in four_result_list:
four_result_list[x_list[0]][0].append(rate)
four_result_list[x_list[0]][1].append(
x_list[1]) # 对应的twitter名
else:
four_result_list[x_list[0]] = [[rate], [x_list[1]]]
if x_list[1] in tw_result_list:
tw_result_list[x_list[1]][0].append(rate)
tw_result_list[x_list[1]][1].append(
x_list[0]) # 对应的twitter名
else:
tw_result_list[x_list[1]] = [[rate], [x_list[0]]]
for item in four_result_list.keys():
s = dict()
for i in range(0, len(four_result_list[item][1])):
s[four_result_list[item][1][i]] = four_result_list[item][0][i]
s = sorted(s.items(), key=lambda j: j[1], reverse=True)
s = map(lambda x: x[0], s)
four_result_list[item] = list(s)
if k is not None:
if len(four_result_list[item]) > k - 1:
four_result_list[item] = four_result_list[item][0:k]
for item in tw_result_list.keys():
s = dict()
for i in range(0, len(tw_result_list[item][1])):
s[tw_result_list[item][1][i]] = tw_result_list[item][0][i]
s = sorted(s.items(), key=lambda j: j[1], reverse=True)
s = map(lambda x: x[0], s)
tw_result_list[item] = list(s)
if k is not None:
if len(tw_result_list[item]) > k - 1:
tw_result_list[item] = tw_result_list[item][0:k]
self.four_result = four_result_list
self.tw_result = tw_result_list
def set_network(self):
logger.debug("save node")
f = open('E:/graduate/twitter_foursquare/foursquare/users/user',
encoding='utf-8')
twitter_file = open('E:/graduate/twitter_foursquare/twitter/user',
encoding='utf-8')
for line in twitter_file:
tw_list = line.split('\t')
user_name = tw_list[0]
if len(user_name) > 1 and len(tw_list) >= 5:
self.tw_G.add_node(user_name,
time='',
location='',
word='',
fourID=None)
# foursquare 目前只将对应的twitterid存了进去，其它的属性未存,
# location=tw_list[4], realname=tw_list[2], bio=tw_list[3], home=tw_list[4]
num = 0
for line in f:
lis = line.split('\t')
userFourID = lis[0]
if 'com' in lis[7]:
userTwitterID = lis[7].split('twitter.com/')[1]
if self.tw_G.has_node(userTwitterID):
num = num + 1
self.four_G.add_node(userFourID,
twitterID=userTwitterID,
location='',
time='',
word='',
in_test=0)
self.tw_G.node[userTwitterID][
"fourID"] = userFourID # 标明 tw对应的four ID
self.f_all_anchor_list.append(userFourID)
self.t_all_anchor_list.append(userTwitterID)
else:
self.four_G.add_node(userFourID,
twitterID=None,
location='',
time='',
word='',
in_test=0)
f.close()
twitter_file.close()
def get_devices_list():
logger.debug("Getting the list of running devices...")
devices = []
input_devices = None
try:
input_devices = subprocess.check_output(["adb", "devices"])
except subprocess.CalledProcessError:
logger.error("Could not find attached devices!")
return devices
lines = input_devices.splitlines()
for i in range(0,len(lines)): #first line just announces the list of devices
words = lines[i].split('\t')
if len(words) == 2 and words[1].strip() == 'device':
devices.append(words[0].strip())
logger.debug("Device list:\n[%s]", '\n'.join(devices))
return devices
def sendCommand(self, command_string, timeout_time=20, retry_count=3):
"""Send a command via adb.
Args:
command_string: adb command to run
timeout_time: number of seconds to wait for command to respond before
retrying
retry_count: number of times to retry command before raising
WaitForResponseTimedOutError
Returns:
string output of command
Raises:
WaitForResponseTimedOutError if device does not respond to command within time
"""
adb_cmd = "adb %s %s" % (self._target_arg, command_string)
logger.debug("about to run %s" % adb_cmd)
return runCommand(adb_cmd, timeout_time=timeout_time, retry_count=retry_count)
def get_devices_list():
logger.debug("Getting the list of running devices...")
devices = []
input_devices = None
try:
input_devices = subprocess.check_output(["adb", "devices"])
except subprocess.CalledProcessError:
logger.error("Could not find attached devices!")
return devices
lines = input_devices.splitlines()
for i in range(
0, len(lines)): #first line just announces the list of devices
words = lines[i].split('\t')
if len(words) == 2 and words[1].strip() == 'device':
devices.append(words[0].strip())
logger.debug("Device list:\n[%s]", '\n'.join(devices))
return devices
def install_package(self, apk_path):
logger.debug("Installing application [%s]..." % apk_path)
if not self.is_alive():
logger.error("The device [%s] is not running!" % str(self.device_name))
return False
if apk_path == "":
logger.warning("The path to the application to install is not specified!")
return False
try:
with open(os.devnull, 'w') as f_null:
subprocess.check_call(["adb", "-s", self.device_name, "install", "-r", apk_path], stderr=f_null)
except subprocess.CalledProcessError:
logger.error("Could not install application [%s] on the device!" % apk_path)
return False
logger.debug("Application [%s] is installed!" % apk_path)
return True
def get_file(self, what, to_dir):
logger.debug("Coping file [%s] from device to directory [%s]..." % (what, to_dir))
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return False
if what == "":
logger.warning("The name of the file to download is not specified!")
return False
try:
with open(os.devnull, 'w') as f_null:
subprocess.check_call(["adb", "-s", self.device_name, "pull", what, to_dir], stderr=f_null)
except subprocess.CalledProcessError:
logger.error("Could not download file [%s] from the device!" % what)
return False
logger.debug("File [%s] is downloaded!" % what)
return True
def get_package_uid(self, package_name):
logger.debug("Getting UID of the package [%s]..." % package_name)
uid = -1
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return uid
if package_name == "":
logger.warning("The name of the package is not specified!")
return uid
uid_lines = subprocess.check_output(['adb', '-s', self.device_name, 'shell', 'cat', '/data/system/packages.list'])
lines = uid_lines.splitlines()
for i in range(0,len(lines)): #first line just announces the list of devices
words = lines[i].split(' ')
if words[0].strip() == package_name:
uid = int(words[1].strip())
break
logger.debug("The UID of the package [%s] is [%d]" % (package_name, uid))
return uid
def sendCommand(self, command_string, timeout_time=20, retry_count=3):
"""Send a command via adb.
Args:
command_string: adb command to run
timeout_time: number of seconds to wait for command to respond before
retrying
retry_count: number of times to retry command before raising
WaitForResponseTimedOutError
Returns:
string output of command
Raises:
WaitForResponseTimedOutError if device does not respond to command within time
"""
adb_cmd = "adb %s %s" % (self._target_arg, command_string)
logger.debug("about to run %s" % adb_cmd)
return runCommand(adb_cmd,
timeout_time=timeout_time,
retry_count=retry_count)
def processDexLoadMsg(device, resultsDirPath, stadynaAnalyser, stadynaMsg):
logger.debug("Processing OP_DEX_LOAD message...")
#call to str is required to omit 'u' symbol before the strings
source = str(stadynaMsg.get(consts.JSON_DEX_SOURCE))
output = str(stadynaMsg.get(consts.JSON_DEX_OUTPUT))
# source = stadynaMsg.get(consts.JSON_DEX_SOURCE)
# output = stadynaMsg.get(consts.JSON_DEX_OUTPUT)
stack = utils.transformStack(stadynaMsg.get(consts.JSON_STACK))
if not device.get_file(source, resultsDirPath):
logger.error("Could not get file [%s] from the device for analysis!" % source)
return
_, fileName = os.path.split(source)
anFilePath = os.path.join(resultsDirPath, fileName)
if not (os.path.exists(anFilePath)):
logger.error("There is no local file [%s] to analyse!" % anFilePath)
return
stadynaAnalyser.processDexLoad(anFilePath, source, output, stack)
logger.debug("OP_DEX_LOAD message processed!")
def getDeviceSerialsList():
logger.debug("Getting the list of running deviceSerials...")
deviceSerials = []
input_devices = None
try:
input_devices = runCommand("adb devices")
except WaitForResponseTimedOutError:
logger.error("Command timeout exception!")
return deviceSerials
except:
logger.error("Could not find attached devices!")
return deviceSerials
lines = input_devices.splitlines()
for i in range(0,len(lines)): #first line just announces the list of deviceSerials
words = lines[i].split('\t')
if (len(words) == 2 and (words[1].strip() == 'device' or words[1].strip() == 'offline')):
deviceSerials.append(words[0].strip())
logger.debug("Device list: %s", deviceSerials)
return deviceSerials
def _getDexLoadPathFromStack(self, stack):
logger.debug("Processing dex load stack...")
#test code
# print "\n"
# for stackEntry in stack:
# print stackEntry
# print "\n"
#end of test code
#we iterate over stack (not over entries in sources)
#because it is possible that there are several dexload entries
#in the stack. Thus, we look for the most recent one.
for stackEntryPos in xrange(1, len(stack)):
stackEntry = stack[stackEntryPos]
for dexloadPathFromSources in self._sources_dexload:
dexloadSrcFromSources = dexloadPathFromSources[0]
dexloadDstFromSources = dexloadPathFromSources[1]
if stackEntry != dexloadSrcFromSources:
continue
prevStackEntry = stack[stackEntryPos - 1]
if prevStackEntry == dexloadDstFromSources:
logger.debug("The method, which calls dexload, is found [%s%s%s]!" % dexloadSrcFromSources)
return dexloadPathFromSources
logger.debug("The called dexload method was not detected in sources!")
return None
def is_alive(self):
logger.debug("Checking if a device [%s] is alive..." %
self.device_name)
if not self.device_name:
logger.error("The device object is not instantiated!")
return False
input_devices = None
try:
input_devices = subprocess.check_output(["adb", "devices"])
except subprocess.CalledProcessError:
logger.error("Could not find the attached device [%s]!" %
self.device_name)
return False
lines = input_devices.splitlines()
for i in range(
1, len(lines)): #first line just announces the list of devices
words = lines[i].split('\t')
if len(words) == 2 and words[1].strip() == 'device':
trimmed = words[0].strip()
if trimmed == self.device_name:
logger.debug("Device [%s] is alive!" % self.device_name)
return True
logger.debug("Device [%s] is not running!" % self.device_name)
return False
def _getDexLoadPathFromStack(self, stack):
logger.debug("Processing dex load stack...")
#test code
# print "\n"
# for stackEntry in stack:
# print stackEntry
# print "\n"
#end of test code
#we iterate over stack (not over entries in sources)
#because it is possible that there are several dexload entries
#in the stack. Thus, we look for the most recent one.
for stackEntryPos in xrange(1, len(stack)):
stackEntry = stack[stackEntryPos]
for dexloadPathFromSources in self._sources_dexload:
dexloadSrcFromSources = dexloadPathFromSources[0]
dexloadDstFromSources = dexloadPathFromSources[1]
if stackEntry != dexloadSrcFromSources:
continue
prevStackEntry = stack[stackEntryPos - 1]
if prevStackEntry == dexloadDstFromSources:
logger.debug(
"The method, which calls dexload, is found [%s%s%s]!" %
dexloadSrcFromSources)
return dexloadPathFromSources
logger.debug("The called dexload method was not detected in sources!")
return None
def start_activity(self, package_name, activity_name):
#adb shell am start -n com.package.name/com.package.name.ActivityName
logger.debug("Starting activity [%s] of the package [%s]..." % (package_name, activity_name))
if not self.is_alive():
logger.error("The device [%s] is not running!" % self.device_name)
return
if not package_name:
logger.warning("The name of the package is not specified!")
return
if not activity_name:
logger.warning("The name of the activity is not specified!")
return
run_string = package_name + '/' + activity_name
try:
with open(os.devnull, 'w') as f_null:
subprocess.check_call(['adb', '-s', self.device_name, 'shell', 'am start', '-n', run_string], stderr=f_null)
except subprocess.CalledProcessError:
logger.error("Could not run activity!")
return