def extractDirectSyscalls(self, folder): #exceptList = ["lib", "grep", "sed", "bash", "sh"] exceptList = [ "ld.so", "libc.so", "libdl.so", "libcrypt.so", "libnss_compat.so", "libnsl.so", "libnss_files.so", "libnss_nis.so", "libpthread.so", "libm.so", "libresolv.so", "librt.so", "libutil.so", "libnss_dns.so", "gosu" ] lib = ".so" fileList = list() filesAdded = set() finalSyscallSet = set() for fileName in os.listdir(folder): if (util.isElf(folder + "/" + fileName)): if (lib in fileName): tmpFileName = re.sub("-.*so", ".so", fileName) tmpFileName = tmpFileName[:tmpFileName.index(".so")] tmpFileName = tmpFileName + ".so" else: tmpFileName = fileName if (tmpFileName not in exceptList and tmpFileName not in filesAdded): fileList.append(folder + "/" + fileName) filesAdded.add(tmpFileName) #libWoVersionName = fileName #if ( fileName.startswith("lib") ): # libWoVersionName = fileName[:fileName.index(".so")] # libWoVersionName = libWoVersionName + ".so" #if ( libWoVersionName not in libWoVersion ): # libWoVersion.add(libWoVersionName) # fileList.append(folder + "/" + fileName) # for exceptItem in exceptList: # #if ( lib in fileName or fileName.startswith(exceptItem) or util.isGo(folder + "/" + fileName, self.logger) ): # if ( fileName.startswith(exceptItem) or util.isGo(folder + "/" + fileName, self.logger) ): # removeList.append(folder + "/" + fileName) # break finalSet = set(fileList) # - set(removeList) for filePath in finalSet: self.logger.debug("extraction direct syscall for %s", filePath) #temp = util.extractDirectSyscalls(filePath, self.logger) #self.directSyscallCount += temp #self.logger.debug("directSyscall for %s is %d", filePath, temp) #temp = util.extractLibcSyscalls(filePath, self.logger) #self.libcSyscallCount += temp #self.logger.debug("libcSyscall for %s is %d", filePath, temp) binAnalysis = binaryAnalysis.BinaryAnalysis(filePath, self.logger) syscallSet, successCount, failCount = binAnalysis.extractDirectSyscalls( ) self.logger.debug( "Successfull direct syscalls: %d list: %s, Failed direct syscalls: %d", successCount, str(syscallSet), failCount) #self.logger.warning("Failed syscalls: %d", failCount) finalSyscallSet.update(syscallSet) return finalSyscallSet
(options, args) = parser.parse_args() if isValidOpts(options): rootLogger = setLogPath("directsyscallextractor.log") # exceptList = ["lib", "grep", "sed", "bash", "sh", "touch", "mkdir", "systemctl-systemd", "chmod", "sv", "ssh-keygen", "sysctl", "env", "svlogd", "sleep", "mount", "blkid", "stat", "cat", "systemd-path", "dpkg-query", "df", "readlink", "rm", "cp", "systemd-sysctl", "hostnamectl", "loginctl", "uname", "id", "which", "dumb-init", "find", "basename", "awk", "hostname", "modprobe", "chown", "sort", "ip", "ldconfig", "egrep", "ldconfig.real", "mv", "dirname", "whoami", "date"] exceptList = [] lib = ".so" for folderName in os.listdir(options.outputpath): fileList = list() removeList = list() # rootLogger.info("////////////Checking image: %s//////////////////", folderName) folderName = os.path.join(options.outputpath, folderName) if (os.path.isdir(folderName)): for fileName in os.listdir(folderName): if (util.isElf(folderName + "/" + fileName)): fileList.append(folderName + "/" + fileName) for exceptItem in exceptList: #if ( lib in fileName or fileName.startswith(exceptItem) or util.isGo(folderName + "/" + fileName, rootLogger) ): if (fileName.startswith(exceptItem) or util.isGo( folderName + "/" + fileName, rootLogger)): removeList.append(folderName + "/" + fileName) break finalSet = set(fileList) - set(removeList) # rootLogger.info("List of binaries for %s: %s", folderName, str(finalSet)) for filePath in finalSet: #rootLogger.debug("extraction direct syscall for %s", filePath) temp1 = util.extractDirectSyscalls(filePath, rootLogger) temp2 = util.extractLibcSyscalls(filePath, rootLogger) if (temp1 != 0 or temp2 != 0): rootLogger.info(
def processSyscalls(binprofiler, bininput, apptolibmap, appname, libccfginput, debug, rootLogger): if binprofiler and (not apptolibmap or not appname or not bininput): parser.error( "Options --appname, --apptolibmap and --bininput should be provided when enabling the --binprofiler feature." ) return False libcGraph = graph.Graph(rootLogger) libcGraph.createGraphFromInput(libccfginput, ":") libSet = set( ) #Libraries which should be added to the both import table and svf analysis output otherLibSet = set( ) #Libraries which should only be added to the import table (e.g. apr, apr-util) if (binprofiler): appName = appname appToLibMap = None try: appToLibFile = open(apptolibmap, 'r') appToLibStr = appToLibFile.read() appToLibMap = json.loads(appToLibStr) except Exception as e: rootLogger.warning( "Trying to load app to lib map json from: %s, but doesn't exist: %s", apptolibmap, str(e)) rootLogger.debug("Finished loading json") sys.exit(-1) for app in appToLibMap["apps"]: for key, value in app.items(): if (key.strip() == appName.strip()): for lib in value["libs"]: libSet.add(cleanLib(lib)) for lib in value["otherlibs"]: otherLibSet.add(cleanLib(lib)) importTableSyscalls = set() if (binprofiler): #TODO Extract required library system calls from binary profiler lib = ".so" filesAdded = set() for fileName in os.listdir(bininput): if (util.isElf(bininput + "/" + fileName)): if (lib in fileName): tmpFileName = cleanLib(fileName) else: tmpFileName = fileName if (tmpFileName not in filesAdded and (tmpFileName in otherLibSet or tmpFileName in libSet or tmpFileName == appname)): filePath = bininput + "/" + fileName myBinary = binaryAnalysis.BinaryAnalysis( filePath, rootLogger) directSyscallSet, successCount, failCount = myBinary.extractDirectSyscalls( ) indirectSyscallSet = myBinary.extractIndirectSyscalls( libcGraph) importTableSyscalls.update(directSyscallSet) importTableSyscalls.update(indirectSyscallSet) rootLogger.info("len(importTableSyscalls): %d", len(importTableSyscalls)) return importTableSyscalls
binList = set() libList = set() modList = set() indirectList = set() fileList = set() filesAdded = set() libcList = [ "ld.so", "libc.so", "libdl.so", "libcrypt.so", "libnss_compat.so", "libnsl.so", "libnss_files.so", "libnss_nis.so", "libpthread.so", "libm.so", "libresolv.so", "librt.so", "libutil.so", "libnss_dns.so" ] for fileName in os.listdir(options.input): if (util.isElf(options.input + "/" + fileName)): if (lib in fileName): tmpFileName = re.sub("-.*so", ".so", fileName) tmpFileName = tmpFileName[:tmpFileName.index(".so")] tmpFileName = tmpFileName + ".so" else: tmpFileName = fileName if (tmpFileName not in filesAdded): if (tmpFileName not in libcList): indirectList.add(options.input + "/" + fileName) if (fileName.startswith("lib") and lib in fileName): libList.add(options.input + "/" + fileName) elif (fileName.startswith("mod") and lib in fileName): modList.add(options.input + "/" + fileName) else: binList.add(options.input + "/" + fileName)
def processSyscalls(targetcfg, cfginput, mastermain, workermain, edgefilterlist, libsyscalls, libsyscallpath, binprofiler, bininput, apptolibmap, appname, output, debug, rootLogger, cfginputseparator=":"): if not cfginput or not targetcfg or not mastermain or not workermain: parser.error("All options -c, -f, -m and -w should be provided.") return False if libsyscalls and (not apptolibmap or not libsyscallpath or not appname): parser.error( "Options --appname, --apptolibmap and --libsyscallpath should be provided when enabling the --libsyscalls feature." ) return False if binprofiler and (not apptolibmap or not appname or not bininput or not output): parser.error( "Options -o, --appname, --apptolibmap and --bininput should be provided when enabling the --binprofiler feature." ) return False syscallList = list() i = 0 while i < 400: syscallList.append("syscall(" + str(i) + ")") syscallList.append("syscall(" + str(i) + ")") syscallList.append("syscall ( " + str(i) + " )") syscallList.append("syscall( " + str(i) + " )") i += 1 workerMainFuncList = list() if ("," in workermain): workerMainFuncList = workermain.split(",") else: workerMainFuncList.append(workermain) edgeFilterList = list() if (edgefilterlist): if ("," in edgefilterlist): edgeFilterList = edgefilterlist.split(",") else: edgeFilterList.append(edgefilterlist) else: edgeFilterList = workerMainFuncList applicationGraph = graph.Graph(rootLogger) applicationGraph.createGraphFromInputWithFilter(targetcfg, "->", edgeFilterList) libcGraph = graph.Graph(rootLogger) libcGraph.createGraphFromInput(cfginput, cfginputseparator) rootLogger.info("-------------Extraction Master leave functions---------") masterFunctions = applicationGraph.getLeavesFromStartNode( mastermain, list(), list()) rootLogger.info("-------------Extraction Worker leave functions---------") workerFunctions = set() for workerFunc in workerMainFuncList: workerFunctions.update( applicationGraph.getLeavesFromStartNode(workerFunc, list(), list())) masterSyscalls = set() for masterFunc in masterFunctions: rootLogger.debug("masterfunc: %s", masterFunc) masterSyscalls.update(libcGraph.getSyscallFromStartNode(masterFunc)) workerSyscalls = set() for workerFunc in workerFunctions: rootLogger.debug("workerfunc: %s", workerFunc) workerSyscalls.update(libcGraph.getSyscallFromStartNode(workerFunc)) rootLogger.info( "len(masterSyscalls): %d len(workerSyscalls): %d before adding library syscalls", len(masterSyscalls), len(workerSyscalls)) libSet = set( ) #Libraries which should be added to the both import table and svf analysis output otherLibSet = set( ) #Libraries which should only be added to the import table (e.g. apr, apr-util) if (libsyscalls or binprofiler): appName = appname appToLibMap = None try: appToLibFile = open(apptolibmap, 'r') appToLibStr = appToLibFile.read() appToLibMap = json.loads(appToLibStr) except Exception as e: rootLogger.warning( "Trying to load app to lib map json from: %s, but doesn't exist: %s", apptolibmap, str(e)) rootLogger.debug("Finished loading json") sys.exit(-1) for app in appToLibMap["apps"]: for key, value in app.items(): if (key.strip() == appName.strip()): for lib in value["libs"]: libSet.add(cleanLib(lib)) for lib in value["otherlibs"]: otherLibSet.add(cleanLib(lib)) if (libsyscalls and not binprofiler): libSyscallFile = open(libsyscallpath, 'r') libSyscallLine = libSyscallFile.readline() while libSyscallLine: splittedLine = libSyscallLine.split() if (len(splittedLine) > 2 and cleanLib(splittedLine[1]) in libSet): for syscallNum in splittedLine[2:]: if ("{" in syscallNum): syscallNum = syscallNum.replace("{", "") if ("," in syscallNum): syscallNum = syscallNum.replace(",", "") if ("}" in syscallNum): syscallNum = syscallNum.replace("}", "") masterSyscalls.add(int(syscallNum)) workerSyscalls.add(int(syscallNum)) libSyscallLine = libSyscallFile.readline() importTableSyscalls = set() if (not libsyscalls and binprofiler): #TODO Extract required library system calls from binary profiler lib = ".so" filesAdded = set() for fileName in os.listdir(bininput): if (util.isElf(bininput + "/" + fileName)): if (lib in fileName): tmpFileName = cleanLib(fileName) else: tmpFileName = fileName if (tmpFileName not in filesAdded and (tmpFileName in otherLibSet or tmpFileName in libSet or tmpFileName == appname)): filePath = bininput + "/" + fileName myBinary = binaryAnalysis.BinaryAnalysis( filePath, rootLogger) directSyscallSet, successCount, failCount = myBinary.extractDirectSyscalls( ) indirectSyscallSet = myBinary.extractIndirectSyscalls( libcGraph) importTableSyscalls.update(directSyscallSet) importTableSyscalls.update(indirectSyscallSet) if (tmpFileName in libSet): masterSyscalls.update(directSyscallSet) masterSyscalls.update(indirectSyscallSet) workerSyscalls.update(directSyscallSet) workerSyscalls.update(indirectSyscallSet) rootLogger.info( "len(importTableSyscalls): %d len(masterSyscalls): %d len(workerSyscalls): %d after adding library syscalls", len(importTableSyscalls), len(masterSyscalls), len(workerSyscalls)) #print ("master: ") #print (masterSyscalls) #print ("worker: ") #print (workerSyscalls) translator = sycall.Syscall(rootLogger) syscallmap = translator.createMap() mminusw = sorted(masterSyscalls.difference(workerSyscalls)) wminusm = sorted(workerSyscalls.difference(masterSyscalls)) importTableSyscallNames = set() masterSyscallNames = set() workerSyscallNames = set() blImportTableSyscallNames = set() blMasterSyscallNames = set() blWorkerSyscallNames = set() if (binprofiler): i = 0 while (i < 400): if i not in importTableSyscalls: if (syscallmap.get(i, None)): blImportTableSyscallNames.add(syscallmap[i]) i += 1 for syscall in importTableSyscalls: if (syscallmap.get(syscall, None)): importTableSyscallNames.add(syscallmap[syscall]) print("------- main -------") i = 0 while (i < 400): if i not in masterSyscalls: if (syscallmap.get(i, None)): blMasterSyscallNames.add(syscallmap[i]) i += 1 for syscall in masterSyscalls: if (syscallmap.get(syscall, None)): masterSyscallNames.add(syscallmap[syscall]) print(syscallmap.get(syscall, "")) print("------- child -------") i = 0 while (i < 400): if i not in workerSyscalls: if (syscallmap.get(i, None)): blWorkerSyscallNames.add(syscallmap[i]) i += 1 for syscall in workerSyscalls: if (syscallmap.get(syscall, None)): workerSyscallNames.add(syscallmap[syscall]) print(syscallmap.get(syscall, "")) if (binprofiler): outputDict = dict() outputDict['importTable'] = importTableSyscallNames outputDict['master'] = masterSyscallNames outputDict['worker'] = workerSyscallNames outputDict['blImportTable'] = blImportTableSyscallNames outputDict['blMaster'] = blMasterSyscallNames outputDict['blWorker'] = blWorkerSyscallNames util.writeDictToFile(outputDict, output) ''' mminusw = sorted(mainset.difference(workerset)) wminusm = sorted(workerset.difference(mainset)) #mminusc = sorted(mainset.difference(cachemgrset)) #cminusm = sorted(cachemgrset.difference(mainset)) rootLogger = setLogPath("graph.log") translator = sycall.Syscall(rootLogger) syscallmap = translator.createMap() ''' print("------- main minus worker -------") for syscall in mminusw: print(syscallmap[syscall]) print("------- worker minus main -------") for syscall in wminusm: print(syscallmap[syscall])