def trackSO(self):
call_list = ['open', 'mmap']
if self.mem_utils.WORD_SIZE == 4 or self.compat32:
call_list.append('mmap2')
self.open_syscall = syscall.Syscall(None, self.cell_name, self.cell, self.param, self.mem_utils, self.task_utils,
self.context_manager, None, self.sharedSyscall, self.lgr, None, call_list=call_list,
soMap=self.soMap, targetFS=self.targetFS, skip_and_mail=False, compat32=self.compat32)
def processSeparatedCalls(procsepfile, callgraphfile, sep):
f = open(procsepfile)
for line in f:
words = line.split(":")
initPoint = words[0].strip()
calledFunc = words[1].strip()
if initPoint not in processInitLibMap:
processInitLibMap[initPoint] = []
processInitLibMap[initPoint].append(calledFunc)
# Create the graph
rootLogger = setLogPath("graph.log")
myGraph = graph.Graph(rootLogger)
myGraph.createGraphFromInput(callgraphfile, sep)
syscallList = list()
i = 0
while i < 400:
syscallList.append("syscall(" + str(i) + ")")
syscallList.append("syscall(" + str(i) + ")")
syscallList.append("syscall ( " + str(i) + " )")
syscallList.append("syscall( " + str(i) + " )")
i += 1
for initPoint in processInitLibMap:
for func in processInitLibMap[initPoint]:
leaves = myGraph.getLeavesFromStartNode(func, syscallList, list())
for leaf in leaves:
leaf = "".join(leaf.split())
words = re.split('[()]', leaf)
syscallno = words[1]
if initPoint not in processInitSyscallMap:
processInitSyscallMap[initPoint] = set()
processInitSyscallMap[initPoint].add(int(syscallno))
translator = sycall.Syscall(rootLogger)
syscallmap = translator.createMap()
for initPoint in processInitSyscallMap:
syscallset = processInitSyscallMap[initPoint]
syscalllist = sorted(syscallset)
print("------- " + initPoint + " --------")
for syscall in syscalllist:
print(str(syscall))
def trackSO(self):
self.open_syscall = syscall.Syscall(None,
self.cell_name,
self.cell,
self.param,
self.mem_utils,
self.task_utils,
self.context_manager,
None,
self.sharedSyscall,
self.lgr,
None,
call_list=['open'],
soMap=self.soMap,
targetFS=self.targetFS,
skip_and_mail=False,
compat32=self.compat32)
self.lgr.debug('TrackThreads watching open syscall for %s is %s' %
(self.cell_name, str(self.open_syscall)))
myGraph.addEdge(funcLine.strip() + "64", funcLine.strip())
myGraph.addEdge(funcLine.strip(), funcLine.strip() + "64")
if ( funcLine.strip().endswith("64") ):
myGraph.addEdge(funcLine.strip()[:-2], funcLine.strip())
myGraph.addEdge(funcLine.strip(), funcLine.strip()[:-2])'''
leaves = myGraph.getLeavesFromStartNode(funcLine.strip(),
syscallList, list())
allSyscalls = allSyscalls.union(leaves)
funcLine = funcFile.readline()
syscallList = list()
for syscallStr in allSyscalls:
syscallNum = int(syscallStr[8:-1])
bisect.insort(syscallList, syscallNum)
print(str(len(syscallList)))
print(syscallList)
syscallMapper = syscall.Syscall(rootLogger)
syscallMap = syscallMapper.createMap()
blackList = []
i = 1
while i < 400:
if (i not in syscallList and syscallMap.get(i, None)
and syscallMap[i] not in exceptList):
blackList.append(syscallMap[i])
i += 1
print("Num of black listed syscalls: " + str(len(blackList)))
seccompProfile = seccomp.Seccomp(rootLogger)
blackListProfile = seccompProfile.createProfile(blackList)
outputPath = options.output
if (options.output is None):
sys.exit(-1)
sensitiveSyscallSet = set()
sensitiveSyscallFile = open(options.sensitivesyscalls, 'r')
sensitiveSyscallLine = sensitiveSyscallFile.readline()
while (sensitiveSyscallLine):
sensitiveSyscallSet.add(sensitiveSyscallLine.strip())
sensitiveSyscallLine = sensitiveSyscallFile.readline()
sensitiveSyscallOutfile = open(options.sensitivestatspath, 'w')
syscallReductionFile = open(options.syscallreductionpath, 'w')
sensitiveSyscallStatLine = "{};{};{};{}\n"
syscallReductionStatLine = "{};{};{};{}\n"
syscallTranslator = syscall.Syscall(rootLogger)
syscallMap = syscallTranslator.createMap()
cfginput = options.cfginput
binbasepath = options.binpath
cfgbasepath = options.cfgpath
outputbasepath = options.outputpath
binprofiler = True
if (options.libdebloating):
syscallReductionFile.write(
syscallReductionStatLine.format("Application",
"# Import Table Syscalls",
"# Piecewise Master",
"# Piecewise Worker",
"# Temporal Master Syscalls",
"# Temporal Worker Syscalls"))
dest="inputpath",
default=None,
nargs=1,
help="Input path")
parser.add_option("-d",
"--debug",
dest="debug",
action="store_true",
default=False,
help="Debug enabled/disabled")
(options, args) = parser.parse_args()
if isValidOpts(options):
rootLogger = setLogPath("syscall.log")
syscallExtractor = syscall.Syscall(rootLogger)
#auditdMap = syscallExtractor.createMapWithAuditd()
syscallMap = syscallExtractor.createMap()
exceptList = [
"access", "arch_prctl", "brk", "close", "execve", "exit_group",
"fcntl", "fstat", "geteuid", "lseek", "mmap", "mprotect", "munmap",
"openat", "prlimit64", "read", "rt_sigaction", "rt_sigprocmask",
"set_robust_list", "set_tid_address", "stat", "statfs", "write",
"setns", "capget", "capset", "chdir", "fchown", "futex",
"getdents64", "getpid", "getppid", "lstat", "openat", "prctl",
"setgid", "setgroups", "setuid", "stat", "io_setup", "getdents",
"clone", "readlinkat", "newfstatat", "getrandom", "sigaltstack",
"getresgid", "getresuid", "setresgid", "setresuid", "alarm",
"getsid", "getpgrp", "epoll_pwait", "vfork"
]
def createFineGrainedSeccompProfile(self, tempOutputFolder, resultsFolder):
self.logger.debug("tempOutputFolder: %s", tempOutputFolder)
allSyscalls = set()
muslSyscallList = list()
glibcSyscallList = list()
i = 0
while i < 400:
muslSyscallList.append("syscall(" + str(i) + ")")
glibcSyscallList.append("syscall(" + str(i) + ")")
glibcSyscallList.append("syscall ( " + str(i) + " )")
glibcSyscallList.append("syscall( " + str(i) + " )")
i += 1
glibcGraph = graph.Graph(self.logger)
glibcGraph.createGraphFromInput(self.glibcCfgpath, ":")
glibcWrapperListTemp = []
if (self.strictMode):
for func in self.glibcFuncList:
glibcWrapperListTemp.extend(
glibcGraph.getSyscallFromStartNode(func))
else:
i = 0
while i < 400:
glibcWrapperListTemp.append(i)
i += 1
glibcWrapperList = set(glibcWrapperListTemp)
muslGraph = graph.Graph(self.logger)
muslGraph.createGraphFromInput(self.muslCfgpath, "->")
muslWrapperListTemp = []
if (self.strictMode):
for func in self.muslFuncList:
muslWrapperListTemp.extend(
muslGraph.getSyscallFromStartNode(func))
else:
i = 0
while i < 400:
muslWrapperListTemp.append(i)
i += 1
muslWrapperList = set(muslWrapperListTemp)
# self.logger.debug("glibcWrapperList: %s", str(glibcWrapperList))
# self.logger.debug("muslWrapperList: %s", str(muslWrapperList))
#Go through extra CFGs such as libaio to extract lib->syscall mapping
for fileName in os.listdir(self.cfgFolderPath):
self.logger.debug("Adding cfg: %s", fileName)
glibcGraph.createGraphFromInput(
self.cfgFolderPath + "/" + fileName, "->")
muslGraph.createGraphFromInput(self.cfgFolderPath + "/" + fileName,
"->")
exceptList = [
"access", "arch_prctl", "brk", "close", "execve", "exit_group",
"fcntl", "fstat", "geteuid", "lseek", "mmap", "mprotect", "munmap",
"openat", "prlimit64", "read", "rt_sigaction", "rt_sigprocmask",
"set_robust_list", "set_tid_address", "stat", "statfs", "write",
"setns", "capget", "capset", "chdir", "fchown", "futex",
"getdents64", "getpid", "getppid", "lstat", "openat", "prctl",
"setgid", "setgroups", "setuid", "stat", "io_setup", "getdents",
"clone", "readlinkat", "newfstatat", "getrandom", "sigaltstack",
"getresgid", "getresuid", "setresgid", "setresuid", "alarm",
"getsid", "getpgrp", "epoll_pwait", "vfork"
]
javaExceptList = [
"open", "getcwd", "openat", "close", "fopen", "fclose", "link",
"unlink", "unlinkat", "mknod", "rename", "renameat", "mkdir",
"rmdir", "readlink", "realpath", "symlink", "stat", "lstat",
"fstat", "fstatat", "chown", "lchown", "fchown", "chmod", "fchmod",
"utimes", "futimes", "lutimes", "readdir", "read", "write",
"access", "getpwuid", "getgrgid", "statvfs", "clock_getres",
"get_mempolicy", "gettid", "getcpu", "fallocate", "memfd_create",
"fstatat64", "newfstatat"
]
libsWithCfg = set()
libsInLibc = set()
functionStarts = set()
for fileName in os.listdir(self.cfgFolderPath):
libsWithCfg.add(fileName)
libsInLibc.add("libcrypt.callgraph.out")
libsInLibc.add("libdl.callgraph.out")
libsInLibc.add("libnsl.callgraph.out")
libsInLibc.add("libnss_compat.callgraph.out")
libsInLibc.add("libnss_files.callgraph.out")
libsInLibc.add("libnss_nis.callgraph.out")
libsInLibc.add("libpthread.callgraph.out")
libsInLibc.add("libm.callgraph.out")
libsInLibc.add("libresolv.callgraph.out")
libsInLibc.add("librt.callgraph.out")
#iterate over ELF files
#IF library which has CFG add to graph
#ELIF binary or library without CFG add to starting nodes
cfgAvailable = False
for fileName in os.listdir(tempOutputFolder):
self.logger.debug("fileName: %s", fileName)
if (fileName.startswith("lib") and fileName != "libs.out"):
cfgAvailable = True
tmpFileName = re.sub("-.*so", ".so", fileName)
tmpFileName = tmpFileName[:tmpFileName.index(".so")]
tmpFileName = tmpFileName + ".callgraph.out"
self.logger.debug("tmpFileName: %s", tmpFileName)
if (tmpFileName in libsWithCfg):
glibcGraph.createGraphFromInput(
self.cfgFolderPath + "/" + tmpFileName, "->")
elif (tmpFileName in libsInLibc):
cfgAvailable = True
else:
cfgAvailable = False
if (not fileName.startswith("lib") or not cfgAvailable):
self.logger.info("Adding function starts for %s", fileName)
functionList = util.extractImportedFunctions(
tempOutputFolder + "/" + fileName, self.logger)
if (not functionList):
self.logger.warning(
"Function extraction for file: %s failed!", fileName)
functionStarts.update(set(functionList))
tmpSet = set()
allSyscalls = set()
for function in functionStarts:
leaves = glibcGraph.getLeavesFromStartNode(function,
glibcSyscallList,
list())
tmpSet = tmpSet.union(leaves)
syscallList = list()
for syscallStr in tmpSet:
syscallStr = syscallStr.replace("syscall( ", "syscall(")
syscallStr = syscallStr.replace("syscall ( ", "syscall(")
syscallStr = syscallStr.replace(" )", ")")
syscallNum = int(syscallStr[8:-1])
allSyscalls.add(syscallNum)
syscallMapper = syscall.Syscall(self.logger)
syscallMap = syscallMapper.createMap(self.maptype)
blackList = set()
i = 0
while i < 400:
if (i not in allSyscalls and syscallMap.get(i, None)
and syscallMap[i] not in exceptList):
blackList.add(syscallMap[i])
i += 1
self.logger.info("Results for %s:///////////////////////////////////",
self.name)
self.logger.info("%s: len(blacklist): %d", self.name, len(blackList))
self.logger.info("%s: blacklist: %s", self.name, str(blackList))
self.logger.info(
"//////////////////////////////////////////////////////////////////"
)
def createSeccompProfile(self, tempOutputFolder, resultsFolder):
returnCode = 0
if os.geteuid() != 0:
self.logger.error("This script must be run as ROOT only!")
exit("This script must be run as ROOT only. Exiting.")
self.logger.debug("tempOutputFolder: %s", tempOutputFolder)
allSyscalls = set()
muslSyscallList = list()
glibcSyscallList = list()
i = 0
while i < 400:
muslSyscallList.append("syscall(" + str(i) + ")")
glibcSyscallList.append("syscall(" + str(i) + ")")
glibcSyscallList.append("syscall ( " + str(i) + " )")
glibcSyscallList.append("syscall( " + str(i) + " )")
i += 1
fineGrainCfgs = dict()
glibcGraph = graph.Graph(self.logger)
glibcGraph.createGraphFromInput(self.glibcCfgpath, ":")
glibcWrapperListTemp = []
if (self.strictMode):
for func in self.glibcFuncList:
glibcWrapperListTemp.extend(
glibcGraph.getSyscallFromStartNode(func))
else:
i = 0
while i < 400:
glibcWrapperListTemp.append(i)
i += 1
glibcWrapperList = set(glibcWrapperListTemp)
muslGraph = graph.Graph(self.logger)
muslGraph.createGraphFromInput(self.muslCfgpath, "->")
muslWrapperListTemp = []
if (self.strictMode):
for func in self.muslFuncList:
muslWrapperListTemp.extend(
muslGraph.getSyscallFromStartNode(func))
else:
i = 0
while i < 400:
muslWrapperListTemp.append(i)
i += 1
muslWrapperList = set(muslWrapperListTemp)
# self.logger.debug("glibcWrapperList: %s", str(glibcWrapperList))
# self.logger.debug("muslWrapperList: %s", str(muslWrapperList))
#TODO Separate libaio-like CFGs from fine-grained CFGs
#Go through extra CFGs such as libaio to extract lib->syscall mapping
#for fileName in os.listdir(self.cfgFolderPath):
# self.logger.debug("Adding cfg: %s", fileName)
# glibcGraph.createGraphFromInput(self.cfgFolderPath + "/" + fileName, "->")
# muslGraph.createGraphFromInput(self.cfgFolderPath + "/" + fileName, "->")
#time.sleep(10)
exceptList = [
"access", "arch_prctl", "brk", "close", "execve", "exit_group",
"fcntl", "fstat", "geteuid", "lseek", "mmap", "mprotect", "munmap",
"openat", "prlimit64", "read", "rt_sigaction", "rt_sigprocmask",
"set_robust_list", "set_tid_address", "stat", "statfs", "write",
"setns", "capget", "capset", "chdir", "fchown", "futex",
"getdents64", "getpid", "getppid", "lstat", "openat", "prctl",
"setgid", "setgroups", "setuid", "stat", "io_setup", "getdents",
"clone", "readlinkat", "newfstatat", "getrandom", "sigaltstack",
"getresgid", "getresuid", "setresgid", "setresuid", "alarm",
"getsid", "getpgrp", "epoll_pwait", "vfork"
]
javaExceptList = [
"open", "getcwd", "openat", "close", "fopen", "fclose", "link",
"unlink", "unlinkat", "mknod", "rename", "renameat", "mkdir",
"rmdir", "readlink", "realpath", "symlink", "stat", "lstat",
"fstat", "fstatat", "chown", "lchown", "fchown", "chmod", "fchmod",
"utimes", "futimes", "lutimes", "readdir", "read", "write",
"access", "getpwuid", "getgrgid", "statvfs", "clock_getres",
"get_mempolicy", "gettid", "getcpu", "fallocate", "memfd_create",
"fstatat64", "newfstatat"
]
binaryReady = False
libFileReady = False
languageReady = False
try:
self.logger.debug("Checking cache in %s", tempOutputFolder)
myFile = open(tempOutputFolder + "/" + C.CACHE, 'r')
binaryReady = True
myFile = open(tempOutputFolder + "/" + C.LIBFILENAME, 'r')
libFileReady = True
# myFile = open(tempOutputFolder + "/" + C.LANGFILENAME, 'r')
# languageReady = True
except OSError as e:
self.logger.info(
"Cache doesn't exist, must extract binaries and libraries")
self.logger.debug("binaryReady: %s libFileReady: %s", str(binaryReady),
str(libFileReady))
myContainer = container.Container(self.imagePath, self.options,
self.logger, self.args)
self.containerName = myContainer.getContainerName()
if (not myContainer.pruneVolumes()):
self.logger.warning(
"Pruning volumes failed, storage may run out of space\n")
returncode, out, err = util.runCommand("mkdir -p " + tempOutputFolder)
if (returncode != 0):
self.logger.error("Failed to create directory: %s with error: %s",
tempOutputFolder, err)
else:
self.logger.debug("Successfully created directory: %s",
tempOutputFolder)
ttr = 10
logSleepTime = 60
sysdigTotalRunCount = 3
if (binaryReady):
sysdigTotalRunCount = 1
sysdigRunCount = 1
if (self.name == "softwareag-apigateway"):
logSleepTime = 60
if (self.name == "cirros"):
logSleepTime = 120
psListAll = set()
self.logger.info("--->Starting MONITOR phase:")
while (sysdigRunCount <= sysdigTotalRunCount):
myMonitor = processMonitorFactory.Factory(
self.logger,
self.monitoringTool,
psListFilePath=self.binLibList)
#mySysdig = sysdig.Sysdig(self.logger)
self.logger.debug(
"Trying to kill and delete container which might not be running in loop... Not a problem if returns error"
)
str(myContainer.kill())
str(myContainer.delete())
self.logger.info(
"Running sysdig multiple times. Run count: %d from total: %d",
sysdigRunCount, sysdigTotalRunCount)
sysdigRunCount += 1
#sysdigResult = mySysdig.runSysdigWithDuration(logSleepTime)
monitorResult = myMonitor.runWithDuration(logSleepTime)
if (not monitorResult):
self.logger.error(
"Running sysdig with execve failed, not continuing for container: %s",
self.name)
self.logger.error(
"Please make sure sysdig is installed and you are running the script with root privileges. If problem consists please contact our support team."
)
self.errorMessage = "Running sysdig with execve failed"
if (monitorResult and
myContainer.runWithoutSeccomp()): #myContainer.run() ):
self.status = True
self.logger.info(
"Ran container sleeping for %d seconds to generate logs and extract execve system calls",
logSleepTime)
time.sleep(logSleepTime)
myMonitor.waitUntilComplete()
originalLogs = myContainer.checkLogs()
self.logger.debug("originalLog: %s", originalLogs)
time.sleep(10)
if (not myContainer.checkStatus()):
self.logger.warning(
"Container exited after running, trying to run in attached mode!"
)
self.logger.debug(str(myContainer.delete()))
if (not myContainer.runInAttachedMode()):
self.errorMessage = "Container didn't run in attached mode either, forfeiting!"
self.logger.error(
"Container didn't run in attached mode either, forfeiting!"
)
self.logger.error(
"There is a problem launching a container for %s. Please validate you can run the container without Confine. If so, contact our support team.",
self.name)
self.logger.debug(str(myContainer.delete()))
return C.NOATTACH
else:
time.sleep(10)
if (not myContainer.checkStatus()):
self.errorMessage = "Container got killed after running in attached mode as well!"
self.logger.error(
"Container got killed after running in attached mode as well, forfeiting!"
)
self.logger.error(
"There is a problem launching a container for %s. Please validate you can run the container without Confine. If so, contact our support team.",
self.name)
self.logger.debug(str(myContainer.kill()))
self.logger.debug(str(myContainer.delete()))
return C.CONSTOP
self.runnable = True
self.logger.debug(
"Ran container %s successfully, sleeping for %d seconds",
self.name, ttr)
time.sleep(ttr)
self.logger.debug(
"Finished sleeping, extracting psNames for %s", self.name)
self.logger.debug(
"Starting to identify running processes and required binaries and libraries through dynamic analysis."
)
if (not binaryReady):
psList = myMonitor.extractPsNames(
"execve", myContainer.getContainerName(),
myContainer.getContainerId())
if (not psList):
self.logger.error(
"PS List is None from extractPsNames(). Retrying this container: %s",
self.name)
self.logger.debug(str(myContainer.kill()))
self.logger.debug(str(myContainer.delete()))
self.errorMessage = "PS List is None from extractPsNames(), error in sysdig, retrying this container"
return C.SYSDIGERR
if (len(psList) == 0):
self.logger.error(
"PS List is None from extractPsNames(). Retrying this container: %s",
self.name)
self.logger.debug(str(myContainer.kill()))
self.logger.debug(str(myContainer.delete()))
self.errorMessage = "PS List is None from extractPsNames(), error in sysdig, retrying this container"
return C.NOPROCESS
self.logger.info("len(psList) from sysdig: %d",
len(psList))
# TODO: Do we need to do this? Or can we just rely on copyFromContainerWithLibs below
psList = psList.union(myContainer.extractLibsFromProc())
self.logger.debug(
"len(psList) after extracting proc list: %d",
len(psList))
self.logger.debug("Container: %s PS List: %s", self.name,
str(psList))
self.logger.debug(
"Container: %s extracted psList with %d elements",
self.name, len(psList))
self.logger.debug("Entering not binaryReady")
if (not util.deleteAllFilesInFolder(
tempOutputFolder, self.logger)):
self.logger.error(
"Failed to delete files in temporary output folder, exiting..."
)
self.errorMessage = "Failed to delete files in temporary output folder"
sys.exit(-1)
psListAll.update(psList)
self.logger.info(
"Container: %s extracted psList with %d elements",
self.name, len(psListAll))
if (self.status):
if (not binaryReady):
self.logger.info("Container: %s PS List: %s", self.name,
str(psListAll))
self.logger.info(
"Starting to copy identified binaries and libraries (This can take some time...)"
) #Will try to copy from different paths. Some might not exist. Errors are normal.")
if (self.extractAllBinaries):
psListAll.update(myContainer.extractAllBinaries())
for binaryPath in psListAll:
if (binaryPath.strip() != ""):
myContainer.copyFromContainerWithLibs(
binaryPath, tempOutputFolder)
#if ( not myContainer.copyFromContainerWithLibs(binaryPath, tempOutputFolder) ):
# self.logger.error("Problem copying files from container!")
binaryReady = True
myFile = open(tempOutputFolder + "/" + C.CACHE, 'w')
myFile.write("complete")
myFile.flush()
myFile.close()
self.logger.info(
"Finished copying identified binaries and libraries")
self.logger.info("<---Finished MONITOR phase\n")
self.logger.debug(str(myContainer.kill()))
self.logger.debug(str(myContainer.delete()))
if (binaryReady):
self.logger.info("--->Starting Direct Syscall Extraction")
self.logger.info("Extracting direct system call invocations")
directSyscallSet = self.extractDirectSyscalls(tempOutputFolder)
self.logger.info("<---Finished Direct Syscall Extraction\n")
if (not libFileReady):
self.logger.info("--->Starting ANALYZE phase")
self.logger.info(
"Extracting imported functions and storing in libs.out"
)
self.extractAllImportedFunctions(tempOutputFolder,
C.LIBFILENAME)
self.logger.info("<---Finished ANALYZE phase\n")
#if ( not languageReady ):
self.extractBinaryType(tempOutputFolder)
isMusl = self.usesMusl(tempOutputFolder)
funcFilePath = tempOutputFolder + "/" + C.LIBFILENAME
funcFile = open(funcFilePath, 'r')
funcLine = funcFile.readline()
if (not funcLine and not os.path.isfile(
os.path.join(self.goFolderPath,
self.name + ".syscalls"))
and len(directSyscallSet) == 0):
self.logger.info(
"%s container can't be hardened because no functions can be extracted from binaries and no direct syscalls found",
self.name)
self.errorMessage = "container can't be hardened because no functions can be extracted from binaries and no direct syscalls found"
return C.NOFUNCS
self.logger.info(
"--->Starting INTEGRATE phase, extracting the list required system calls"
)
functionStartsOriginal = set()
functionStartsFineGrain = set()
if (self.fineGrain):
#TODO Fix fine grained analysis
#1. Create CFG for each library
#2. Extract leaves from all imported functions in libs.out
#3. Create a list of required functions for each library
#4. Use fine grained version or all imported for libraries without CFG
libsWithCfg = set()
libsInLibc = set()
for fileName in os.listdir(self.cfgFolderPath):
libsWithCfg.add(fileName)
libsInLibc.add("libcrypt.callgraph.out")
libsInLibc.add("libdl.callgraph.out")
libsInLibc.add("libnsl.callgraph.out")
libsInLibc.add("libnss_compat.callgraph.out")
libsInLibc.add("libnss_files.callgraph.out")
libsInLibc.add("libnss_nis.callgraph.out")
libsInLibc.add("libpthread.callgraph.out")
libsInLibc.add("libm.callgraph.out")
libsInLibc.add("libresolv.callgraph.out")
libsInLibc.add("librt.callgraph.out")
libsInLibc.add("libutil.callgraph.out")
libsInLibc.add("libnss_dns.callgraph.out")
cfgAvailable = False
for fileName in os.listdir(tempOutputFolder):
self.logger.debug("fileName: %s", fileName)
tmpFileName = fileName
functionList = set()
if (fileName.startswith("lib")
and fileName != "libs.out"):
cfgAvailable = True
tmpFileName = re.sub("-.*so", ".so", fileName)
tmpFileName = tmpFileName[:tmpFileName.index(".so"
)]
tmpFileName = tmpFileName + ".callgraph.out"
self.logger.debug("tmpFileName: %s", tmpFileName)
if (tmpFileName in libsWithCfg):
tmpGraph = graph.Graph(self.logger)
tmpGraph.createGraphFromInput(
self.cfgFolderPath + "/" + tmpFileName, "->")
funcFile.seek(0)
funcLine = funcFile.readline()
while (funcLine):
funcName = funcLine.strip()
leaves = tmpGraph.getLeavesFromStartNode(
funcName, list(), list())
if (len(leaves) != 0
and funcName not in leaves):
#self.logger.debug("funcName: %s leaves: %s", funcName, str(leaves))
functionList.update(set(leaves))
funcLine = funcFile.readline()
elif (tmpFileName in libsInLibc):
continue
else:
self.logger.info("Adding function starts for %s",
fileName)
functionList = util.extractImportedFunctions(
tempOutputFolder + "/" + fileName, self.logger)
if (not functionList):
self.logger.warning(
"Function extraction for file: %s failed!",
fileName)
functionStartsFineGrain.update(set(functionList))
funcFile.seek(0)
funcLine = funcFile.readline()
while (funcLine):
funcLine = funcLine.strip()
functionStartsOriginal.add(funcLine)
funcLine = funcFile.readline()
funcFile.close()
self.logger.info(
"Traversing libc call graph to identify required system calls"
)
tmpSet = set()
allSyscallsOriginal = set()
for function in functionStartsOriginal:
if (isMusl):
leaves = muslGraph.getLeavesFromStartNode(
function, muslSyscallList, list())
else:
leaves = glibcGraph.getLeavesFromStartNode(
function, glibcSyscallList, list())
#self.logger.debug("function: %s, tmpSet: %s", function, tmpSet)
tmpSet = tmpSet.union(leaves)
for syscallStr in tmpSet:
syscallStr = syscallStr.replace("syscall( ", "syscall(")
syscallStr = syscallStr.replace("syscall ( ", "syscall(")
syscallStr = syscallStr.replace(" )", ")")
syscallNum = int(syscallStr[8:-1])
allSyscallsOriginal.add(syscallNum)
self.logger.debug("allSyscallsOriginal: %s",
str(allSyscallsOriginal))
allSyscallsFineGrain = set()
if (self.fineGrain):
tmpSet = set()
for function in functionStartsFineGrain:
#if ( function == "fork" ):
# self.logger.debug("/////////////////////////////////////////FORK has been found///////////////////////////////////")
if (isMusl):
leaves = muslGraph.getLeavesFromStartNode(
function, muslSyscallList, list())
else:
leaves = glibcGraph.getLeavesFromStartNode(
function, glibcSyscallList, list())
tmpSet = tmpSet.union(leaves)
for syscallStr in tmpSet:
syscallStr = syscallStr.replace(
"syscall( ", "syscall(")
syscallStr = syscallStr.replace(
"syscall ( ", "syscall(")
syscallStr = syscallStr.replace(" )", ")")
syscallNum = int(syscallStr[8:-1])
allSyscallsFineGrain.add(syscallNum)
#Check if we have go syscalls
staticSyscallList = []
try:
staticSyscallListFile = open(
os.path.join(self.goFolderPath,
self.name + ".syscalls"), 'r')
syscallLine = staticSyscallListFile.readline()
while (syscallLine):
staticSyscallList.append(int(syscallLine.strip()))
syscallLine = staticSyscallListFile.readline()
except Exception as e:
self.logger.debug(
"Can't extract syscalls from: %s",
os.path.join(
self.goFolderPath, self.name +
".syscalls (probably not a golang developed application)"
))
self.logger.debug(
"After reading file: %s len(staticSyscallList): %d",
os.path.join(self.goFolderPath, self.name + ".syscalls"),
len(staticSyscallList))
syscallMapper = syscall.Syscall(self.logger)
syscallMap = syscallMapper.createMap(self.maptype)
self.logger.info("Generating final system call filter list")
blackListOriginal = []
i = 1
while i < 400:
if ((self.directSyscallCount == 0
and self.libcSyscallCount == 0)
or (isMusl and i in muslWrapperList)
or (i in glibcWrapperList)):
if (i not in directSyscallSet
and i not in staticSyscallList
and i not in allSyscallsOriginal
and syscallMap.get(i, None)
and syscallMap[i] not in exceptList):
if (("Java" in self.languageSet
and syscallMap[i] not in javaExceptList)
or ("Java" not in self.languageSet)):
blackListOriginal.append(syscallMap[i])
i += 1
blackListFineGrain = []
if (self.fineGrain):
i = 1
while i < 400:
if ((self.directSyscallCount == 0
and self.libcSyscallCount == 0)
or (isMusl and i in muslWrapperList)
or (i in glibcWrapperList)):
if (i not in directSyscallSet
and i not in staticSyscallList
and i not in allSyscallsFineGrain
and syscallMap.get(i, None)
and syscallMap[i] not in exceptList):
if (("Java" in self.languageSet
and syscallMap[i] not in javaExceptList)
or ("Java" not in self.languageSet)):
blackListFineGrain.append(syscallMap[i])
i += 1
self.logger.info(
"************************************************************************************"
)
self.logger.info(
"Container Name: %s Num of filtered syscalls (original): %s",
self.name, str(len(blackListOriginal)))
self.logger.info(
"************************************************************************************"
)
self.logger.info("<---Finished INTEGRATE phase\n")
self.blSyscallsOriginal = blackListOriginal
self.blSyscallOriginalCount = len(blackListOriginal)
if (self.fineGrain):
self.logger.info(
"Container Name: %s Num of filtered syscalls (fine grained): %s",
self.name, str(len(blackListFineGrain)))
self.blSyscallsFineGrain = blackListFineGrain
self.blSyscallFineGrainCount = len(blackListFineGrain)
seccompProfile = seccomp.Seccomp(self.logger)
if (self.fineGrain):
blackListProfile = seccompProfile.createProfile(
blackListFineGrain)
else:
blackListProfile = seccompProfile.createProfile(
blackListOriginal)
if ("/" in self.name):
outputPath = resultsFolder + "/" + self.name.replace(
"/", "-") + ".seccomp.json"
else:
outputPath = resultsFolder + "/" + self.name + ".seccomp.json"
outputFile = open(outputPath, 'w')
outputFile.write(blackListProfile)
outputFile.flush()
outputFile.close()
self.logger.info(
"--->Validating generated Seccomp profile: %s", outputPath)
if (myContainer.runWithSeccompProfile(outputPath)):
time.sleep(logSleepTime)
debloatedLogs = myContainer.checkLogs()
if (len(originalLogs) == len(debloatedLogs)):
time.sleep(3)
if (myContainer.checkStatus()):
self.logger.info(
"************************************************************************************"
)
self.logger.info(
"Finished validation. Container for image: %s was hardened SUCCESSFULLY!",
self.name)
self.logger.info(
"************************************************************************************"
)
self.debloatStatus = True
returnCode = 0
else:
self.logger.warning(
"Container for image: %s was hardened with problems. Dies after running!",
self.name)
self.errorMessage = "Container was hardened with problems. Dies after running!"
returnCode = C.HSTOPS
else:
self.logger.warning(
"Container for image: %s was hardened with problems: len(original): %d len(seccomp): %d original: %s seccomp: %s",
self.name, len(originalLogs), len(debloatedLogs),
originalLogs, debloatedLogs)
self.errorMessage = "Unknown problem in hardening container!"
returnCode = C.HLOGLEN
if (self.isDependent):
self.logger.info(
"Not killing container: %s because it is a dependent for hardening another container",
self.name)
else:
if (not myContainer.kill() and self.debloatStatus):
self.logger.warning(
"Container can't be killed even though successfully hardened! Hardening has been unsuccessfull!"
)
self.errorMessage = "Container can't be killed even though successfully hardened! Hardening has been unsuccessfull!"
self.debloatStatus = False
returnCode = C.HNOKILL
else:
self.errorMessage = "Unknown problem in hardening container!"
returnCode = C.HNORUN
if (not self.isDependent):
self.logger.debug(str(myContainer.delete()))
return returnCode
libcCfg.createGraphFromInput(options.callgraph, options.separator)
dockerSyscalls = [
"access", "arch_prctl", "brk", "close", "execve", "exit_group",
"fcntl", "fstat", "geteuid", "lseek", "mmap", "mprotect", "munmap",
"openat", "prlimit64", "read", "rt_sigaction", "rt_sigprocmask",
"set_robust_list", "set_tid_address", "stat", "statfs", "write",
"setns", "capget", "capset", "chdir", "fchown", "futex",
"getdents64", "getpid", "getppid", "lstat", "openat", "prctl",
"setgid", "setgroups", "setuid", "stat", "io_setup", "getdents",
"clone", "readlinkat", "newfstatat", "getrandom", "sigaltstack",
"getresgid", "getresuid", "setresgid", "setresuid", "alarm",
"getsid", "getpgrp", "epoll_pwait", "vfork"
]
syscallObj = syscall.Syscall(rootLogger)
syscallMap = syscallObj.createMap(options.maptype)
syscallInverseMap = syscallObj.getInverseMap()
dockerSyscallNums = set()
for syscallStr in dockerSyscalls:
rootLogger.debug("syscallInverseMap.get(syscallStr, -1): %s",
str(syscallInverseMap.get(syscallStr, -1)))
dockerSyscallNums.add(syscallInverseMap.get(syscallStr, -1))
containerSyscallSet = set()
containerSyscallSet.update(dockerSyscallNums)
lib = ".so"
binList = set()
libList = set()
modList = set()
def trackSO(self):
callnum = self.task_utils.syscallNumber('open')
self.open_syscall = syscall.Syscall(None, self.cell_name, self.cell, self.param, self.mem_utils, self.task_utils,
self.context_manager, None, self.sharedSyscall, self.lgr, None, callnum_list=[callnum],
soMap=self.soMap, targetFS=self.targetFS, skip_and_mail=False)
def processSyscalls(targetcfg,
cfginput,
mastermain,
workermain,
edgefilterlist,
libsyscalls,
libsyscallpath,
binprofiler,
bininput,
apptolibmap,
appname,
output,
debug,
rootLogger,
cfginputseparator=":"):
if not cfginput or not targetcfg or not mastermain or not workermain:
parser.error("All options -c, -f, -m and -w should be provided.")
return False
if libsyscalls and (not apptolibmap or not libsyscallpath or not appname):
parser.error(
"Options --appname, --apptolibmap and --libsyscallpath should be provided when enabling the --libsyscalls feature."
)
return False
if binprofiler and (not apptolibmap or not appname or not bininput
or not output):
parser.error(
"Options -o, --appname, --apptolibmap and --bininput should be provided when enabling the --binprofiler feature."
)
return False
syscallList = list()
i = 0
while i < 400:
syscallList.append("syscall(" + str(i) + ")")
syscallList.append("syscall(" + str(i) + ")")
syscallList.append("syscall ( " + str(i) + " )")
syscallList.append("syscall( " + str(i) + " )")
i += 1
workerMainFuncList = list()
if ("," in workermain):
workerMainFuncList = workermain.split(",")
else:
workerMainFuncList.append(workermain)
edgeFilterList = list()
if (edgefilterlist):
if ("," in edgefilterlist):
edgeFilterList = edgefilterlist.split(",")
else:
edgeFilterList.append(edgefilterlist)
else:
edgeFilterList = workerMainFuncList
applicationGraph = graph.Graph(rootLogger)
applicationGraph.createGraphFromInputWithFilter(targetcfg, "->",
edgeFilterList)
libcGraph = graph.Graph(rootLogger)
libcGraph.createGraphFromInput(cfginput, cfginputseparator)
rootLogger.info("-------------Extraction Master leave functions---------")
masterFunctions = applicationGraph.getLeavesFromStartNode(
mastermain, list(), list())
rootLogger.info("-------------Extraction Worker leave functions---------")
workerFunctions = set()
for workerFunc in workerMainFuncList:
workerFunctions.update(
applicationGraph.getLeavesFromStartNode(workerFunc, list(),
list()))
masterSyscalls = set()
for masterFunc in masterFunctions:
rootLogger.debug("masterfunc: %s", masterFunc)
masterSyscalls.update(libcGraph.getSyscallFromStartNode(masterFunc))
workerSyscalls = set()
for workerFunc in workerFunctions:
rootLogger.debug("workerfunc: %s", workerFunc)
workerSyscalls.update(libcGraph.getSyscallFromStartNode(workerFunc))
rootLogger.info(
"len(masterSyscalls): %d len(workerSyscalls): %d before adding library syscalls",
len(masterSyscalls), len(workerSyscalls))
libSet = set(
) #Libraries which should be added to the both import table and svf analysis output
otherLibSet = set(
) #Libraries which should only be added to the import table (e.g. apr, apr-util)
if (libsyscalls or binprofiler):
appName = appname
appToLibMap = None
try:
appToLibFile = open(apptolibmap, 'r')
appToLibStr = appToLibFile.read()
appToLibMap = json.loads(appToLibStr)
except Exception as e:
rootLogger.warning(
"Trying to load app to lib map json from: %s, but doesn't exist: %s",
apptolibmap, str(e))
rootLogger.debug("Finished loading json")
sys.exit(-1)
for app in appToLibMap["apps"]:
for key, value in app.items():
if (key.strip() == appName.strip()):
for lib in value["libs"]:
libSet.add(cleanLib(lib))
for lib in value["otherlibs"]:
otherLibSet.add(cleanLib(lib))
if (libsyscalls and not binprofiler):
libSyscallFile = open(libsyscallpath, 'r')
libSyscallLine = libSyscallFile.readline()
while libSyscallLine:
splittedLine = libSyscallLine.split()
if (len(splittedLine) > 2 and cleanLib(splittedLine[1]) in libSet):
for syscallNum in splittedLine[2:]:
if ("{" in syscallNum):
syscallNum = syscallNum.replace("{", "")
if ("," in syscallNum):
syscallNum = syscallNum.replace(",", "")
if ("}" in syscallNum):
syscallNum = syscallNum.replace("}", "")
masterSyscalls.add(int(syscallNum))
workerSyscalls.add(int(syscallNum))
libSyscallLine = libSyscallFile.readline()
importTableSyscalls = set()
if (not libsyscalls and binprofiler):
#TODO Extract required library system calls from binary profiler
lib = ".so"
filesAdded = set()
for fileName in os.listdir(bininput):
if (util.isElf(bininput + "/" + fileName)):
if (lib in fileName):
tmpFileName = cleanLib(fileName)
else:
tmpFileName = fileName
if (tmpFileName not in filesAdded and
(tmpFileName in otherLibSet or tmpFileName in libSet
or tmpFileName == appname)):
filePath = bininput + "/" + fileName
myBinary = binaryAnalysis.BinaryAnalysis(
filePath, rootLogger)
directSyscallSet, successCount, failCount = myBinary.extractDirectSyscalls(
)
indirectSyscallSet = myBinary.extractIndirectSyscalls(
libcGraph)
importTableSyscalls.update(directSyscallSet)
importTableSyscalls.update(indirectSyscallSet)
if (tmpFileName in libSet):
masterSyscalls.update(directSyscallSet)
masterSyscalls.update(indirectSyscallSet)
workerSyscalls.update(directSyscallSet)
workerSyscalls.update(indirectSyscallSet)
rootLogger.info(
"len(importTableSyscalls): %d len(masterSyscalls): %d len(workerSyscalls): %d after adding library syscalls",
len(importTableSyscalls), len(masterSyscalls), len(workerSyscalls))
#print ("master: ")
#print (masterSyscalls)
#print ("worker: ")
#print (workerSyscalls)
translator = sycall.Syscall(rootLogger)
syscallmap = translator.createMap()
mminusw = sorted(masterSyscalls.difference(workerSyscalls))
wminusm = sorted(workerSyscalls.difference(masterSyscalls))
importTableSyscallNames = set()
masterSyscallNames = set()
workerSyscallNames = set()
blImportTableSyscallNames = set()
blMasterSyscallNames = set()
blWorkerSyscallNames = set()
if (binprofiler):
i = 0
while (i < 400):
if i not in importTableSyscalls:
if (syscallmap.get(i, None)):
blImportTableSyscallNames.add(syscallmap[i])
i += 1
for syscall in importTableSyscalls:
if (syscallmap.get(syscall, None)):
importTableSyscallNames.add(syscallmap[syscall])
print("------- main -------")
i = 0
while (i < 400):
if i not in masterSyscalls:
if (syscallmap.get(i, None)):
blMasterSyscallNames.add(syscallmap[i])
i += 1
for syscall in masterSyscalls:
if (syscallmap.get(syscall, None)):
masterSyscallNames.add(syscallmap[syscall])
print(syscallmap.get(syscall, ""))
print("------- child -------")
i = 0
while (i < 400):
if i not in workerSyscalls:
if (syscallmap.get(i, None)):
blWorkerSyscallNames.add(syscallmap[i])
i += 1
for syscall in workerSyscalls:
if (syscallmap.get(syscall, None)):
workerSyscallNames.add(syscallmap[syscall])
print(syscallmap.get(syscall, ""))
if (binprofiler):
outputDict = dict()
outputDict['importTable'] = importTableSyscallNames
outputDict['master'] = masterSyscallNames
outputDict['worker'] = workerSyscallNames
outputDict['blImportTable'] = blImportTableSyscallNames
outputDict['blMaster'] = blMasterSyscallNames
outputDict['blWorker'] = blWorkerSyscallNames
util.writeDictToFile(outputDict, output)
'''
mminusw = sorted(mainset.difference(workerset))
wminusm = sorted(workerset.difference(mainset))
#mminusc = sorted(mainset.difference(cachemgrset))
#cminusm = sorted(cachemgrset.difference(mainset))
rootLogger = setLogPath("graph.log")
translator = sycall.Syscall(rootLogger)
syscallmap = translator.createMap()
'''
print("------- main minus worker -------")
for syscall in mminusw:
print(syscallmap[syscall])
print("------- worker minus main -------")
for syscall in wminusm:
print(syscallmap[syscall])