def a2b(results, stringTraces, traceInvalid): """ Supposed to identify candidate functions that are linked by a direct edge. Yet to be implemented. """ from tools import sortDictionary funcs, scores = sortDictionary(results) funcs = funcs[::-1] scores = scores[::-1] # find connection between identified functions i = 0 while scores[i] != 0.0: i += 1 funcs = funcs[:i] scores = scores[:i] chains = [] chainsDict = {} # TODO: Finish me! i = 0 while i < len(funcs): neighbors = traceInvalid.neighbors() func = funcs[i] while len(neighbors) != 0: pass i += 1
def getDigraph(self): from tools import sortDictionary digr = Trace.getDigraph(self) for bb in self.way: calls = sortDictionary(bb.calls)[1] digr.add_node_attribute(bb.startAddr, ("calls",calls)) digr.add_node_attribute(bb.startAddr, ("type", bb.type)) return digr
def _a3core(seqs, seqRef): from tools import sortDictionary scoresIndexes = {} for seq in seqs: rangesCommon = seqRef.getCommonRanges(seq)[0] # add sentinel range rangesCommon.append([seqRef.len]) for i in range(len(rangesCommon)-1): r = rangesCommon[i] baseScore = rangesCommon[i+1][0] - r[-1] for i in range(len(r)): index = r[i] if index not in scoresIndexes: scoresIndexes[index] = 0 scoresIndexes[index] += baseScore + i scores = {seqRef.rawSeq[index] : scoresIndexes[index] for index in scoresIndexes} normalizedScores = _Helper.normalizeScores(scores) keys, values = sortDictionary(normalizedScores) # print keys[::-1] # print values[::-1] return normalizedScores
def A_WEAZEL_BB(protoRuns): """ Sub-algorithm of A-WEAZEL analyzing bb-traces of decision-functions. @param protoRuns: The basic-block protocol-runs to analyze. @type protoRuns: List of ProtocolRunBb @return: A graph of decision and implementation bbs @rtype: List of DecisionBb """ # Which node is present in which protocol-run? notEmptyProtoRuns = [protoRun for protoRun in protoRuns if protoRun.diGraph is not None] if len(notEmptyProtoRuns) == 0: print "Got only empty proto-runs in A_WEAZEL_BB. There seems to be something wrong with your tracer (srsly)." return None allNodesWeight = {} for protoRun in notEmptyProtoRuns: for node in protoRun.diGraph.nodes(): if node not in allNodesWeight: allNodesWeight[node] = 0 allNodesWeight[node] += 1 # Now get the nodes with the least weight for each protocol-run. foundImplementation = False decisionNodes = {} for protoRun in notEmptyProtoRuns: nodesWeight = {} for node in protoRun.diGraph.nodes(): nodesWeight[node] = allNodesWeight[node] nodes, weights = sortDictionary(nodesWeight) minWeight = weights[0] nodesMinWeight = [nodes[i] for i in range(len(weights)) if weights[i] == minWeight] # Find all nodes with higher weight that lead to the minimal weight nodes. for nodeMW in nodesMinWeight: nodeMWIncidents = protoRun.diGraph.incidents(nodeMW) for nodeMWIncident in nodeMWIncidents: if nodesWeight[nodeMWIncident] > minWeight: if not nodeMWIncident in decisionNodes: decisionNodes[nodeMWIncident] = DecisionBb(nodeMWIncident) foundImplementation = True decisionNodes[nodeMWIncident].addEdge(protoRun, ImplementationBb(nodeMW)) # Finally add those decision nodes in which not all possible decisions were made during runtime. class IncompleteDecisionBbCandidate: def __init__(self, addr, nExits): self.addr = addr self.nExits = nExits is not None and nExits or 0 self.exits = [] def addExit(self, exit): if exit not in self.exits: self.exits.append(exit) def isCompletelyDefined(self): return len(self.exits) == self.nExits incompleteDecisionBbCandidates = {} for protoRun in notEmptyProtoRuns: for bb in protoRun.trace.basicBlocks.values(): if not bb.isCompletelyDefined(): if bb.startAddr not in incompleteDecisionBbCandidates: incompleteDecisionBbCandidates[bb.startAddr] = IncompleteDecisionBbCandidate(bb.startAddr, bb.nExits) for knownExit in bb.knownExits.values(): incompleteDecisionBbCandidates[bb.startAddr].addExit(knownExit.bb.startAddr) elif bb.startAddr in incompleteDecisionBbCandidates: incompleteDecisionBbCandidates.pop(bb.startAddr) for decisionBb in incompleteDecisionBbCandidates.values(): if not decisionBb.isCompletelyDefined(): if decisionBb.addr not in decisionNodes: decisionNodes[decisionBb.addr] = DecisionBb(decisionBb.addr) decisionNodes[decisionBb.addr].setAttribute(DecisionBb.SPECIFIC_ATTRIBUTES.SUSPICIOUS_EDGES, "%d" % (decisionBb.nExits - len(decisionBb.exits))) return foundImplementation, decisionNodes.values()
def A_WEAZEL(self, protoRuns, fileIoAction=None, basePathTraceData=None, recursionDepth=0, funcsToExclude=[], funcsToInclude=[]): """ The final algorithm for the identification of decision and handling functionality. @todo: unify variable naming ("name" instead of "addr" etc.) @rtype: List of DecisionFunc """ class Group: def __init__(self, exFunc): self.decisionFunc = exFunc.callStack[-2] self.refExFunc = exFunc self.exFuncs = [exFunc] self.protoRuns = [exFunc.protoRun] self.closed = False def addExclusiveFunction(self, exFunc): """ Adds an exclusive function to the group. It is checked whether the given ex-func fits into the group. @param exFunc: The ex-func to add @return: Boolean flag indicating success. """ if self.closed: return False if self.refExFunc.callStackSingatureLength > len(exFunc.callStack): return False # Check if the ex-func shares a call-stack with the reference trace. for i in range(self.refExFunc.callStackSingatureLength): if self.refExFunc.callStack[-(i+1)] != exFunc.callStack[-(i+1)]: return False if exFunc not in self.exFuncs: self.exFuncs.append(exFunc) if exFunc.protoRun not in self.protoRuns: self.protoRuns.append(exFunc.protoRun) return True def close(self): """ Closes the group and updates call-stacks of protocol-runs. """ self.closed = True self.refCallStack = self.refExFunc.callStack[-self.refExFunc.callStackSingatureLength:] def getRefCallStackDecisionFunc(self): """ @return: List (possibly empty) in case the decision func has a callstack, None in case the decision func has no callstack at all. """ if not self.closed: return None return self.refCallStack[:-2] def getRefCallStackExclusiveFunc(self): if not self.closed: return None return self.refCallStack[:-1] def dominatesRefCallStackDecisionFunc(self, otherGroup): ownCallStack = self.getRefCallStackDecisionFunc() otherCallStack = otherGroup.getRefCallStackDecisionFunc() if ownCallStack is None or otherCallStack is None: return ownCallStack == otherCallStack if len(ownCallStack) < len(otherCallStack): return False for i in range(len(otherCallStack)): if ownCallStack[-(i+1)] != otherCallStack[-(i+1)]: return False return True def isDecisionFuncTop(self): # TODO: Does this work with Pure-FTPd? return (len(self.refExFunc.callStack) == 2) class ExclusiveFunction: def __init__(self, name, callStack, callStackSingatureLength, protoRun): """ @param name: The addr/name of the sub-function @param callStack: The respective call-stack (without the sub-function itself) @param callStackSingatureLength: Number of functions on the call-stack that need to be checked in order to identify this specific sub-function call. @param protoRun: The respective protocol-run """ self.name = name self.callStack = protoRun.callStack + callStack #TODO: was old code faulty or good for something? self.protoRun = protoRun self.callStackSingatureLength = callStackSingatureLength # Group proto-runs according to exclusive function calls. sCommonFuncs = sets.Set(protoRuns[0].diGraph.nodes()) for protoRun in protoRuns[1:]: sCommonFuncs = sCommonFuncs.intersection(protoRun.diGraph.nodes()) # create set of functions to ignore sExcludeFuncs = sets.Set(funcsToExclude) sIncludeFuncs = sets.Set(funcsToInclude) # now determine exclusive functions exFuncs = [] for protoRun in protoRuns: sFuncs = sets.Set(protoRun.diGraph.nodes()) sExFuncs = (sFuncs - (sCommonFuncs - sIncludeFuncs)) - sExcludeFuncs # remove the starting node of each run sExFuncs.discard(protoRun.trace.startingNode) # first get sub-traces for all identified exclusive functions topExFuncs = {} for exFunc in sExFuncs: ways = protoRun.trace.getWaysBetween(None, exFunc) # filter out functions dominated by other function waysUndominated = [] for way in ways: wayDominated = False for func in way[:-1]: if func in sExFuncs: wayDominated = True break if not wayDominated: waysUndominated.append(way) if len(waysUndominated) > 0: topExFuncs[exFunc] = waysUndominated # DEBUG #print "\t" * recursionDepth + "top-ex-funcs:" #for x in topExFuncs: print "\t" * recursionDepth + "%x" % x ####### for topExFunc in topExFuncs: # Find out how many levels of call-stack back tracing are needed in order to unambiguously differentiate between all ways. ways = topExFuncs[topExFunc] for iWay in range(len(ways)): # get a set of all indexes without iWay tmp = range(len(ways)) tmp.pop(iWay) sIOtherWays = sets.Set(tmp) way = ways[iWay] # walk the callstack backwards and check for differences level = 0 for i in range(len(way)-1)[::-1]: level += 1 # are there still identical callstacks if len(sIOtherWays) == 0: break # check for all remaining callstacks iOtherWays = list(sIOtherWays) for iOtherWay in iOtherWays: otherWay = ways[iOtherWay] if i not in range(len(otherWay)) or otherWay[i] != way[i]: sIOtherWays.remove(iOtherWay) callStack = way exFuncs.append(ExclusiveFunction(topExFunc, callStack, level, protoRun)) # now group the identified exclusive functions ## sort exFuncs according to the respective call-stack signature lengths tmp = {i:exFuncs[i].callStackSingatureLength for i in range(len(exFuncs))} import tools exFuncIndexes, nop = tools.sortDictionary(tmp) groups = [] decisionFuncs = {} for iExFunc in exFuncIndexes[::-1]: exFunc = exFuncs[iExFunc] fitsInExistingGroup = False for group in groups: fitsInExistingGroup = group.addExclusiveFunction(exFunc) if fitsInExistingGroup: # DEBUG # print "\t" * recursionDepth + "Ex-func %x fits in group of ex-func %x (df %x)" % (exFunc.name, group.refExFunc.name, group.decisionFunc) ####### break # If the current ex-func does not fit into any existing group, create a new group for it. if not fitsInExistingGroup: newGroup = Group(exFunc) groups.append(newGroup) if newGroup.decisionFunc not in decisionFuncs: decisionFuncs[newGroup.decisionFunc] = DecisionFunc(newGroup.decisionFunc) decisionFuncs[newGroup.decisionFunc].addGroup(newGroup) # DEBUG # print "\t"*recursionDepth + "Identified the following decision funcs:" # for df in decisionFuncs: # print "\t"*recursionDepth + "\t%x" % df ####### # Finally analyze the decision functions. for df in decisionFuncs.values(): bbProtoRuns = [] # Analyze each group. # NOTE: That groups are already ordered by the length of their call-stack traces! # This is important for deciding if a decision func needs to be traced on bb level for a certain group or not. for iGroup in range(len(df.groups)): group = df.groups[iGroup] group.close() # If an ex-func is only accessed by a single protocol-run, we assume the ex-func to be a concrete implementation function (e.g. a cmd handler). # Recursion ends in this case. if len(group.protoRuns) == 1: df.addEdge(group.protoRuns[0], ImplementationFunc(group.refExFunc.name)) else: subProtoRuns = [] for exFunc in group.exFuncs: # TODO: The following should return traces not including the given call-stack subCgTraces = exFunc.protoRun.trace.getTracesWithCallStack(exFunc.callStack) subCgTrace = subCgTraces[0] # Note how we necessarily always get exactly one subtrace here. subCallStack = group.getRefCallStackExclusiveFunc() subProtoRun = protocols.protocol.RunFunc(exFunc.protoRun.protoProc, subCgTrace, subCallStack) subProtoRuns.append(subProtoRun) addrExFunc = group.refExFunc.name # Recursively invoke the algorithm for all groups. # DEBUG # print "\t"* recursionDepth + "Invoking A-WEAZEL for exfunc %x with %d proto-runs." % (addrExFunc, len(subProtoRuns)) ####### processingGraphs = self.A_WEAZEL(subProtoRuns, fileIoAction, basePathTraceData, recursionDepth+1, funcsToInclude=funcsToInclude, funcsToExclude=funcsToExclude) # check if any sub-processing graphs were identified if len(processingGraphs) == 0: # if not, the ex-func is a multi protocol run implementation function multiIf = ImplementationFunc(addrExFunc) # get protocol processors protoProcs = sets.Set() for protoRun in subProtoRuns: protoProcs.add(protoRun.protoProc) implBbProtoRuns = self._getBasicBlockTraces(list(protoProcs), addrExFunc, group.getRefCallStackExclusiveFunc(), fileIoAction, basePathTraceData) isDecisionFunc, bbGraphs = Wiesel.A_WEAZEL_BB(implBbProtoRuns) multiIf.addBbGraphs(bbGraphs, implBbProtoRuns) processingGraphs = [multiIf] # check if the ex-func is a 'special' one (i.e. was found to exhibit different return values for different protocol runs) if addrExFunc in funcsToInclude and addrExFunc not in funcsToExclude: # if so, check if the ex-func was already identified as decider alreadyIdentified = False for topDecisionFunc in processingGraphs: if topDecisionFunc.addr == addrExFunc: alreadyIdentified = True break if not alreadyIdentified: # if it was not already identified, make it the top node and record bb traces intermediateDf = DecisionFunc(addrExFunc) intermediateDf.addEdges(subProtoRuns, processingGraphs) intermediateBbProtoRuns = self._getBasicBlockTracesAWeazel(subProtoRuns, intermediateDf.addr, group.getRefCallStackExclusiveFunc(), fileIoAction, basePathTraceData) # TODO: aggregate bb-traces r = Wiesel.A_WEAZEL_BB(intermediateBbProtoRuns) if r is not None: isRealDecisionFunc, bbGraphs = r intermediateDf.addBbGraphs(bbGraphs, intermediateBbProtoRuns) intermediateDf.setAttribute(DecisionFunc.SPECIFIC_ATTRIBUTES.IMPORTANT, str(isRealDecisionFunc)) # swap processing graphs processingGraphs = [intermediateDf] # add the generated processing graphs to the node of the decision function df.addEdges(subProtoRuns, processingGraphs) # Check if the decision-func was already traced for the given call-stack callStackAlreadyTraced = False for iPrevGroup in range(iGroup): if df.groups[iPrevGroup].dominatesRefCallStackDecisionFunc(group): callStackAlreadyTraced = True if not callStackAlreadyTraced and not df.addr in funcsToExclude: # Aggregate traces of all protocol-runs through the decision-function bbProtoRuns += self._getBasicBlockTracesAWeazel(protoRuns, df.addr, group.getRefCallStackDecisionFunc(), fileIoAction, basePathTraceData) r = Wiesel.A_WEAZEL_BB(bbProtoRuns) if r is not None: isRealDecisionFunc, bbGraphs = r df.addBbGraphs(bbGraphs, bbProtoRuns) df.setAttribute(DecisionFunc.SPECIFIC_ATTRIBUTES.IMPORTANT, str(isRealDecisionFunc)) else: print "Failed to analyze %x on bb-level." % df.addr return decisionFuncs.values()