def a2b(results, stringTraces, traceInvalid):
"""
Supposed to identify candidate functions that are linked by a direct edge.
Yet to be implemented.
"""
from tools import sortDictionary
funcs, scores = sortDictionary(results)
funcs = funcs[::-1]
scores = scores[::-1]
# find connection between identified functions
i = 0
while scores[i] != 0.0:
i += 1
funcs = funcs[:i]
scores = scores[:i]
chains = []
chainsDict = {}
# TODO: Finish me!
i = 0
while i < len(funcs):
neighbors = traceInvalid.neighbors()
func = funcs[i]
while len(neighbors) != 0:
pass
i += 1
def getDigraph(self):
from tools import sortDictionary
digr = Trace.getDigraph(self)
for bb in self.way:
calls = sortDictionary(bb.calls)[1]
digr.add_node_attribute(bb.startAddr, ("calls",calls))
digr.add_node_attribute(bb.startAddr, ("type", bb.type))
return digr
def _a3core(seqs, seqRef):
from tools import sortDictionary
scoresIndexes = {}
for seq in seqs:
rangesCommon = seqRef.getCommonRanges(seq)[0]
# add sentinel range
rangesCommon.append([seqRef.len])
for i in range(len(rangesCommon)-1):
r = rangesCommon[i]
baseScore = rangesCommon[i+1][0] - r[-1]
for i in range(len(r)):
index = r[i]
if index not in scoresIndexes:
scoresIndexes[index] = 0
scoresIndexes[index] += baseScore + i
scores = {seqRef.rawSeq[index] : scoresIndexes[index] for index in scoresIndexes}
normalizedScores = _Helper.normalizeScores(scores)
keys, values = sortDictionary(normalizedScores)
# print keys[::-1]
# print values[::-1]
return normalizedScores
def A_WEAZEL_BB(protoRuns):
"""
Sub-algorithm of A-WEAZEL analyzing bb-traces of decision-functions.
@param protoRuns: The basic-block protocol-runs to analyze.
@type protoRuns: List of ProtocolRunBb
@return: A graph of decision and implementation bbs
@rtype: List of DecisionBb
"""
# Which node is present in which protocol-run?
notEmptyProtoRuns = [protoRun for protoRun in protoRuns if protoRun.diGraph is not None]
if len(notEmptyProtoRuns) == 0:
print "Got only empty proto-runs in A_WEAZEL_BB. There seems to be something wrong with your tracer (srsly)."
return None
allNodesWeight = {}
for protoRun in notEmptyProtoRuns:
for node in protoRun.diGraph.nodes():
if node not in allNodesWeight:
allNodesWeight[node] = 0
allNodesWeight[node] += 1
# Now get the nodes with the least weight for each protocol-run.
foundImplementation = False
decisionNodes = {}
for protoRun in notEmptyProtoRuns:
nodesWeight = {}
for node in protoRun.diGraph.nodes():
nodesWeight[node] = allNodesWeight[node]
nodes, weights = sortDictionary(nodesWeight)
minWeight = weights[0]
nodesMinWeight = [nodes[i] for i in range(len(weights)) if weights[i] == minWeight]
# Find all nodes with higher weight that lead to the minimal weight nodes.
for nodeMW in nodesMinWeight:
nodeMWIncidents = protoRun.diGraph.incidents(nodeMW)
for nodeMWIncident in nodeMWIncidents:
if nodesWeight[nodeMWIncident] > minWeight:
if not nodeMWIncident in decisionNodes:
decisionNodes[nodeMWIncident] = DecisionBb(nodeMWIncident)
foundImplementation = True
decisionNodes[nodeMWIncident].addEdge(protoRun, ImplementationBb(nodeMW))
# Finally add those decision nodes in which not all possible decisions were made during runtime.
class IncompleteDecisionBbCandidate:
def __init__(self, addr, nExits):
self.addr = addr
self.nExits = nExits is not None and nExits or 0
self.exits = []
def addExit(self, exit):
if exit not in self.exits:
self.exits.append(exit)
def isCompletelyDefined(self):
return len(self.exits) == self.nExits
incompleteDecisionBbCandidates = {}
for protoRun in notEmptyProtoRuns:
for bb in protoRun.trace.basicBlocks.values():
if not bb.isCompletelyDefined():
if bb.startAddr not in incompleteDecisionBbCandidates:
incompleteDecisionBbCandidates[bb.startAddr] = IncompleteDecisionBbCandidate(bb.startAddr, bb.nExits)
for knownExit in bb.knownExits.values():
incompleteDecisionBbCandidates[bb.startAddr].addExit(knownExit.bb.startAddr)
elif bb.startAddr in incompleteDecisionBbCandidates:
incompleteDecisionBbCandidates.pop(bb.startAddr)
for decisionBb in incompleteDecisionBbCandidates.values():
if not decisionBb.isCompletelyDefined():
if decisionBb.addr not in decisionNodes:
decisionNodes[decisionBb.addr] = DecisionBb(decisionBb.addr)
decisionNodes[decisionBb.addr].setAttribute(DecisionBb.SPECIFIC_ATTRIBUTES.SUSPICIOUS_EDGES, "%d" % (decisionBb.nExits - len(decisionBb.exits)))
return foundImplementation, decisionNodes.values()
def A_WEAZEL(self, protoRuns, fileIoAction=None, basePathTraceData=None, recursionDepth=0, funcsToExclude=[], funcsToInclude=[]):
"""
The final algorithm for the identification of decision and handling functionality.
@todo: unify variable naming ("name" instead of "addr" etc.)
@rtype: List of DecisionFunc
"""
class Group:
def __init__(self, exFunc):
self.decisionFunc = exFunc.callStack[-2]
self.refExFunc = exFunc
self.exFuncs = [exFunc]
self.protoRuns = [exFunc.protoRun]
self.closed = False
def addExclusiveFunction(self, exFunc):
"""
Adds an exclusive function to the group. It is checked whether the given ex-func fits into the group.
@param exFunc: The ex-func to add
@return: Boolean flag indicating success.
"""
if self.closed:
return False
if self.refExFunc.callStackSingatureLength > len(exFunc.callStack):
return False
# Check if the ex-func shares a call-stack with the reference trace.
for i in range(self.refExFunc.callStackSingatureLength):
if self.refExFunc.callStack[-(i+1)] != exFunc.callStack[-(i+1)]:
return False
if exFunc not in self.exFuncs:
self.exFuncs.append(exFunc)
if exFunc.protoRun not in self.protoRuns:
self.protoRuns.append(exFunc.protoRun)
return True
def close(self):
"""
Closes the group and updates call-stacks of protocol-runs.
"""
self.closed = True
self.refCallStack = self.refExFunc.callStack[-self.refExFunc.callStackSingatureLength:]
def getRefCallStackDecisionFunc(self):
"""
@return: List (possibly empty) in case the decision func has a callstack, None in case the decision func has no callstack at all.
"""
if not self.closed:
return None
return self.refCallStack[:-2]
def getRefCallStackExclusiveFunc(self):
if not self.closed:
return None
return self.refCallStack[:-1]
def dominatesRefCallStackDecisionFunc(self, otherGroup):
ownCallStack = self.getRefCallStackDecisionFunc()
otherCallStack = otherGroup.getRefCallStackDecisionFunc()
if ownCallStack is None or otherCallStack is None:
return ownCallStack == otherCallStack
if len(ownCallStack) < len(otherCallStack):
return False
for i in range(len(otherCallStack)):
if ownCallStack[-(i+1)] != otherCallStack[-(i+1)]:
return False
return True
def isDecisionFuncTop(self):
# TODO: Does this work with Pure-FTPd?
return (len(self.refExFunc.callStack) == 2)
class ExclusiveFunction:
def __init__(self, name, callStack, callStackSingatureLength, protoRun):
"""
@param name: The addr/name of the sub-function
@param callStack: The respective call-stack (without the sub-function itself)
@param callStackSingatureLength: Number of functions on the call-stack that need to be checked in order to identify this specific sub-function call.
@param protoRun: The respective protocol-run
"""
self.name = name
self.callStack = protoRun.callStack + callStack #TODO: was old code faulty or good for something?
self.protoRun = protoRun
self.callStackSingatureLength = callStackSingatureLength
# Group proto-runs according to exclusive function calls.
sCommonFuncs = sets.Set(protoRuns[0].diGraph.nodes())
for protoRun in protoRuns[1:]:
sCommonFuncs = sCommonFuncs.intersection(protoRun.diGraph.nodes())
# create set of functions to ignore
sExcludeFuncs = sets.Set(funcsToExclude)
sIncludeFuncs = sets.Set(funcsToInclude)
# now determine exclusive functions
exFuncs = []
for protoRun in protoRuns:
sFuncs = sets.Set(protoRun.diGraph.nodes())
sExFuncs = (sFuncs - (sCommonFuncs - sIncludeFuncs)) - sExcludeFuncs
# remove the starting node of each run
sExFuncs.discard(protoRun.trace.startingNode)
# first get sub-traces for all identified exclusive functions
topExFuncs = {}
for exFunc in sExFuncs:
ways = protoRun.trace.getWaysBetween(None, exFunc)
# filter out functions dominated by other function
waysUndominated = []
for way in ways:
wayDominated = False
for func in way[:-1]:
if func in sExFuncs:
wayDominated = True
break
if not wayDominated:
waysUndominated.append(way)
if len(waysUndominated) > 0:
topExFuncs[exFunc] = waysUndominated
# DEBUG
#print "\t" * recursionDepth + "top-ex-funcs:"
#for x in topExFuncs: print "\t" * recursionDepth + "%x" % x
#######
for topExFunc in topExFuncs:
# Find out how many levels of call-stack back tracing are needed in order to unambiguously differentiate between all ways.
ways = topExFuncs[topExFunc]
for iWay in range(len(ways)):
# get a set of all indexes without iWay
tmp = range(len(ways))
tmp.pop(iWay)
sIOtherWays = sets.Set(tmp)
way = ways[iWay]
# walk the callstack backwards and check for differences
level = 0
for i in range(len(way)-1)[::-1]:
level += 1
# are there still identical callstacks
if len(sIOtherWays) == 0:
break
# check for all remaining callstacks
iOtherWays = list(sIOtherWays)
for iOtherWay in iOtherWays:
otherWay = ways[iOtherWay]
if i not in range(len(otherWay)) or otherWay[i] != way[i]:
sIOtherWays.remove(iOtherWay)
callStack = way
exFuncs.append(ExclusiveFunction(topExFunc, callStack, level, protoRun))
# now group the identified exclusive functions
## sort exFuncs according to the respective call-stack signature lengths
tmp = {i:exFuncs[i].callStackSingatureLength for i in range(len(exFuncs))}
import tools
exFuncIndexes, nop = tools.sortDictionary(tmp)
groups = []
decisionFuncs = {}
for iExFunc in exFuncIndexes[::-1]:
exFunc = exFuncs[iExFunc]
fitsInExistingGroup = False
for group in groups:
fitsInExistingGroup = group.addExclusiveFunction(exFunc)
if fitsInExistingGroup:
# DEBUG
# print "\t" * recursionDepth + "Ex-func %x fits in group of ex-func %x (df %x)" % (exFunc.name, group.refExFunc.name, group.decisionFunc)
#######
break
# If the current ex-func does not fit into any existing group, create a new group for it.
if not fitsInExistingGroup:
newGroup = Group(exFunc)
groups.append(newGroup)
if newGroup.decisionFunc not in decisionFuncs:
decisionFuncs[newGroup.decisionFunc] = DecisionFunc(newGroup.decisionFunc)
decisionFuncs[newGroup.decisionFunc].addGroup(newGroup)
# DEBUG
# print "\t"*recursionDepth + "Identified the following decision funcs:"
# for df in decisionFuncs:
# print "\t"*recursionDepth + "\t%x" % df
#######
# Finally analyze the decision functions.
for df in decisionFuncs.values():
bbProtoRuns = []
# Analyze each group.
# NOTE: That groups are already ordered by the length of their call-stack traces!
# This is important for deciding if a decision func needs to be traced on bb level for a certain group or not.
for iGroup in range(len(df.groups)):
group = df.groups[iGroup]
group.close()
# If an ex-func is only accessed by a single protocol-run, we assume the ex-func to be a concrete implementation function (e.g. a cmd handler).
# Recursion ends in this case.
if len(group.protoRuns) == 1:
df.addEdge(group.protoRuns[0], ImplementationFunc(group.refExFunc.name))
else:
subProtoRuns = []
for exFunc in group.exFuncs:
# TODO: The following should return traces not including the given call-stack
subCgTraces = exFunc.protoRun.trace.getTracesWithCallStack(exFunc.callStack)
subCgTrace = subCgTraces[0] # Note how we necessarily always get exactly one subtrace here.
subCallStack = group.getRefCallStackExclusiveFunc()
subProtoRun = protocols.protocol.RunFunc(exFunc.protoRun.protoProc, subCgTrace, subCallStack)
subProtoRuns.append(subProtoRun)
addrExFunc = group.refExFunc.name
# Recursively invoke the algorithm for all groups.
# DEBUG
# print "\t"* recursionDepth + "Invoking A-WEAZEL for exfunc %x with %d proto-runs." % (addrExFunc, len(subProtoRuns))
#######
processingGraphs = self.A_WEAZEL(subProtoRuns, fileIoAction, basePathTraceData, recursionDepth+1, funcsToInclude=funcsToInclude, funcsToExclude=funcsToExclude)
# check if any sub-processing graphs were identified
if len(processingGraphs) == 0:
# if not, the ex-func is a multi protocol run implementation function
multiIf = ImplementationFunc(addrExFunc)
# get protocol processors
protoProcs = sets.Set()
for protoRun in subProtoRuns: protoProcs.add(protoRun.protoProc)
implBbProtoRuns = self._getBasicBlockTraces(list(protoProcs), addrExFunc, group.getRefCallStackExclusiveFunc(), fileIoAction, basePathTraceData)
isDecisionFunc, bbGraphs = Wiesel.A_WEAZEL_BB(implBbProtoRuns)
multiIf.addBbGraphs(bbGraphs, implBbProtoRuns)
processingGraphs = [multiIf]
# check if the ex-func is a 'special' one (i.e. was found to exhibit different return values for different protocol runs)
if addrExFunc in funcsToInclude and addrExFunc not in funcsToExclude:
# if so, check if the ex-func was already identified as decider
alreadyIdentified = False
for topDecisionFunc in processingGraphs:
if topDecisionFunc.addr == addrExFunc:
alreadyIdentified = True
break
if not alreadyIdentified:
# if it was not already identified, make it the top node and record bb traces
intermediateDf = DecisionFunc(addrExFunc)
intermediateDf.addEdges(subProtoRuns, processingGraphs)
intermediateBbProtoRuns = self._getBasicBlockTracesAWeazel(subProtoRuns, intermediateDf.addr, group.getRefCallStackExclusiveFunc(), fileIoAction, basePathTraceData)
# TODO: aggregate bb-traces
r = Wiesel.A_WEAZEL_BB(intermediateBbProtoRuns)
if r is not None:
isRealDecisionFunc, bbGraphs = r
intermediateDf.addBbGraphs(bbGraphs, intermediateBbProtoRuns)
intermediateDf.setAttribute(DecisionFunc.SPECIFIC_ATTRIBUTES.IMPORTANT, str(isRealDecisionFunc))
# swap processing graphs
processingGraphs = [intermediateDf]
# add the generated processing graphs to the node of the decision function
df.addEdges(subProtoRuns, processingGraphs)
# Check if the decision-func was already traced for the given call-stack
callStackAlreadyTraced = False
for iPrevGroup in range(iGroup):
if df.groups[iPrevGroup].dominatesRefCallStackDecisionFunc(group):
callStackAlreadyTraced = True
if not callStackAlreadyTraced and not df.addr in funcsToExclude:
# Aggregate traces of all protocol-runs through the decision-function
bbProtoRuns += self._getBasicBlockTracesAWeazel(protoRuns, df.addr, group.getRefCallStackDecisionFunc(), fileIoAction, basePathTraceData)
r = Wiesel.A_WEAZEL_BB(bbProtoRuns)
if r is not None:
isRealDecisionFunc, bbGraphs = r
df.addBbGraphs(bbGraphs, bbProtoRuns)
df.setAttribute(DecisionFunc.SPECIFIC_ATTRIBUTES.IMPORTANT, str(isRealDecisionFunc))
else:
print "Failed to analyze %x on bb-level." % df.addr
return decisionFuncs.values()
