def doOneFunctionFileRW(workerParams):
sourcesList3k = workerParams['sourcesList3k']
tarGraph = workerParams['tarGraph']
refGraph = workerParams['refGraph']
exeName = workerParams['exeName']
funcfilename = workerParams['funcFileName']
gradesCache = []
timeout = [False,time.time()]
def toggleTimeout():
timeout[0] = True
# wait 3 hours
timeOutTimer = threading.Timer(timeOutInSecs,toggleTimeout)
timeOutTimer.start()
allFields = []
if len(refGraph.vs) > 3000:
print "skipping cuz > 3000 - func=" + funcfilename + ",exe-" + exeName
return allFields
# startVal of hack - only take the last 2 confs with k=3 (trace = true OR false)
#bestKSolverDeltas = KBest(KToKeep,lambda x: 100-x['delta'],lambda x:x['delta'])
#bestKSolverBroken = KBest(KToKeep,lambda x: 1000-x['broken'],lambda x:x['broken'])
GradeSystemsInfoInst = GradeSystemsInfo(sourcesList3k, gradeSystemNames)
startForXlsRecord = time.time()
#this cache could probebly get optimized
for refvIndex, refv in enumerate(refGraph.vs):
gradesCache.append([])
#gradesCache[refv['id']] = {}
for tarvIndex, tarv in enumerate(tarGraph.vs):
gradesDict = getMatchedCmdsWithGrade(refv['code'],tarv['code'])
#for key in grade.keys():
gradesCache[refvIndex].append(gradesDict)
#gradesCache[refvIndex][tarvIndex] = grade
objectsCount = 0
for g in split2k.ksplitGraphObject(myK,tarGraph,True):
if timeout[0] == True:
print "Breaking out cuz of timeout, time=" + str(int(time.time()-timeout[1])) + " from - func=" + funcfilename + ",exe-" + exeName
return []
objectsCount += 1
for refGraphletContainer in GradeSystemsInfoInst:
refGraphlet = refGraphletContainer.getObject()
(isIso,Map12,Map21) = refGraphlet.isomorphic_vf2(g,None,None,None,None,True,False,None,None)
if isIso:
nodeGradesInfos = []
for (x,y) in itertools.imap(lambda x,y: (int(x['id']),int(y['id'])),map(lambda x:refGraphlet.vs[x],range(0,len(refGraphlet.vs))),map(lambda x:g.vs[Map12[x]],range(0,len(g.vs))) ):
nodeGradesInfos.append(gradesCache[x][y])
gradeSystemsFinals = OrderedDict()
for methodType in ['contain','ratio']:
gradeSystemsFinals[methodType] = numpy.mean(map(lambda nodeGradesInfo: nodeGradesInfo['gradesDict'][methodType],nodeGradesInfos))
if "Solver"=="Solver":
if gradeSystemsFinals['ratio'] > 50:
conDict = GraphletsConstraints(nodeGradesInfos)
newTarNodes = list(conDict.getRW())
refNodes = map(lambda x:x['refCode'],nodeGradesInfos)
grades = map(compareDisAsmCode,refNodes,newTarNodes)
for methodType in ['contain','ratio']:
gradeSystemsFinals['Solver_' + methodType] = numpy.mean(map(lambda x:x[methodType],grades))
#delta = int(gradeSystemsFinals['Solver_' + methodType] - gradeSystemsFinals[methodType])
#refFullCode = ";".join(refNodes)
#tarFullCOde = ";".join(newTarNodes)
#bestKSolverDeltas.insert({'delta':delta,'ref':refFullCode,'tar':tarFullCOde})
#bestKSolverBroken.insert({'broken':conDict.getBrokenNumber(),'ref':refFullCode,'tar':tarFullCOde})
#gradeSystemsDict[gradeSystemName]['bestSolImprovmentList'].insert({'delta':delta,'ref':refFullCode,'tar':tarFullCOde}
else:
for methodType in ['contain','ratio']:
gradeSystemsFinals['Solver_' + methodType] = gradeSystemsFinals[methodType]
else:
for methodType in ['contain','ratio']:
gradeSystemsFinals['Solver_' + methodType] = numpy.mean(map(lambda nodeGradesInfo: nodeGradesInfo['gradesDict'][methodType],nodeGradesInfos))
for gradeSystemName in gradeSystemNames:
refGraphletContainer[gradeSystemName] = max(gradeSystemsFinals[gradeSystemName],refGraphletContainer[gradeSystemName])
endForXlsRecord = time.time()
for gradeSystemName in gradeSystemNames:
counters = GradeSystemsInfoInst.tallyForGradeSystem(gradeSystemName,borders)
#counters = gradeSystemsDict[gradeSystemName]['DictCounter'].tallyCounters()
#["exe","funcname","#objects","K","Trace","System","Generator","GradeSystem","TimeStr","TimeNumber"]
fields = [exeName,funcfilename,str(objectsCount),str(myK),"TRUE",methodSystemName,"normal",gradeSystemName,
"\""+getDurationStr(int(endForXlsRecord-startForXlsRecord))+"\"",str(endForXlsRecord-startForXlsRecord)]
for i in range(0,len(borders)):
fields.append("%.4f" % (float(counters[i]) / float(len(GradeSystemsInfoInst))))
"""
if (gradeSystemName == "Solver_contain"):
fields.append(str(bestKSolverDeltas.max()))
fields.append(str(bestKSolverDeltas))
fields.append(str(bestKSolverBroken.max()))
fields.append(str(bestKSolverBroken))
else:
for i in range(0,4):
fields.append("see Solver_contain")
sets = GradeSystemsInfoInst.detectPatchForGradeSystem(gradeSystemName,[60,70,80])
sets['total'] = len(tarGraph.vs)
fields.append(str(sets))
"""
allFields.append(fields)
timeOutTimer.cancel()
del timeOutTimer
return allFields
def doOneFunctionFileRW(workerParams):
sourcesList3k = workerParams['sourcesList3k']
tarGraph = workerParams['tarGraph']
refGraph = workerParams['refGraph']
exeName = workerParams['exeName']
funcfilename = workerParams['funcFileName']
gradesCache = []
timeout = [False, time.time()]
def toggleTimeout():
timeout[0] = True
# wait 3 hours
timeOutTimer = threading.Timer(timeOutInSecs, toggleTimeout)
timeOutTimer.start()
allFields = []
if len(refGraph.vs) > 3000:
print "skipping cuz > 3000 - func=" + funcfilename + ",exe-" + exeName
return allFields
# startVal of hack - only take the last 2 confs with k=3 (trace = true OR false)
#bestKSolverDeltas = KBest(KToKeep,lambda x: 100-x['delta'],lambda x:x['delta'])
#bestKSolverBroken = KBest(KToKeep,lambda x: 1000-x['broken'],lambda x:x['broken'])
GradeSystemsInfoInst = GradeSystemsInfo(sourcesList3k, gradeSystemNames)
startForXlsRecord = time.time()
#this cache could probebly get optimized
for refvIndex, refv in enumerate(refGraph.vs):
gradesCache.append([])
#gradesCache[refv['id']] = {}
for tarvIndex, tarv in enumerate(tarGraph.vs):
gradesDict = getMatchedCmdsWithGrade(refv['code'], tarv['code'])
#for key in grade.keys():
gradesCache[refvIndex].append(gradesDict)
#gradesCache[refvIndex][tarvIndex] = grade
objectsCount = 0
for g in split2k.ksplitGraphObject(myK, tarGraph, True):
if timeout[0] == True:
print "Breaking out cuz of timeout, time=" + str(
int(time.time() - timeout[1])
) + " from - func=" + funcfilename + ",exe-" + exeName
return []
objectsCount += 1
for refGraphletContainer in GradeSystemsInfoInst:
refGraphlet = refGraphletContainer.getObject()
(isIso, Map12,
Map21) = refGraphlet.isomorphic_vf2(g, None, None, None, None,
True, False, None, None)
if isIso:
nodeGradesInfos = []
for (x, y) in itertools.imap(
lambda x, y: (int(x['id']), int(y['id'])),
map(lambda x: refGraphlet.vs[x],
range(0, len(refGraphlet.vs))),
map(lambda x: g.vs[Map12[x]], range(0, len(g.vs)))):
nodeGradesInfos.append(gradesCache[x][y])
gradeSystemsFinals = OrderedDict()
for methodType in ['contain', 'ratio']:
gradeSystemsFinals[methodType] = numpy.mean(
map(
lambda nodeGradesInfo: nodeGradesInfo['gradesDict']
[methodType], nodeGradesInfos))
if "Solver" == "Solver":
if gradeSystemsFinals['ratio'] > 50:
conDict = GraphletsConstraints(nodeGradesInfos)
newTarNodes = list(conDict.getRW())
refNodes = map(lambda x: x['refCode'], nodeGradesInfos)
grades = map(compareDisAsmCode, refNodes, newTarNodes)
for methodType in ['contain', 'ratio']:
gradeSystemsFinals['Solver_' +
methodType] = numpy.mean(
map(lambda x: x[methodType],
grades))
#delta = int(gradeSystemsFinals['Solver_' + methodType] - gradeSystemsFinals[methodType])
#refFullCode = ";".join(refNodes)
#tarFullCOde = ";".join(newTarNodes)
#bestKSolverDeltas.insert({'delta':delta,'ref':refFullCode,'tar':tarFullCOde})
#bestKSolverBroken.insert({'broken':conDict.getBrokenNumber(),'ref':refFullCode,'tar':tarFullCOde})
#gradeSystemsDict[gradeSystemName]['bestSolImprovmentList'].insert({'delta':delta,'ref':refFullCode,'tar':tarFullCOde}
else:
for methodType in ['contain', 'ratio']:
gradeSystemsFinals[
'Solver_' +
methodType] = gradeSystemsFinals[methodType]
else:
for methodType in ['contain', 'ratio']:
gradeSystemsFinals[
'Solver_' + methodType] = numpy.mean(
map(
lambda nodeGradesInfo: nodeGradesInfo[
'gradesDict'][methodType],
nodeGradesInfos))
for gradeSystemName in gradeSystemNames:
refGraphletContainer[gradeSystemName] = max(
gradeSystemsFinals[gradeSystemName],
refGraphletContainer[gradeSystemName])
endForXlsRecord = time.time()
for gradeSystemName in gradeSystemNames:
counters = GradeSystemsInfoInst.tallyForGradeSystem(
gradeSystemName, borders)
#counters = gradeSystemsDict[gradeSystemName]['DictCounter'].tallyCounters()
#["exe","funcname","#objects","K","Trace","System","Generator","GradeSystem","TimeStr","TimeNumber"]
fields = [
exeName, funcfilename,
str(objectsCount),
str(myK), "TRUE", methodSystemName, "normal", gradeSystemName,
"\"" + getDurationStr(int(endForXlsRecord - startForXlsRecord)) +
"\"",
str(endForXlsRecord - startForXlsRecord)
]
for i in range(0, len(borders)):
fields.append(
"%.4f" %
(float(counters[i]) / float(len(GradeSystemsInfoInst))))
"""
if (gradeSystemName == "Solver_contain"):
fields.append(str(bestKSolverDeltas.max()))
fields.append(str(bestKSolverDeltas))
fields.append(str(bestKSolverBroken.max()))
fields.append(str(bestKSolverBroken))
else:
for i in range(0,4):
fields.append("see Solver_contain")
sets = GradeSystemsInfoInst.detectPatchForGradeSystem(gradeSystemName,[60,70,80])
sets['total'] = len(tarGraph.vs)
fields.append(str(sets))
"""
allFields.append(fields)
timeOutTimer.cancel()
del timeOutTimer
return allFields
print "loading - " + exeName + " ... " ,
currentExeDir = os.path.join(targetsDir,os.path.join(exeName,functionsGraphsDirectoryName))
for funcFileName in filter(lambda x:x.endswith(myExt),os.listdir(currentExeDir)):
tarGraph = igraph.read(os.path.join(currentExeDir,funcFileName))
tarGraph['name'] = funcFileName
#funcFileName and exe are only for the timeout print
yield {'tarGraph':tarGraph,'refGraph':refGraph,'sourcesList3k':sourcesList3k,'funcFileName':funcFileName,'exeName':exeName}
print "finished loading " + exeName
params = [doOneFunctionFileRW,superFunctionsGenerator()]
#if (mapper != itertools.imap):
# params.append(50)
for allFields in mapper(*params):
for fields in allFields:
reportFile.writeLine(fields)
if __name__ == '__main__':
startVal = time.time()
CompareWithKSplit()
end = time.time()
print "total runtime = " + str(getDurationStr(int(end-startVal)))
sendMail()
tarGraph['name'] = funcFileName
#funcFileName and exe are only for the timeout print
yield {
'tarGraph': tarGraph,
'refGraph': refGraph,
'sourcesList3k': sourcesList3k,
'funcFileName': funcFileName,
'exeName': exeName
}
print "finished loading " + exeName
params = [doOneFunctionFileRW, superFunctionsGenerator()]
#if (mapper != itertools.imap):
# params.append(50)
for allFields in mapper(*params):
for fields in allFields:
reportFile.writeLine(fields)
if __name__ == '__main__':
startVal = time.time()
CompareWithKSplit()
end = time.time()
print "total runtime = " + str(getDurationStr(int(end - startVal)))
sendMail()
