def CompareWithKSplit(): refGraph = igraph.read(os.path.join(sourcesDir,sourceFunctionName)) refGraph['name'] = sourceFunctionName if (os.name != "nt"): #p = multiprocessing.Pool(initializer=workerInit,initargs=[len(refGraph.vs)]) p = multiprocessing.Pool() mapper = p.imap_unordered #mapper = itertools.imap else: mapper = itertools.imap #workerInit(len(refGraph.vs)) reportFile = CounterXlsReport(csvName + "-K=" + str(myK) + "-" + sourceFunctionName) print "Prepping db - (func files) - target=" +os.path.join(sourcesDir,sourceFunctionName) sourcesList3k = list(mapper(identity,split2k.ksplitGraphObject(myK,refGraph,True))) print "For k=" + str(myK) + " we have - " + str(len(sourcesList3k)) print "end db prep" from simple_db import DBSimpleClient db = DBSimpleClient() params = [doOneFunctionFileRW,itertools.izip(db.get_all(),itertools.cycle([sourcesList3k]))] #if (mapper != itertools.imap): # params.append(50) for allFields in mapper(*params): for fields in allFields: reportFile.writeLine(fields)
def CompareWithKSplit(): refGraph = igraph.read(os.path.join(sourcesDir, sourceFunctionName)) refGraph['name'] = sourceFunctionName if (os.name != "nt"): #p = multiprocessing.Pool(initializer=workerInit,initargs=[len(refGraph.vs)]) p = multiprocessing.Pool() mapper = p.imap_unordered #mapper = itertools.imap else: mapper = itertools.imap #workerInit(len(refGraph.vs)) reportFile = CounterXlsReport(csvName + "-K=" + str(myK) + "-" + sourceFunctionName) print "Prepping db - (func files) - target=" + os.path.join( sourcesDir, sourceFunctionName) sourcesList3k = list( mapper(identity, split2k.ksplitGraphObject(myK, refGraph, True))) print "For k=" + str(myK) + " we have - " + str(len(sourcesList3k)) print "end db prep" def superFunctionsGenerator(): for exeName in os.listdir(targetsDir): print "loading - " + exeName + " ... ", currentExeDir = os.path.join( targetsDir, os.path.join(exeName, functionsGraphsDirectoryName)) for funcFileName in filter(lambda x: x.endswith(myExt), os.listdir(currentExeDir)): tarGraph = igraph.read( os.path.join(currentExeDir, funcFileName)) tarGraph['name'] = funcFileName #funcFileName and exe are only for the timeout print yield { 'tarGraph': tarGraph, 'refGraph': refGraph, 'sourcesList3k': sourcesList3k, 'funcFileName': funcFileName, 'exeName': exeName } print "finished loading " + exeName params = [doOneFunctionFileRW, superFunctionsGenerator()] #if (mapper != itertools.imap): # params.append(50) for allFields in mapper(*params): for fields in allFields: reportFile.writeLine(fields)
def CompareWithKSplit(): refGraph = igraph.read(os.path.join(sourcesDir,sourceFunctionName)) refGraph['name'] = sourceFunctionName if (os.name != "nt"): #p = multiprocessing.Pool(initializer=workerInit,initargs=[len(refGraph.vs)]) p = multiprocessing.Pool() mapper = p.imap_unordered #mapper = itertools.imap else: mapper = itertools.imap #workerInit(len(refGraph.vs)) reportFile = CounterXlsReport(csvName + "-K=" + str(myK) + "-" + sourceFunctionName) print "Prepping db - (func files) - target=" +os.path.join(sourcesDir,sourceFunctionName) sourcesList3k = list(mapper(identity,split2k.ksplitGraphObject(myK,refGraph,True))) print "For k=" + str(myK) + " we have - " + str(len(sourcesList3k)) print "end db prep" def inWhiteList(value,atrbName): for whiteListEntry in filter(lambda x: x['source']==sourceName,whiteList): #print "CHECK " + atrbName if whiteListEntry[atrbName] == value: return True return False def superFunctionsGenerator(): for exeName in filter(lambda x:inWhiteList(x,'exe'),os.listdir(targetsDir)): print "loading - " + exeName + " ... " currentExeDir = os.path.join(targetsDir,os.path.join(exeName,functionsGraphsDirectoryName)) for funcFileName in filter(lambda x:inWhiteList(os.path.splitext(x)[0],'functionName'),filter(lambda x:x.endswith(myExt),os.listdir(currentExeDir))): print "FUNC begin - " + funcFileName + " ... " tarGraph = igraph.read(os.path.join(currentExeDir,funcFileName)) tarGraph['name'] = funcFileName #funcFileName and exe are only for the timeout print yield {'tarGraph':tarGraph,'refGraph':refGraph,'sourcesList3k':sourcesList3k,'funcFileName':funcFileName,'exeName':exeName} print "FUNC finished " + funcFileName print "finished loading " + exeName params = [doOneFunctionFileRW,superFunctionsGenerator()] #if (mapper != itertools.imap): # params.append(50) for allFields in mapper(*params): for fields in allFields: reportFile.writeLine(fields)
def CompareWithKSplit(): refGraph = igraph.read(os.path.join(sourcesDir, sourceFunctionName)) refGraph['name'] = sourceFunctionName if (os.name != "nt"): #p = multiprocessing.Pool(initializer=workerInit,initargs=[len(refGraph.vs)]) p = multiprocessing.Pool() mapper = p.imap_unordered #mapper = itertools.imap else: mapper = itertools.imap #workerInit(len(refGraph.vs)) reportFile = CounterXlsReport(csvName + "-K=" + str(myK) + "-" + sourceFunctionName) print "Prepping db - (func files) - target=" + os.path.join( sourcesDir, sourceFunctionName) sourcesList3k = list( mapper(identity, split2k.ksplitGraphObject(myK, refGraph, True))) print "For k=" + str(myK) + " we have - " + str(len(sourcesList3k)) print "end db prep" from simple_db import DBSimpleClient db = DBSimpleClient() params = [ doOneFunctionFileRW, itertools.izip(db.get_all(), itertools.cycle([sourcesList3k])) ] #if (mapper != itertools.imap): # params.append(50) for allFields in mapper(*params): for fields in allFields: reportFile.writeLine(fields)
def CompareWithKSplit(sourcesDirectory, targetsDir): print targetsDir print 'target', targetsDir refGraph = igraph.read(sourcesDirectory) #print refGraph sourceFunctionName = sourcesDirectory.split('\\')[-1] refGraph['name'] = sourceFunctionName if (os.name != "nt"): # judge the platform nt = windows, posix = linux #p = multiprocessing.Pool(initializer=workerInit,initargs=[len(refGraph.vs)]) p = multiprocessing.Pool() mapper = p.imap_unordered #mapper = itertools.imap else: mapper = itertools.imap #workerInit(len(refGraph.vs)) sourcesExeName = sourcesDirectory.split('\\')[-2] targetsExeName = targetsDir.split('\\')[-1] familyName = targetsDir.split('\\')[-2] reportFile = CounterXlsReport(sourcesExeName + "-" + familyName + "@" + targetsExeName + "-K=" + str(myK) + "-" + sourceFunctionName) # the name of result file print "Prepping db - (func files) - target=" +sourcesDirectory sourcesList3k = list(mapper(identity,split2k.ksplitGraphObject(myK,refGraph,True))) #print "For k=" + str(myK) + " we have - " + str(len(sourcesList3k)) #print "end db prep" def superFunctionsGenerator(currentExeDir): #for exeName in os.listdir(targetsDir): #print "loading - " + exeName + " ... " , #print exeName #print functionsGraphsDirectoryName #currentExeDir = os.path.join(targetsDir,os.path.join(exeName)) #print currentExeDir ##exit(0) for funcFileName in filter(lambda x:x.endswith(myExt),os.listdir(currentExeDir)): #print funcFileName tarGraph = igraph.read(os.path.join(currentExeDir,funcFileName)) tarGraph['name'] = funcFileName #funcFileName and exe are only for the timeout print yield {'tarGraph':tarGraph,'refGraph':refGraph,'sourcesList3k':sourcesList3k,'funcFileName':funcFileName,'exeName':targetsExeName}