def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) appManifest = argv[4] csvAppDepends = argv[5] #app_dir = op.join(otbDir,"Applications") # Standard Manifest parsing, extract simple and full dependencies [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) [appGroups,appModuleList,appSourceList] = manifestParser.parseManifest(appManifest) # add application sources to sourceList for item in appSourceList: sourceList[item] = appSourceList[item] appDependsList = manifestParser.buildSimpleDep(otbDir,appModuleList,sourceList) #manifestParser.printDepList(appDependsList) manifestParser.outputCSVEdgeList(appDependsList,csvAppDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) appManifest = argv[4] csvAppDepends = argv[5] #app_dir = op.join(otbDir,"Applications") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) [appGroups, appModuleList, appSourceList] = manifestParser.parseManifest(appManifest) # add application sources to sourceList for item in appSourceList: sourceList[item] = appSourceList[item] appDependsList = manifestParser.buildSimpleDep(otbDir, appModuleList, sourceList) #manifestParser.printDepList(appDependsList) manifestParser.outputCSVEdgeList(appDependsList, csvAppDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) exManifest = argv[4] csvExDepends = argv[5] # Standard Manifest parsing, extract simple and full dependencies [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) [exGroups,exModuleList,exSourceList] = manifestParser.parseManifest(exManifest) exDependsList = manifestParser.buildSimpleDep(otbDir,exModuleList,sourceList) # clean the dependencies : remove modules already in fullDepList cleanDepList = {} for mod in exDependsList: cleanDepList[mod] = {} for dep in exDependsList[mod]: if not dep in fullDepList[mod]: cleanDepList[mod][dep] = 1 #manifestParser.printDepList(exDependsList) manifestParser.outputCSVEdgeList(cleanDepList,csvExDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) outManifest = argv[4] if len(argv) >= 6: csvTestDepends = argv[5] else: csvTestDepends = None testing_dir = op.join(otbDir, "Testing") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) # make sure every module is in depList and fullDepList (even if it has no dependencies) for mod in moduleList: if not depList.has_key(mod): depList[mod] = {} if not fullDepList.has_key(mod): fullDepList[mod] = {} OldFolderPartition = buildOldFolderPartition(moduleList) testCxx = {} outFD = open(outManifest, 'wb') outFD.write( "# Monolithic path, Current dir, group name, module name, subDir name, comment\n" ) # parse all cxx test files : analyse them and extract their dependencies for (d, f) in codeParser.FindBinaries(testing_dir): fullPath = op.join(d, f) shortPath = fullPath.replace(otbDir, '.') # skip Testing/Utilities , will not be used anymore if shortPath.startswith("./Testing/Utilities/"): continue moduleDestination = "TBD" groupDestination = "TBD" res = parseTestCxx(fullPath) if res["isTestDriver"]: # no need to dispatch test drivers, they can be generated again continue [testDepList, thirdPartyDep] = getTestDependencies(res["includes"], sourceList) # try to clean the dependency list (remove inherited modules) ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"] cleanTestDepList = [] depListToRemove = [] for dep1 in testDepList: # register the "from" field testDepList[dep1]["from"] = shortPath for dep2 in testDepList: if dep2 == dep1: continue # avoid IO modules to 'eat' usefull dependencies if dep1 in ignoreModules: continue if (dep2 in fullDepList[dep1]) and \ (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in testDepList: if not dep in depListToRemove: cleanTestDepList.append(dep) # build all dependencies of the test testFullDepList = [] for dep in testDepList: for subDep in fullDepList[dep]: if not subDep in testFullDepList: testFullDepList.append(subDep) # start guessing luckyGuess = None guessStep = 1 # try to get the list of module used to partition the corresponding source directory guessModules = [] guessSourceDir = op.split(shortPath.replace("./Testing", "."))[0] if OldFolderPartition.has_key(guessSourceDir): guessModules = OldFolderPartition[guessSourceDir].keys() # special case for Testing/Application -> ApplicationEngine if guessSourceDir == "./Applications": guessModules.append("ApplicationEngine") # first filter : find modules that appear in cleanTestDepList and in guessModules overlappingModules = [] for dep in cleanTestDepList: if dep in guessModules: overlappingModules.append(dep) if len(overlappingModules) == 1: luckyGuess = overlappingModules[0] # second filter : find the source file with the closest name if not luckyGuess: guessStep += 1 [matchFile, matchPercent] = findClosestSourceName(f, sourceList) if (sourceList[matchFile] in testDepList) and (matchPercent > 50.0): luckyGuess = sourceList[matchFile] elif (sourceList[matchFile] in testFullDepList) and (matchPercent > 70.0): luckyGuess = sourceList[matchFile] # third filter : ThirdParty if not luckyGuess: guessStep += 1 if guessSourceDir == "./Utilities" or len(testDepList) == 0: groupDestination = "ThirdParty" if len(thirdPartyDep) == 1: luckyGuess = thirdPartyDep[0] # fourth filter : if there is only one dependency in cleanTestDepList : take it if not luckyGuess: guessStep += 1 if len(cleanTestDepList) == 1: luckyGuess = cleanTestDepList[0] # fifth filter : separate IO test from non-IO test if not luckyGuess: guessStep += 1 if (f.find("Reader") >= 0) or (f.find("Reading") >= 0) or \ (f.find("Write") >= 0) or (f.find("Writing") >= 0) or \ (f.find("ImageIO") >= 0) or (guessSourceDir == "./Code/IO"): # remove non-IO deps from cleanTestDepList and look what's left ioCleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) == "IO": ioCleanDep.append(dep) # ImageIO should be low priority compared to other IO modules if (len(ioCleanDep) == 2) and ("ImageIO" in ioCleanDep): ioCleanDep.remove("ImageIO") if len(ioCleanDep) == 1: luckyGuess = ioCleanDep[0] else: # remove non-IO deps from cleanTestDepList and look what's left nonIOcleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) != "IO": nonIOcleanDep.append(dep) if len(nonIOcleanDep) == 1: luckyGuess = nonIOcleanDep[0] elif len(nonIOcleanDep) == 2: # compare the 2 possible modules based on their group groupAandB = [ manifestParser.getGroup(nonIOcleanDep[0], groups), manifestParser.getGroup(nonIOcleanDep[1], groups) ] levelAandB = [0, 0] for idx in [0, 1]: if groupAandB[idx] == "Core": levelAandB[idx] = 1 elif groupAandB[idx] == "Filtering": levelAandB[idx] = 2 else: levelAandB[idx] = 3 if levelAandB[0] > levelAandB[1]: luckyGuess = nonIOcleanDep[0] if levelAandB[0] < levelAandB[1]: luckyGuess = nonIOcleanDep[1] if luckyGuess: moduleDestination = luckyGuess else: pass #print f + " -> " + str(testDepList) #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )" # if module is found and not group, deduce group if groupDestination == "TBD" and moduleDestination != "TBD": groupDestination = manifestParser.getGroup(moduleDestination, groups) if not res["hasMain"]: # manually add dependency to TestKernel for cxx using a test driver # the include to otbTestMain.h header is not located in the cxx testDepList["TestKernel"] = { "from": shortPath, "to": "./Code/Testing/otbTestMain.h" } testCxx[shortPath] = { "depList": testDepList, "thirdPartyDep": thirdPartyDep, "group": groupDestination, "module": moduleDestination } outFD.write(shortPath + "," + op.basename(op.dirname(shortPath)) + "," + groupDestination + "," + moduleDestination + ",test,\n") outFD.close() # sum all test dependencies in every module allTestDepends = gatherTestDepends(testCxx, fullDepList) # clean the test depends (i.e. ImageIO is dragged by TestKernel) cleanTestDepends = {} for mod in allTestDepends: cleanTestDepends[mod] = {} for dep1 in allTestDepends[mod]: isClean = True for dep2 in allTestDepends[mod]: if dep1 == dep2: continue if dep1 in fullDepList[dep2]: isClean = False break if isClean: cleanTestDepends[mod][dep1] = 1 if csvTestDepends: manifestParser.outputCSVEdgeList(cleanTestDepends, csvTestDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) outManifest = argv[4] if len(argv) >= 6: csvTestDepends = argv[5] else: csvTestDepends = None testing_dir = op.join(otbDir, "Testing") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) # make sure every module is in depList and fullDepList (even if it has no dependencies) for mod in moduleList: if not depList.has_key(mod): depList[mod] = {} if not fullDepList.has_key(mod): fullDepList[mod] = {} OldFolderPartition = buildOldFolderPartition(moduleList) testCxx = {} outFD = open(outManifest, "wb") outFD.write("# Monolithic path, Current dir, group name, module name, subDir name, comment\n") # parse all cxx test files : analyse them and extract their dependencies for (d, f) in codeParser.FindBinaries(testing_dir): fullPath = op.join(d, f) shortPath = fullPath.replace(otbDir, ".") # skip Testing/Utilities , will not be used anymore if shortPath.startswith("./Testing/Utilities/"): continue moduleDestination = "TBD" groupDestination = "TBD" res = parseTestCxx(fullPath) if res["isTestDriver"]: # no need to dispatch test drivers, they can be generated again continue [testDepList, thirdPartyDep] = getTestDependencies(res["includes"], sourceList) # try to clean the dependency list (remove inherited modules) ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"] cleanTestDepList = [] depListToRemove = [] for dep1 in testDepList: # register the "from" field testDepList[dep1]["from"] = shortPath for dep2 in testDepList: if dep2 == dep1: continue # avoid IO modules to 'eat' usefull dependencies if dep1 in ignoreModules: continue if (dep2 in fullDepList[dep1]) and (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in testDepList: if not dep in depListToRemove: cleanTestDepList.append(dep) # build all dependencies of the test testFullDepList = [] for dep in testDepList: for subDep in fullDepList[dep]: if not subDep in testFullDepList: testFullDepList.append(subDep) # start guessing luckyGuess = None guessStep = 1 # try to get the list of module used to partition the corresponding source directory guessModules = [] guessSourceDir = op.split(shortPath.replace("./Testing", "."))[0] if OldFolderPartition.has_key(guessSourceDir): guessModules = OldFolderPartition[guessSourceDir].keys() # special case for Testing/Application -> ApplicationEngine if guessSourceDir == "./Applications": guessModules.append("ApplicationEngine") # first filter : find modules that appear in cleanTestDepList and in guessModules overlappingModules = [] for dep in cleanTestDepList: if dep in guessModules: overlappingModules.append(dep) if len(overlappingModules) == 1: luckyGuess = overlappingModules[0] # second filter : find the source file with the closest name if not luckyGuess: guessStep += 1 [matchFile, matchPercent] = findClosestSourceName(f, sourceList) if (sourceList[matchFile] in testDepList) and (matchPercent > 50.0): luckyGuess = sourceList[matchFile] elif (sourceList[matchFile] in testFullDepList) and (matchPercent > 70.0): luckyGuess = sourceList[matchFile] # third filter : ThirdParty if not luckyGuess: guessStep += 1 if guessSourceDir == "./Utilities" or len(testDepList) == 0: groupDestination = "ThirdParty" if len(thirdPartyDep) == 1: luckyGuess = thirdPartyDep[0] # fourth filter : if there is only one dependency in cleanTestDepList : take it if not luckyGuess: guessStep += 1 if len(cleanTestDepList) == 1: luckyGuess = cleanTestDepList[0] # fifth filter : separate IO test from non-IO test if not luckyGuess: guessStep += 1 if ( (f.find("Reader") >= 0) or (f.find("Reading") >= 0) or (f.find("Write") >= 0) or (f.find("Writing") >= 0) or (f.find("ImageIO") >= 0) or (guessSourceDir == "./Code/IO") ): # remove non-IO deps from cleanTestDepList and look what's left ioCleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) == "IO": ioCleanDep.append(dep) # ImageIO should be low priority compared to other IO modules if (len(ioCleanDep) == 2) and ("ImageIO" in ioCleanDep): ioCleanDep.remove("ImageIO") if len(ioCleanDep) == 1: luckyGuess = ioCleanDep[0] else: # remove non-IO deps from cleanTestDepList and look what's left nonIOcleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) != "IO": nonIOcleanDep.append(dep) if len(nonIOcleanDep) == 1: luckyGuess = nonIOcleanDep[0] elif len(nonIOcleanDep) == 2: # compare the 2 possible modules based on their group groupAandB = [ manifestParser.getGroup(nonIOcleanDep[0], groups), manifestParser.getGroup(nonIOcleanDep[1], groups), ] levelAandB = [0, 0] for idx in [0, 1]: if groupAandB[idx] == "Core": levelAandB[idx] = 1 elif groupAandB[idx] == "Filtering": levelAandB[idx] = 2 else: levelAandB[idx] = 3 if levelAandB[0] > levelAandB[1]: luckyGuess = nonIOcleanDep[0] if levelAandB[0] < levelAandB[1]: luckyGuess = nonIOcleanDep[1] if luckyGuess: moduleDestination = luckyGuess else: pass # print f + " -> " + str(testDepList) # print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )" # if module is found and not group, deduce group if groupDestination == "TBD" and moduleDestination != "TBD": groupDestination = manifestParser.getGroup(moduleDestination, groups) if not res["hasMain"]: # manually add dependency to TestKernel for cxx using a test driver # the include to otbTestMain.h header is not located in the cxx testDepList["TestKernel"] = {"from": shortPath, "to": "./Code/Testing/otbTestMain.h"} testCxx[shortPath] = { "depList": testDepList, "thirdPartyDep": thirdPartyDep, "group": groupDestination, "module": moduleDestination, } outFD.write( shortPath + "," + op.basename(op.dirname(shortPath)) + "," + groupDestination + "," + moduleDestination + ",test,\n" ) outFD.close() # sum all test dependencies in every module allTestDepends = gatherTestDepends(testCxx, fullDepList) # clean the test depends (i.e. ImageIO is dragged by TestKernel) cleanTestDepends = {} for mod in allTestDepends: cleanTestDepends[mod] = {} for dep1 in allTestDepends[mod]: isClean = True for dep2 in allTestDepends[mod]: if dep1 == dep2: continue if dep1 in fullDepList[dep2]: isClean = False break if isClean: cleanTestDepends[mod][dep1] = 1 if csvTestDepends: manifestParser.outputCSVEdgeList(cleanTestDepends, csvTestDepends)