def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) appManifest = argv[4] csvAppDepends = argv[5] #app_dir = op.join(otbDir,"Applications") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) [appGroups, appModuleList, appSourceList] = manifestParser.parseManifest(appManifest) # add application sources to sourceList for item in appSourceList: sourceList[item] = appSourceList[item] appDependsList = manifestParser.buildSimpleDep(otbDir, appModuleList, sourceList) #manifestParser.printDepList(appDependsList) manifestParser.outputCSVEdgeList(appDependsList, csvAppDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) appManifest = argv[4] csvAppDepends = argv[5] #app_dir = op.join(otbDir,"Applications") # Standard Manifest parsing, extract simple and full dependencies [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) [appGroups,appModuleList,appSourceList] = manifestParser.parseManifest(appManifest) # add application sources to sourceList for item in appSourceList: sourceList[item] = appSourceList[item] appDependsList = manifestParser.buildSimpleDep(otbDir,appModuleList,sourceList) #manifestParser.printDepList(appDependsList) manifestParser.outputCSVEdgeList(appDependsList,csvAppDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) exManifest = argv[4] csvExDepends = argv[5] # Standard Manifest parsing, extract simple and full dependencies [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) [exGroups,exModuleList,exSourceList] = manifestParser.parseManifest(exManifest) exDependsList = manifestParser.buildSimpleDep(otbDir,exModuleList,sourceList) # clean the dependencies : remove modules already in fullDepList cleanDepList = {} for mod in exDependsList: cleanDepList[mod] = {} for dep in exDependsList[mod]: if not dep in fullDepList[mod]: cleanDepList[mod][dep] = 1 #manifestParser.printDepList(exDependsList) manifestParser.outputCSVEdgeList(cleanDepList,csvExDepends)
def main(argv): manifestPath = argv[1] otbDir = argv[2] if len(argv) >= 4: outPath = argv[3] else: outPath = "" print "Warning ! Results will be produced in "+otbDir+" , the current content will be modified" resp = raw_input("Press 'y' to continue : ") if resp.lower() != "y": return 0 [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath) for mod in moduleList: for src in moduleList[mod]: if src.endswith(".h"): nextContent = parserHeader(op.join(otbDir,src),mod) if outPath == "": nextPath = op.join(otbDir,src) else: nextPath = op.join(outPath,src) call(["mkdir","-p",op.dirname(nextPath)]) fd = open(nextPath,'wb') fd.writelines(nextContent) fd.close() return 0
def main(argv): manifestPath = argv[1] otbDir = argv[2] if len(argv) >= 4: outPath = argv[3] else: outPath = "" print "Warning ! Results will be produced in " + otbDir + " , the current content will be modified" resp = raw_input("Press 'y' to continue : ") if resp.lower() != "y": return 0 [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) for mod in moduleList: for src in moduleList[mod]: if src.endswith(".h"): nextContent = parserHeader(op.join(otbDir, src), mod) if outPath == "": nextPath = op.join(otbDir, src) else: nextPath = op.join(outPath, src) call(["mkdir", "-p", op.dirname(nextPath)]) fd = open(nextPath, 'wb') fd.writelines(nextContent) fd.close() return 0
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) outManifest = argv[4] if len(argv) >= 6: csvTestDepends = argv[5] else: csvTestDepends = None testing_dir = op.join(otbDir, "Testing") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) # make sure every module is in depList and fullDepList (even if it has no dependencies) for mod in moduleList: if not depList.has_key(mod): depList[mod] = {} if not fullDepList.has_key(mod): fullDepList[mod] = {} OldFolderPartition = buildOldFolderPartition(moduleList) testCxx = {} outFD = open(outManifest, 'wb') outFD.write( "# Monolithic path, Current dir, group name, module name, subDir name, comment\n" ) # parse all cxx test files : analyse them and extract their dependencies for (d, f) in codeParser.FindBinaries(testing_dir): fullPath = op.join(d, f) shortPath = fullPath.replace(otbDir, '.') # skip Testing/Utilities , will not be used anymore if shortPath.startswith("./Testing/Utilities/"): continue moduleDestination = "TBD" groupDestination = "TBD" res = parseTestCxx(fullPath) if res["isTestDriver"]: # no need to dispatch test drivers, they can be generated again continue [testDepList, thirdPartyDep] = getTestDependencies(res["includes"], sourceList) # try to clean the dependency list (remove inherited modules) ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"] cleanTestDepList = [] depListToRemove = [] for dep1 in testDepList: # register the "from" field testDepList[dep1]["from"] = shortPath for dep2 in testDepList: if dep2 == dep1: continue # avoid IO modules to 'eat' usefull dependencies if dep1 in ignoreModules: continue if (dep2 in fullDepList[dep1]) and \ (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in testDepList: if not dep in depListToRemove: cleanTestDepList.append(dep) # build all dependencies of the test testFullDepList = [] for dep in testDepList: for subDep in fullDepList[dep]: if not subDep in testFullDepList: testFullDepList.append(subDep) # start guessing luckyGuess = None guessStep = 1 # try to get the list of module used to partition the corresponding source directory guessModules = [] guessSourceDir = op.split(shortPath.replace("./Testing", "."))[0] if OldFolderPartition.has_key(guessSourceDir): guessModules = OldFolderPartition[guessSourceDir].keys() # special case for Testing/Application -> ApplicationEngine if guessSourceDir == "./Applications": guessModules.append("ApplicationEngine") # first filter : find modules that appear in cleanTestDepList and in guessModules overlappingModules = [] for dep in cleanTestDepList: if dep in guessModules: overlappingModules.append(dep) if len(overlappingModules) == 1: luckyGuess = overlappingModules[0] # second filter : find the source file with the closest name if not luckyGuess: guessStep += 1 [matchFile, matchPercent] = findClosestSourceName(f, sourceList) if (sourceList[matchFile] in testDepList) and (matchPercent > 50.0): luckyGuess = sourceList[matchFile] elif (sourceList[matchFile] in testFullDepList) and (matchPercent > 70.0): luckyGuess = sourceList[matchFile] # third filter : ThirdParty if not luckyGuess: guessStep += 1 if guessSourceDir == "./Utilities" or len(testDepList) == 0: groupDestination = "ThirdParty" if len(thirdPartyDep) == 1: luckyGuess = thirdPartyDep[0] # fourth filter : if there is only one dependency in cleanTestDepList : take it if not luckyGuess: guessStep += 1 if len(cleanTestDepList) == 1: luckyGuess = cleanTestDepList[0] # fifth filter : separate IO test from non-IO test if not luckyGuess: guessStep += 1 if (f.find("Reader") >= 0) or (f.find("Reading") >= 0) or \ (f.find("Write") >= 0) or (f.find("Writing") >= 0) or \ (f.find("ImageIO") >= 0) or (guessSourceDir == "./Code/IO"): # remove non-IO deps from cleanTestDepList and look what's left ioCleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) == "IO": ioCleanDep.append(dep) # ImageIO should be low priority compared to other IO modules if (len(ioCleanDep) == 2) and ("ImageIO" in ioCleanDep): ioCleanDep.remove("ImageIO") if len(ioCleanDep) == 1: luckyGuess = ioCleanDep[0] else: # remove non-IO deps from cleanTestDepList and look what's left nonIOcleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) != "IO": nonIOcleanDep.append(dep) if len(nonIOcleanDep) == 1: luckyGuess = nonIOcleanDep[0] elif len(nonIOcleanDep) == 2: # compare the 2 possible modules based on their group groupAandB = [ manifestParser.getGroup(nonIOcleanDep[0], groups), manifestParser.getGroup(nonIOcleanDep[1], groups) ] levelAandB = [0, 0] for idx in [0, 1]: if groupAandB[idx] == "Core": levelAandB[idx] = 1 elif groupAandB[idx] == "Filtering": levelAandB[idx] = 2 else: levelAandB[idx] = 3 if levelAandB[0] > levelAandB[1]: luckyGuess = nonIOcleanDep[0] if levelAandB[0] < levelAandB[1]: luckyGuess = nonIOcleanDep[1] if luckyGuess: moduleDestination = luckyGuess else: pass #print f + " -> " + str(testDepList) #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )" # if module is found and not group, deduce group if groupDestination == "TBD" and moduleDestination != "TBD": groupDestination = manifestParser.getGroup(moduleDestination, groups) if not res["hasMain"]: # manually add dependency to TestKernel for cxx using a test driver # the include to otbTestMain.h header is not located in the cxx testDepList["TestKernel"] = { "from": shortPath, "to": "./Code/Testing/otbTestMain.h" } testCxx[shortPath] = { "depList": testDepList, "thirdPartyDep": thirdPartyDep, "group": groupDestination, "module": moduleDestination } outFD.write(shortPath + "," + op.basename(op.dirname(shortPath)) + "," + groupDestination + "," + moduleDestination + ",test,\n") outFD.close() # sum all test dependencies in every module allTestDepends = gatherTestDepends(testCxx, fullDepList) # clean the test depends (i.e. ImageIO is dragged by TestKernel) cleanTestDepends = {} for mod in allTestDepends: cleanTestDepends[mod] = {} for dep1 in allTestDepends[mod]: isClean = True for dep2 in allTestDepends[mod]: if dep1 == dep2: continue if dep1 in fullDepList[dep2]: isClean = False break if isClean: cleanTestDepends[mod][dep1] = 1 if csvTestDepends: manifestParser.outputCSVEdgeList(cleanTestDepends, csvTestDepends)
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) outManifest = argv[4] if len(argv) >= 6: csvTestDepends = argv[5] else: csvTestDepends = None testing_dir = op.join(otbDir, "Testing") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) # make sure every module is in depList and fullDepList (even if it has no dependencies) for mod in moduleList: if not depList.has_key(mod): depList[mod] = {} if not fullDepList.has_key(mod): fullDepList[mod] = {} OldFolderPartition = buildOldFolderPartition(moduleList) testCxx = {} outFD = open(outManifest, "wb") outFD.write("# Monolithic path, Current dir, group name, module name, subDir name, comment\n") # parse all cxx test files : analyse them and extract their dependencies for (d, f) in codeParser.FindBinaries(testing_dir): fullPath = op.join(d, f) shortPath = fullPath.replace(otbDir, ".") # skip Testing/Utilities , will not be used anymore if shortPath.startswith("./Testing/Utilities/"): continue moduleDestination = "TBD" groupDestination = "TBD" res = parseTestCxx(fullPath) if res["isTestDriver"]: # no need to dispatch test drivers, they can be generated again continue [testDepList, thirdPartyDep] = getTestDependencies(res["includes"], sourceList) # try to clean the dependency list (remove inherited modules) ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"] cleanTestDepList = [] depListToRemove = [] for dep1 in testDepList: # register the "from" field testDepList[dep1]["from"] = shortPath for dep2 in testDepList: if dep2 == dep1: continue # avoid IO modules to 'eat' usefull dependencies if dep1 in ignoreModules: continue if (dep2 in fullDepList[dep1]) and (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in testDepList: if not dep in depListToRemove: cleanTestDepList.append(dep) # build all dependencies of the test testFullDepList = [] for dep in testDepList: for subDep in fullDepList[dep]: if not subDep in testFullDepList: testFullDepList.append(subDep) # start guessing luckyGuess = None guessStep = 1 # try to get the list of module used to partition the corresponding source directory guessModules = [] guessSourceDir = op.split(shortPath.replace("./Testing", "."))[0] if OldFolderPartition.has_key(guessSourceDir): guessModules = OldFolderPartition[guessSourceDir].keys() # special case for Testing/Application -> ApplicationEngine if guessSourceDir == "./Applications": guessModules.append("ApplicationEngine") # first filter : find modules that appear in cleanTestDepList and in guessModules overlappingModules = [] for dep in cleanTestDepList: if dep in guessModules: overlappingModules.append(dep) if len(overlappingModules) == 1: luckyGuess = overlappingModules[0] # second filter : find the source file with the closest name if not luckyGuess: guessStep += 1 [matchFile, matchPercent] = findClosestSourceName(f, sourceList) if (sourceList[matchFile] in testDepList) and (matchPercent > 50.0): luckyGuess = sourceList[matchFile] elif (sourceList[matchFile] in testFullDepList) and (matchPercent > 70.0): luckyGuess = sourceList[matchFile] # third filter : ThirdParty if not luckyGuess: guessStep += 1 if guessSourceDir == "./Utilities" or len(testDepList) == 0: groupDestination = "ThirdParty" if len(thirdPartyDep) == 1: luckyGuess = thirdPartyDep[0] # fourth filter : if there is only one dependency in cleanTestDepList : take it if not luckyGuess: guessStep += 1 if len(cleanTestDepList) == 1: luckyGuess = cleanTestDepList[0] # fifth filter : separate IO test from non-IO test if not luckyGuess: guessStep += 1 if ( (f.find("Reader") >= 0) or (f.find("Reading") >= 0) or (f.find("Write") >= 0) or (f.find("Writing") >= 0) or (f.find("ImageIO") >= 0) or (guessSourceDir == "./Code/IO") ): # remove non-IO deps from cleanTestDepList and look what's left ioCleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) == "IO": ioCleanDep.append(dep) # ImageIO should be low priority compared to other IO modules if (len(ioCleanDep) == 2) and ("ImageIO" in ioCleanDep): ioCleanDep.remove("ImageIO") if len(ioCleanDep) == 1: luckyGuess = ioCleanDep[0] else: # remove non-IO deps from cleanTestDepList and look what's left nonIOcleanDep = [] for dep in cleanTestDepList: if manifestParser.getGroup(dep, groups) != "IO": nonIOcleanDep.append(dep) if len(nonIOcleanDep) == 1: luckyGuess = nonIOcleanDep[0] elif len(nonIOcleanDep) == 2: # compare the 2 possible modules based on their group groupAandB = [ manifestParser.getGroup(nonIOcleanDep[0], groups), manifestParser.getGroup(nonIOcleanDep[1], groups), ] levelAandB = [0, 0] for idx in [0, 1]: if groupAandB[idx] == "Core": levelAandB[idx] = 1 elif groupAandB[idx] == "Filtering": levelAandB[idx] = 2 else: levelAandB[idx] = 3 if levelAandB[0] > levelAandB[1]: luckyGuess = nonIOcleanDep[0] if levelAandB[0] < levelAandB[1]: luckyGuess = nonIOcleanDep[1] if luckyGuess: moduleDestination = luckyGuess else: pass # print f + " -> " + str(testDepList) # print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )" # if module is found and not group, deduce group if groupDestination == "TBD" and moduleDestination != "TBD": groupDestination = manifestParser.getGroup(moduleDestination, groups) if not res["hasMain"]: # manually add dependency to TestKernel for cxx using a test driver # the include to otbTestMain.h header is not located in the cxx testDepList["TestKernel"] = {"from": shortPath, "to": "./Code/Testing/otbTestMain.h"} testCxx[shortPath] = { "depList": testDepList, "thirdPartyDep": thirdPartyDep, "group": groupDestination, "module": moduleDestination, } outFD.write( shortPath + "," + op.basename(op.dirname(shortPath)) + "," + groupDestination + "," + moduleDestination + ",test,\n" ) outFD.close() # sum all test dependencies in every module allTestDepends = gatherTestDepends(testCxx, fullDepList) # clean the test depends (i.e. ImageIO is dragged by TestKernel) cleanTestDepends = {} for mod in allTestDepends: cleanTestDepends[mod] = {} for dep1 in allTestDepends[mod]: isClean = True for dep2 in allTestDepends[mod]: if dep1 == dep2: continue if dep1 in fullDepList[dep2]: isClean = False break if isClean: cleanTestDepends[mod][dep1] = 1 if csvTestDepends: manifestParser.outputCSVEdgeList(cleanTestDepends, csvTestDepends)
def main(argv): manifest = op.expanduser(argv[1]) otbDir = op.expanduser(argv[2]) outputDir = argv[3] exDepends = op.expanduser(argv[4]) example_dir = op.join(otbDir, "Examples") [groups, moduleList, sourceList] = manifestParser.parseManifest(manifest) for mod in moduleList: if mod == "" or mod == "TBD": continue # remove non-example source files for src in moduleList[mod]: cleanSrc = src.strip("./") if not cleanSrc.startswith("Examples/"): moduleList[mod].remove(src) if len(moduleList[mod]) == 0: continue currentGrp = manifestParser.getGroup(mod, groups) # prepare output directory targetDir = op.join( op.join(op.join(op.join(outputDir, "Modules"), currentGrp), mod), "example") call(["mkdir", "-p", targetDir]) testCode = {} # parse all example files to extract the functions and mains for src in moduleList[mod]: fullSrcPath = op.join(otbDir, src) srcName = op.basename(src) exeName = "${EXE_TESTS}" exeAlias = "${EXE_TESTS1}" currentCMake = op.join(op.dirname(fullSrcPath), "CMakeLists.txt") # get add_test() code corresponding to the example testFunctionName = op.splitext(srcName)[0] + "Test" testCode[srcName] = dispatchTests.findTestFromExe( currentCMake, exeName, exeAlias, [testFunctionName]) # try additional exeName : ${EXE_TESTS2} moreCode = dispatchTests.findTestFromExe(currentCMake, "${EXE_TESTS2}", "", [testFunctionName]) for tName in moreCode: testCode[srcName][tName] = moreCode[tName] # copy (move) example sources # TODO : this should be done in modulizer for every manifests (src, app, test, examples) #command = ["cp",fullSrcPath,op.join(targetDir,srcName)] #call(command) # generate the test driver source code testDriver = op.join(targetDir, "otb" + mod + "ExamplesTests.cxx") fd = open(testDriver, 'wb') fd.write("#include <iostream>\n") fd.write("#include \"otbTestMain.h\"\n") fd.write("void RegisterTests()\n") fd.write("{\n") for src in moduleList[mod]: srcName = op.basename(src) tFunc = op.splitext(op.basename(srcName))[0] + "Test" fd.write(" REGISTER_TEST(" + tFunc + ");\n") fd.write("}\n\n") for src in moduleList[mod]: srcName = op.basename(src) tFunc = op.splitext(op.basename(srcName))[0] + "Test" fd.write("#undef main\n") fd.write("#define main " + tFunc + "\n") fd.write("#include \"" + op.basename(srcName) + "\"\n\n") fd.close() # generate CMakeLists.txt testCmakefile = op.join(targetDir, "CMakeLists.txt") fd = open(testCmakefile, 'wb') # declare each example executable for src in moduleList[mod]: srcName = op.basename(src) exeName = op.splitext(srcName)[0] fd.write("add_executable(" + exeName + " " + srcName + ")\n") fd.write("target_link_libraries("+exeName+\ " ${OTB"+mod+"_LIBRARIES} ${OTB"+mod+"-Example_LIBRARIES})\n\n") fd.write("if( BUILD_TESTING )\n") # TODO : move example baselines according to new dispatch fd.write("set(BASELINE ${OTB_DATA_ROOT}/Baseline/Examples/" + mod + ")\n") fd.write("set(INPUTDATA ${OTB_DATA_ROOT}/Examples)\n") fd.write("set(TEMP ${OTB_BINARY_DIR}/Testing/Temporary)\n") fd.write("set(EXE_TESTS ${CXX_TEST_PATH}/otb" + mod + "ExamplesTests)\n") # declare tests fd.write("\n#----------- TESTS DECLARATION ----------------\n") for srcName in testCode: for tName in testCode[srcName]: for line in testCode[srcName][tName]["code"]: cleanLine = line.replace( testCode[srcName][tName]["exeName"], "${EXE_TESTS}") fd.write(cleanLine) # declare test driver fd.write("add_executable(otb" + mod + "ExamplesTests otb" + mod + "ExamplesTests.cxx)\n") fd.write("target_link_libraries(otb"+mod+"ExamplesTests "+\ "${OTB"+mod+"_LIBRARIES} ${OTB"+mod+"-Example_LIBRARIES} "\ "${OTBTestKernel_LIBRARIES})\n\n") fd.write("endif()\n") fd.close() return
def main(argv): manifest = op.expanduser(argv[1]) otbDir = op.expanduser(argv[2]) outputDir = argv[3] exDepends = op.expanduser(argv[4]) example_dir = op.join(otbDir,"Examples") [groups,moduleList,sourceList] = manifestParser.parseManifest(manifest) for mod in moduleList: if mod == "" or mod == "TBD": continue # remove non-example source files for src in moduleList[mod]: cleanSrc = src.strip("./") if not cleanSrc.startswith("Examples/"): moduleList[mod].remove(src) if len(moduleList[mod]) == 0: continue currentGrp = manifestParser.getGroup(mod,groups) # prepare output directory targetDir = op.join(op.join(op.join(op.join(outputDir,"Modules"),currentGrp),mod),"example") call(["mkdir","-p",targetDir]) testCode = {} # parse all example files to extract the functions and mains for src in moduleList[mod]: fullSrcPath = op.join(otbDir,src) srcName = op.basename(src) exeName = "${EXE_TESTS}" exeAlias = "${EXE_TESTS1}" currentCMake = op.join(op.dirname(fullSrcPath),"CMakeLists.txt") # get add_test() code corresponding to the example testFunctionName = op.splitext(srcName)[0]+"Test" testCode[srcName] = dispatchTests.findTestFromExe(currentCMake,exeName,exeAlias,[testFunctionName]) # try additional exeName : ${EXE_TESTS2} moreCode = dispatchTests.findTestFromExe(currentCMake,"${EXE_TESTS2}","",[testFunctionName]) for tName in moreCode: testCode[srcName][tName] = moreCode[tName] # copy (move) example sources # TODO : this should be done in modulizer for every manifests (src, app, test, examples) #command = ["cp",fullSrcPath,op.join(targetDir,srcName)] #call(command) # generate the test driver source code testDriver = op.join(targetDir,"otb"+mod+"ExamplesTests.cxx") fd = open(testDriver,'wb') fd.write("#include <iostream>\n") fd.write("#include \"otbTestMain.h\"\n") fd.write("void RegisterTests()\n") fd.write("{\n") for src in moduleList[mod]: srcName = op.basename(src) tFunc = op.splitext(op.basename(srcName))[0]+"Test" fd.write(" REGISTER_TEST("+tFunc+");\n") fd.write("}\n\n") for src in moduleList[mod]: srcName = op.basename(src) tFunc = op.splitext(op.basename(srcName))[0]+"Test" fd.write("#undef main\n") fd.write("#define main "+tFunc+"\n") fd.write("#include \""+op.basename(srcName)+"\"\n\n") fd.close() # generate CMakeLists.txt testCmakefile = op.join(targetDir,"CMakeLists.txt") fd = open(testCmakefile,'wb') # declare each example executable for src in moduleList[mod]: srcName = op.basename(src) exeName = op.splitext(srcName)[0] fd.write("add_executable("+exeName+" "+srcName+")\n") fd.write("target_link_libraries("+exeName+\ " ${OTB"+mod+"_LIBRARIES} ${OTB"+mod+"-Example_LIBRARIES})\n\n") fd.write("if( BUILD_TESTING )\n") # TODO : move example baselines according to new dispatch fd.write("set(BASELINE ${OTB_DATA_ROOT}/Baseline/Examples/"+mod+")\n") fd.write("set(INPUTDATA ${OTB_DATA_ROOT}/Examples)\n") fd.write("set(TEMP ${OTB_BINARY_DIR}/Testing/Temporary)\n") fd.write("set(EXE_TESTS ${CXX_TEST_PATH}/otb"+mod+"ExamplesTests)\n") # declare tests fd.write("\n#----------- TESTS DECLARATION ----------------\n") for srcName in testCode: for tName in testCode[srcName]: for line in testCode[srcName][tName]["code"]: cleanLine = line.replace(testCode[srcName][tName]["exeName"],"${EXE_TESTS}") fd.write(cleanLine) # declare test driver fd.write("add_executable(otb"+mod+"ExamplesTests otb"+mod+"ExamplesTests.cxx)\n") fd.write("target_link_libraries(otb"+mod+"ExamplesTests "+\ "${OTB"+mod+"_LIBRARIES} ${OTB"+mod+"-Example_LIBRARIES} "\ "${OTBTestKernel_LIBRARIES})\n\n") fd.write("endif()\n") fd.close() return
def main(argv): manifest = op.expanduser(argv[1]) otbDir = op.expanduser(argv[2]) outputDir = argv[3] testDepends = op.expanduser(argv[4]) testing_dir = op.join(otbDir,"Testing") [groups,moduleList,sourceList] = manifestParser.parseManifest(manifest) for mod in moduleList: if mod == "" or mod == "TBD": continue # remove non-testing source files for src in moduleList[mod]: cleanSrc = src.strip("./") if not cleanSrc.startswith("Testing/"): moduleList[mod].remove(src) if len(moduleList[mod]) == 0: continue currentGrp = "" for grp in groups: if mod in groups[grp]: currentGrp = grp break testMains = {} testFunctions = {} testCode = {} testProp = {} # parse all test files to extract the functions and mains for src in moduleList[mod]: fullSrcPath = op.join(otbDir,src) srcName = op.basename(src) res = createTestManifest.parseTestCxx(fullSrcPath) currentCMake = op.join(op.dirname(fullSrcPath),"CMakeLists.txt") exeName = extractExeName(currentCMake,op.basename(src)) if exeName is "": # this source file is not used -> nothing to do # (or maybe not here ...) continue exeAlias = checkForAlias(currentCMake,exeName) # get add_test() code calling "src" if res["hasMain"]: testMains[srcName] = exeName testCode[srcName] = findTestFromExe(currentCMake,exeName,exeAlias) else: testFunctions[srcName] = res["testFunctions"] testCode[srcName] = findTestFromExe(currentCMake,exeName,exeAlias,res["testFunctions"]) # get set_test_properties() testProp[srcName] = findTestProperties(currentCMake,testCode[srcName].keys()) if len(testCode) == 0: continue targetDir = op.join(op.join(op.join(op.join(outputDir,"Modules"),currentGrp),mod),"test") if op.exists(op.join(targetDir,"CMakeLists.txt")): continue # prepare output directory call(["mkdir","-p",targetDir]) if len(testFunctions)>0: # generate the test driver source code testDriver = op.join(targetDir,"otb"+mod+"TestDriver.cxx") fd = open(testDriver,'wb') fd.write("#include \"otbTestMain.h\"\n") fd.write("void RegisterTests()\n") fd.write("{\n") for srcName in testFunctions: for tFunc in testFunctions[srcName]: fd.write(" REGISTER_TEST("+tFunc+");\n") fd.write("}\n") fd.close() # generate CMakeLists.txt testCmakefile = op.join(targetDir,"CMakeLists.txt") fd = open(testCmakefile,'wb') fd.write("otb_module_test()\n\n") if len(testFunctions)>0: # - declare source files for test driver fd.write("set(OTB"+mod+"Tests\n") fd.write("otb"+mod+"TestDriver.cxx\n") for srcName in testFunctions: fd.write(srcName+"\n") fd.write(")\n\n") # - add test driver executable testdriverdecl = """\ add_executable(otb%sTestDriver ${OTB%sTests}) target_link_libraries(otb%sTestDriver ${OTB%s-Test_LIBRARIES}) otb_module_target_label(otb%sTestDriver) """ % (mod, mod, mod, mod, mod) fd.write(testdriverdecl); # - add other executables for srcName in testMains: testdriverdecl = """\ add_executable(%s %s) target_link_libraries(%s ${OTB%s-Test_LIBRARIES}) otb_module_target_label(otb%sTestDriver) """ % (testMains[srcName], srcName, testMains[srcName], testMains[srcName], mod) fd.write("\n# Tests Declaration\n\n") # add tests for srcName in testCode: for tName in testCode[srcName]: skip=False if tName.count("${"): print "Warning : test name contains a variable : "+tName skip=True if skip: continue tCmakeCode = [] if srcName in testFunctions: exeNameReplaced = False for i, line in zip(range(len(testCode[srcName][tName]["code"])), testCode[srcName][tName]["code"]): line=line.strip(' \t') if i == 0: if "NAME" not in line: line=line.replace(" ", " COMMAND ", 1) line=line.replace("add_test(", "otb_add_test(NAME ") else: line=line.replace("add_test(", "otb_add_test(") # replace large input references if line.find('${OTB_DATA_LARGEINPUT_ROOT}') != -1: start = line.find('${OTB_DATA_LARGEINPUT_ROOT}') end1 = line.find(' ', start) end2 = line.find(')', start) if end1 == -1: end1=end2 if end2 == -1: end2=end1 end = min(end1,end2) before = line[:start] after = line[end:] largepath = line[start + len('${OTB_DATA_LARGEINPUT_ROOT}/'):end] line = before + "LARGEINPUT{" + largepath + "}" + after if exeNameReplaced: tCmakeCode.append(line) else: tCmakeCode.append(line.replace(testCode[srcName][tName]["exeName"],"otb"+mod+"TestDriver",1)) if line.find(testCode[srcName][tName]["exeName"]) >= 0: exeNameReplaced = True else: tCmakeCode = testCode[srcName][tName]["code"] tCmakeCodeFinal = [] # indent for i, line in zip(range(len(tCmakeCode)), tCmakeCode): outputline = line if i != 0: outputline = '%s%s' % (' ', outputline) tCmakeCodeFinal.append(outputline) # add set_property if any if testProp[srcName].has_key(tName): tCmakeCodeFinal += testProp[srcName][tName] fd.writelines(tCmakeCodeFinal) fd.write("\n") fd.close() return
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) outManifest = argv[4] example_dir = op.join(otbDir, "Examples") # Standard Manifest parsing, extract simple and full dependencies [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) # make sure every module is in depList and fullDepList (even if it has no dependencies) for mod in moduleList: if not depList.has_key(mod): depList[mod] = {} if not fullDepList.has_key(mod): fullDepList[mod] = {} depListPerGroup = manifestParser.findGroupDeps(groups, depList) OldFolderPartition = createTestManifest.buildOldFolderPartition(moduleList) exampleCxx = {} outFD = open(outManifest, 'wb') outFD.write( "# Monolithic path, Current dir, group name, module name, subDir name, comment\n" ) # parse all cxx test files : analyse them and extract their dependencies for (d, f) in codeParser.FindBinaries(example_dir): fullPath = op.join(d, f) shortPath = fullPath.replace(otbDir, '.') moduleDestination = "TBD" groupDestination = "TBD" res = createTestManifest.parseTestCxx(fullPath) if res["isTestDriver"]: # no need to dispatch test drivers, they can be generated again continue [exampleDepList, thirdPartyDep ] = createTestManifest.getTestDependencies(res["includes"], sourceList) # if no dependency found, at least put Common if len(exampleDepList) == 0: exampleDepList["Common"] = {"to": "unkown_source"} # try to clean the dependency list (remove inherited modules) ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"] cleanExampleDepList = [] depListToRemove = [] for dep1 in exampleDepList: # register the "from" field exampleDepList[dep1]["from"] = shortPath for dep2 in exampleDepList: if dep2 == dep1: continue # avoid IO modules to 'eat' usefull dependencies if dep1 in ignoreModules: continue if (dep2 in fullDepList[dep1]) and \ (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in exampleDepList: if not dep in depListToRemove: cleanExampleDepList.append(dep) # build all dependencies of the test exampleFullDepList = [] for dep in exampleDepList: for subDep in fullDepList[dep]: if not subDep in exampleFullDepList: exampleFullDepList.append(subDep) # start guessing luckyGuess = None guessStep = 1 # try to get the list of module used to partition the corresponding source directory guessModules = [] guessSourceDir = op.split(shortPath.replace("./Examples", "./Code"))[0] if OldFolderPartition.has_key(guessSourceDir): guessModules = OldFolderPartition[guessSourceDir].keys() # special case for Examples/Application -> ApplicationEngine if guessSourceDir == "./Application": guessModules.append("ApplicationEngine") # first filter : find modules that appear in cleanExampleDepList and in guessModules overlappingModules = [] for dep in cleanExampleDepList: if dep in guessModules: overlappingModules.append(dep) if len(overlappingModules) == 1: luckyGuess = overlappingModules[0] # second filter : find the source file with the closest name if not luckyGuess: guessStep += 1 [matchFile, matchPercent ] = createTestManifest.findClosestSourceName(f, sourceList) if (sourceList[matchFile] in exampleDepList) and (matchPercent > 50.0): luckyGuess = sourceList[matchFile] elif (sourceList[matchFile] in exampleFullDepList) and (matchPercent > 70.0): luckyGuess = sourceList[matchFile] # third guess : # Constrain the search : if the folder containing the test corresponds # to a group name, limit the search to the modules in this group # Also, separate IO examples from non-IO examples if not luckyGuess: folderName = op.basename(d) if folderName == "Classification": folderName = "Learning" if folderName in groups: groupDestination = folderName exampleSmallerDepList = {} for dep in exampleDepList: if groupDestination != "TBD": if dep in groups[groupDestination]: exampleSmallerDepList[dep] = 1 else: if not dep in groups["IO"]: exampleSmallerDepList[dep] = 1 if len(exampleSmallerDepList) == 1: luckyGuess = exampleSmallerDepList.keys()[0] elif len(exampleSmallerDepList) > 1: # filter again to get top-level dependencies doubleCleanDepList = [] depListToRemove = [] for dep1 in exampleSmallerDepList: for dep2 in exampleSmallerDepList: if dep2 == dep1: continue if (dep2 in fullDepList[dep1]) and \ (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in exampleSmallerDepList: if not dep in depListToRemove: doubleCleanDepList.append(dep) if len(doubleCleanDepList) == 1: luckyGuess = doubleCleanDepList[0] elif len(exampleSmallerDepList) == 0: # No dependence in guessed group # choose the most probable module in that group for mod in moduleList: if mod.startswith(folderName) and ( mod in groups[groupDestination]): luckyGuess = mod break # fourth filter : if there is only one dependency in cleanExampleDepList : take it if not luckyGuess: guessStep += 1 if len(cleanExampleDepList) == 1: luckyGuess = cleanExampleDepList[0] # DEBUG if not luckyGuess: print shortPath + " : " + str(exampleDepList.keys()) print shortPath + " : " + str(exampleSmallerDepList.keys()) luckyGuess = "TBD" if luckyGuess: moduleDestination = luckyGuess else: pass #print f + " -> " + str(exampleDepList) #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )" # if module is found and not group, deduce group if groupDestination == "TBD" and moduleDestination != "TBD": groupDestination = manifestParser.getGroup(moduleDestination, groups) exampleCxx[shortPath] = { "depList": exampleDepList, "thirdPartyDep": thirdPartyDep, "group": groupDestination, "module": moduleDestination } outFD.write(shortPath + "," + op.basename(op.dirname(shortPath)) + "," + groupDestination + "," + moduleDestination + ",example,\n") outFD.close()
def main(argv): manifestPath = op.expanduser(argv[1]) moduleDepPath = op.expanduser(argv[2]) otbDir = op.expanduser(argv[3]) outManifest = argv[4] example_dir = op.join(otbDir,"Examples") # Standard Manifest parsing, extract simple and full dependencies [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath) depList = manifestParser.parseDependList(moduleDepPath) fullDepList = manifestParser.buildFullDep(depList) # make sure every module is in depList and fullDepList (even if it has no dependencies) for mod in moduleList: if not depList.has_key(mod): depList[mod] = {} if not fullDepList.has_key(mod): fullDepList[mod] = {} depListPerGroup = manifestParser.findGroupDeps(groups,depList) OldFolderPartition = createTestManifest.buildOldFolderPartition(moduleList) exampleCxx = {} outFD = open(outManifest,'wb') outFD.write("# Monolithic path, Current dir, group name, module name, subDir name, comment\n") # parse all cxx test files : analyse them and extract their dependencies for (d,f) in codeParser.FindBinaries(example_dir): fullPath = op.join(d,f) shortPath = fullPath.replace(otbDir,'.') moduleDestination = "TBD" groupDestination = "TBD" res = createTestManifest.parseTestCxx(fullPath) if res["isTestDriver"]: # no need to dispatch test drivers, they can be generated again continue [exampleDepList,thirdPartyDep] = createTestManifest.getTestDependencies(res["includes"],sourceList) # if no dependency found, at least put Common if len(exampleDepList) == 0: exampleDepList["Common"] = {"to":"unkown_source"} # try to clean the dependency list (remove inherited modules) ignoreModules = ["ImageIO","VectorDataIO","TestKernel"] cleanExampleDepList = [] depListToRemove = [] for dep1 in exampleDepList: # register the "from" field exampleDepList[dep1]["from"] = shortPath for dep2 in exampleDepList: if dep2 == dep1: continue # avoid IO modules to 'eat' usefull dependencies if dep1 in ignoreModules: continue if (dep2 in fullDepList[dep1]) and \ (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in exampleDepList: if not dep in depListToRemove: cleanExampleDepList.append(dep) # build all dependencies of the test exampleFullDepList = [] for dep in exampleDepList: for subDep in fullDepList[dep]: if not subDep in exampleFullDepList: exampleFullDepList.append(subDep) # start guessing luckyGuess = None guessStep = 1 # try to get the list of module used to partition the corresponding source directory guessModules = [] guessSourceDir = op.split(shortPath.replace("./Examples","./Code"))[0] if OldFolderPartition.has_key(guessSourceDir): guessModules = OldFolderPartition[guessSourceDir].keys() # special case for Examples/Application -> ApplicationEngine if guessSourceDir == "./Application": guessModules.append("ApplicationEngine") # first filter : find modules that appear in cleanExampleDepList and in guessModules overlappingModules = [] for dep in cleanExampleDepList: if dep in guessModules: overlappingModules.append(dep) if len(overlappingModules) == 1: luckyGuess = overlappingModules[0] # second filter : find the source file with the closest name if not luckyGuess: guessStep += 1 [matchFile, matchPercent] = createTestManifest.findClosestSourceName(f,sourceList) if (sourceList[matchFile] in exampleDepList) and (matchPercent > 50.0): luckyGuess = sourceList[matchFile] elif (sourceList[matchFile] in exampleFullDepList) and (matchPercent > 70.0): luckyGuess = sourceList[matchFile] # third guess : # Constrain the search : if the folder containing the test corresponds # to a group name, limit the search to the modules in this group # Also, separate IO examples from non-IO examples if not luckyGuess: folderName = op.basename(d) if folderName == "Classification": folderName = "Learning" if folderName in groups: groupDestination = folderName exampleSmallerDepList = {} for dep in exampleDepList: if groupDestination != "TBD": if dep in groups[groupDestination]: exampleSmallerDepList[dep] = 1 else: if not dep in groups["IO"]: exampleSmallerDepList[dep] = 1 if len(exampleSmallerDepList) == 1: luckyGuess = exampleSmallerDepList.keys()[0] elif len(exampleSmallerDepList) > 1: # filter again to get top-level dependencies doubleCleanDepList = [] depListToRemove = [] for dep1 in exampleSmallerDepList: for dep2 in exampleSmallerDepList: if dep2 == dep1: continue if (dep2 in fullDepList[dep1]) and \ (not dep2 in depListToRemove): depListToRemove.append(dep2) for dep in exampleSmallerDepList: if not dep in depListToRemove: doubleCleanDepList.append(dep) if len(doubleCleanDepList) == 1: luckyGuess = doubleCleanDepList[0] elif len(exampleSmallerDepList) == 0: # No dependence in guessed group # choose the most probable module in that group for mod in moduleList: if mod.startswith(folderName) and (mod in groups[groupDestination]): luckyGuess = mod break # fourth filter : if there is only one dependency in cleanExampleDepList : take it if not luckyGuess: guessStep += 1 if len(cleanExampleDepList) == 1: luckyGuess = cleanExampleDepList[0] # DEBUG if not luckyGuess: print shortPath+" : "+str(exampleDepList.keys()) print shortPath+" : "+str(exampleSmallerDepList.keys()) luckyGuess = "TBD" if luckyGuess: moduleDestination = luckyGuess else: pass #print f + " -> " + str(exampleDepList) #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )" # if module is found and not group, deduce group if groupDestination == "TBD" and moduleDestination != "TBD": groupDestination = manifestParser.getGroup(moduleDestination,groups) exampleCxx[shortPath] = {"depList":exampleDepList , "thirdPartyDep":thirdPartyDep, "group":groupDestination, "module":moduleDestination} outFD.write(shortPath+","+op.basename(op.dirname(shortPath))+","+groupDestination+","+moduleDestination+",example,\n") outFD.close()
def main(argv): manifest = op.expanduser(argv[1]) otbDir = op.expanduser(argv[2]) outputDir = argv[3] testDepends = op.expanduser(argv[4]) testing_dir = op.join(otbDir, "Testing") [groups, moduleList, sourceList] = manifestParser.parseManifest(manifest) for mod in moduleList: if mod == "" or mod == "TBD": continue # remove non-testing source files for src in moduleList[mod]: cleanSrc = src.strip("./") if not cleanSrc.startswith("Testing/"): moduleList[mod].remove(src) if len(moduleList[mod]) == 0: continue currentGrp = "" for grp in groups: if mod in groups[grp]: currentGrp = grp break testMains = {} testFunctions = {} testCode = {} testProp = {} # parse all test files to extract the functions and mains for src in moduleList[mod]: fullSrcPath = op.join(otbDir, src) srcName = op.basename(src) res = createTestManifest.parseTestCxx(fullSrcPath) currentCMake = op.join(op.dirname(fullSrcPath), "CMakeLists.txt") exeName = extractExeName(currentCMake, op.basename(src)) if exeName is "": # this source file is not used -> nothing to do # (or maybe not here ...) continue exeAlias = checkForAlias(currentCMake, exeName) # get add_test() code calling "src" if res["hasMain"]: testMains[srcName] = exeName testCode[srcName] = findTestFromExe(currentCMake, exeName, exeAlias) else: testFunctions[srcName] = res["testFunctions"] testCode[srcName] = findTestFromExe(currentCMake, exeName, exeAlias, res["testFunctions"]) # get set_test_properties() testProp[srcName] = findTestProperties(currentCMake, testCode[srcName].keys()) if len(testCode) == 0: continue targetDir = op.join( op.join(op.join(op.join(outputDir, "Modules"), currentGrp), mod), "test") if op.exists(op.join(targetDir, "CMakeLists.txt")): continue # prepare output directory call(["mkdir", "-p", targetDir]) if len(testFunctions) > 0: # generate the test driver source code testDriver = op.join(targetDir, "otb" + mod + "TestDriver.cxx") fd = open(testDriver, 'wb') fd.write("#include \"otbTestMain.h\"\n") fd.write("void RegisterTests()\n") fd.write("{\n") for srcName in testFunctions: for tFunc in testFunctions[srcName]: fd.write(" REGISTER_TEST(" + tFunc + ");\n") fd.write("}\n") fd.close() # generate CMakeLists.txt testCmakefile = op.join(targetDir, "CMakeLists.txt") fd = open(testCmakefile, 'wb') fd.write("otb_module_test()\n\n") if len(testFunctions) > 0: # - declare source files for test driver fd.write("set(OTB" + mod + "Tests\n") fd.write("otb" + mod + "TestDriver.cxx\n") for srcName in testFunctions: fd.write(srcName + "\n") fd.write(")\n\n") # - add test driver executable testdriverdecl = """\ add_executable(otb%sTestDriver ${OTB%sTests}) target_link_libraries(otb%sTestDriver ${OTB%s-Test_LIBRARIES}) otb_module_target_label(otb%sTestDriver) """ % (mod, mod, mod, mod, mod) fd.write(testdriverdecl) # - add other executables for srcName in testMains: testdriverdecl = """\ add_executable(%s %s) target_link_libraries(%s ${OTB%s-Test_LIBRARIES}) otb_module_target_label(otb%sTestDriver) """ % (testMains[srcName], srcName, testMains[srcName], testMains[srcName], mod) fd.write("\n# Tests Declaration\n\n") # add tests for srcName in testCode: for tName in testCode[srcName]: skip = False if tName.count("${"): print "Warning : test name contains a variable : " + tName skip = True if skip: continue tCmakeCode = [] if srcName in testFunctions: exeNameReplaced = False for i, line in zip( range(len(testCode[srcName][tName]["code"])), testCode[srcName][tName]["code"]): line = line.strip(' \t') if i == 0: if "NAME" not in line: line = line.replace(" ", " COMMAND ", 1) line = line.replace("add_test(", "otb_add_test(NAME ") else: line = line.replace("add_test(", "otb_add_test(") # replace large input references if line.find('${OTB_DATA_LARGEINPUT_ROOT}') != -1: start = line.find('${OTB_DATA_LARGEINPUT_ROOT}') end1 = line.find(' ', start) end2 = line.find(')', start) if end1 == -1: end1 = end2 if end2 == -1: end2 = end1 end = min(end1, end2) before = line[:start] after = line[end:] largepath = line[start + len('${OTB_DATA_LARGEINPUT_ROOT}/' ):end] line = before + "LARGEINPUT{" + largepath + "}" + after if exeNameReplaced: tCmakeCode.append(line) else: tCmakeCode.append( line.replace( testCode[srcName][tName]["exeName"], "otb" + mod + "TestDriver", 1)) if line.find(testCode[srcName][tName]["exeName"]) >= 0: exeNameReplaced = True else: tCmakeCode = testCode[srcName][tName]["code"] tCmakeCodeFinal = [] # indent for i, line in zip(range(len(tCmakeCode)), tCmakeCode): outputline = line if i != 0: outputline = '%s%s' % (' ', outputline) tCmakeCodeFinal.append(outputline) # add set_property if any if testProp[srcName].has_key(tName): tCmakeCodeFinal += testProp[srcName][tName] fd.writelines(tCmakeCodeFinal) fd.write("\n") fd.close() return