def main(argv):
    manifestPath = op.expanduser(argv[1])
    moduleDepPath = op.expanduser(argv[2])
    otbDir = op.expanduser(argv[3])
    appManifest = argv[4]
    csvAppDepends = argv[5]

    #app_dir = op.join(otbDir,"Applications")

    # Standard Manifest parsing, extract simple and full dependencies
    [groups, moduleList,
     sourceList] = manifestParser.parseManifest(manifestPath)
    depList = manifestParser.parseDependList(moduleDepPath)
    fullDepList = manifestParser.buildFullDep(depList)

    [appGroups, appModuleList,
     appSourceList] = manifestParser.parseManifest(appManifest)

    # add application sources to sourceList
    for item in appSourceList:
        sourceList[item] = appSourceList[item]

    appDependsList = manifestParser.buildSimpleDep(otbDir, appModuleList,
                                                   sourceList)

    #manifestParser.printDepList(appDependsList)

    manifestParser.outputCSVEdgeList(appDependsList, csvAppDepends)
def main(argv):
  manifestPath = op.expanduser(argv[1])
  moduleDepPath = op.expanduser(argv[2])
  otbDir = op.expanduser(argv[3])
  appManifest = argv[4]
  csvAppDepends = argv[5]
  
  #app_dir = op.join(otbDir,"Applications")
  
  # Standard Manifest parsing, extract simple and full dependencies
  [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath)
  depList = manifestParser.parseDependList(moduleDepPath)
  fullDepList = manifestParser.buildFullDep(depList)
  
  [appGroups,appModuleList,appSourceList] = manifestParser.parseManifest(appManifest)
  
  # add application sources to sourceList
  for item in appSourceList:
    sourceList[item] = appSourceList[item]
  
  appDependsList = manifestParser.buildSimpleDep(otbDir,appModuleList,sourceList)
  
  #manifestParser.printDepList(appDependsList)
  
  manifestParser.outputCSVEdgeList(appDependsList,csvAppDepends)
Beispiel #3
0
def main(argv):
  manifestPath = op.expanduser(argv[1])
  moduleDepPath = op.expanduser(argv[2])
  otbDir = op.expanduser(argv[3])
  exManifest = argv[4]
  csvExDepends = argv[5]
  
  # Standard Manifest parsing, extract simple and full dependencies
  [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath)
  depList = manifestParser.parseDependList(moduleDepPath)
  fullDepList = manifestParser.buildFullDep(depList)
  
  [exGroups,exModuleList,exSourceList] = manifestParser.parseManifest(exManifest)
  
  exDependsList = manifestParser.buildSimpleDep(otbDir,exModuleList,sourceList)
  
  # clean the dependencies : remove modules already in fullDepList
  cleanDepList = {}
  for mod in exDependsList:
    cleanDepList[mod] = {}
    for dep in exDependsList[mod]:
      if not dep in fullDepList[mod]:
        cleanDepList[mod][dep] = 1
  
  #manifestParser.printDepList(exDependsList)
  
  manifestParser.outputCSVEdgeList(cleanDepList,csvExDepends)
Beispiel #4
0
def main(argv):
    manifestPath = op.expanduser(argv[1])
    moduleDepPath = op.expanduser(argv[2])
    otbDir = op.expanduser(argv[3])
    outManifest = argv[4]

    if len(argv) >= 6:
        csvTestDepends = argv[5]
    else:
        csvTestDepends = None

    testing_dir = op.join(otbDir, "Testing")

    # Standard Manifest parsing, extract simple and full dependencies
    [groups, moduleList,
     sourceList] = manifestParser.parseManifest(manifestPath)
    depList = manifestParser.parseDependList(moduleDepPath)
    fullDepList = manifestParser.buildFullDep(depList)
    # make sure every module is in depList and fullDepList (even if it has no dependencies)
    for mod in moduleList:
        if not depList.has_key(mod):
            depList[mod] = {}
        if not fullDepList.has_key(mod):
            fullDepList[mod] = {}

    OldFolderPartition = buildOldFolderPartition(moduleList)

    testCxx = {}

    outFD = open(outManifest, 'wb')
    outFD.write(
        "# Monolithic path, Current dir, group name, module name, subDir name, comment\n"
    )

    # parse all cxx test files : analyse them and extract their dependencies
    for (d, f) in codeParser.FindBinaries(testing_dir):
        fullPath = op.join(d, f)
        shortPath = fullPath.replace(otbDir, '.')

        # skip Testing/Utilities , will not be used anymore
        if shortPath.startswith("./Testing/Utilities/"):
            continue

        moduleDestination = "TBD"
        groupDestination = "TBD"

        res = parseTestCxx(fullPath)

        if res["isTestDriver"]:
            # no need to dispatch test drivers, they can be generated again
            continue

        [testDepList,
         thirdPartyDep] = getTestDependencies(res["includes"], sourceList)

        # try to clean the dependency list (remove inherited modules)
        ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"]
        cleanTestDepList = []
        depListToRemove = []
        for dep1 in testDepList:
            # register the "from" field
            testDepList[dep1]["from"] = shortPath

            for dep2 in testDepList:
                if dep2 == dep1:
                    continue
                # avoid IO modules to 'eat' usefull dependencies
                if dep1 in ignoreModules:
                    continue
                if (dep2 in fullDepList[dep1]) and \
                   (not dep2 in depListToRemove):
                    depListToRemove.append(dep2)
        for dep in testDepList:
            if not dep in depListToRemove:
                cleanTestDepList.append(dep)

        # build all dependencies of the test
        testFullDepList = []
        for dep in testDepList:
            for subDep in fullDepList[dep]:
                if not subDep in testFullDepList:
                    testFullDepList.append(subDep)

        # start guessing
        luckyGuess = None
        guessStep = 1

        # try to get the list of module used to partition the corresponding source directory
        guessModules = []
        guessSourceDir = op.split(shortPath.replace("./Testing", "."))[0]
        if OldFolderPartition.has_key(guessSourceDir):
            guessModules = OldFolderPartition[guessSourceDir].keys()

        # special case for Testing/Application  -> ApplicationEngine
        if guessSourceDir == "./Applications":
            guessModules.append("ApplicationEngine")

        # first filter : find modules that appear in cleanTestDepList and in guessModules
        overlappingModules = []
        for dep in cleanTestDepList:
            if dep in guessModules:
                overlappingModules.append(dep)
        if len(overlappingModules) == 1:
            luckyGuess = overlappingModules[0]

        # second filter : find the source file with the closest name
        if not luckyGuess:
            guessStep += 1
            [matchFile, matchPercent] = findClosestSourceName(f, sourceList)
            if (sourceList[matchFile]
                    in testDepList) and (matchPercent > 50.0):
                luckyGuess = sourceList[matchFile]
            elif (sourceList[matchFile]
                  in testFullDepList) and (matchPercent > 70.0):
                luckyGuess = sourceList[matchFile]

        # third filter : ThirdParty
        if not luckyGuess:
            guessStep += 1
            if guessSourceDir == "./Utilities" or len(testDepList) == 0:
                groupDestination = "ThirdParty"
                if len(thirdPartyDep) == 1:
                    luckyGuess = thirdPartyDep[0]

        # fourth filter : if there is only one dependency in cleanTestDepList : take it
        if not luckyGuess:
            guessStep += 1
            if len(cleanTestDepList) == 1:
                luckyGuess = cleanTestDepList[0]

        # fifth filter : separate IO test from non-IO test
        if not luckyGuess:
            guessStep += 1
            if (f.find("Reader") >= 0) or (f.find("Reading") >= 0) or \
               (f.find("Write") >= 0) or (f.find("Writing") >= 0) or \
               (f.find("ImageIO") >= 0) or (guessSourceDir == "./Code/IO"):
                # remove non-IO deps from cleanTestDepList and look what's left
                ioCleanDep = []
                for dep in cleanTestDepList:
                    if manifestParser.getGroup(dep, groups) == "IO":
                        ioCleanDep.append(dep)
                # ImageIO should be low priority compared to other IO modules
                if (len(ioCleanDep) == 2) and ("ImageIO" in ioCleanDep):
                    ioCleanDep.remove("ImageIO")
                if len(ioCleanDep) == 1:
                    luckyGuess = ioCleanDep[0]
            else:
                # remove non-IO deps from cleanTestDepList and look what's left
                nonIOcleanDep = []
                for dep in cleanTestDepList:
                    if manifestParser.getGroup(dep, groups) != "IO":
                        nonIOcleanDep.append(dep)
                if len(nonIOcleanDep) == 1:
                    luckyGuess = nonIOcleanDep[0]
                elif len(nonIOcleanDep) == 2:
                    # compare the 2 possible modules based on their group
                    groupAandB = [
                        manifestParser.getGroup(nonIOcleanDep[0], groups),
                        manifestParser.getGroup(nonIOcleanDep[1], groups)
                    ]
                    levelAandB = [0, 0]
                    for idx in [0, 1]:
                        if groupAandB[idx] == "Core":
                            levelAandB[idx] = 1
                        elif groupAandB[idx] == "Filtering":
                            levelAandB[idx] = 2
                        else:
                            levelAandB[idx] = 3
                    if levelAandB[0] > levelAandB[1]:
                        luckyGuess = nonIOcleanDep[0]
                    if levelAandB[0] < levelAandB[1]:
                        luckyGuess = nonIOcleanDep[1]

        if luckyGuess:
            moduleDestination = luckyGuess
        else:
            pass
            #print f + " -> " + str(testDepList)
            #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )"

        # if module is found and not group, deduce group
        if groupDestination == "TBD" and moduleDestination != "TBD":
            groupDestination = manifestParser.getGroup(moduleDestination,
                                                       groups)

        if not res["hasMain"]:
            # manually add dependency to TestKernel for cxx using a test driver
            # the include to otbTestMain.h header is not located in the cxx
            testDepList["TestKernel"] = {
                "from": shortPath,
                "to": "./Code/Testing/otbTestMain.h"
            }

        testCxx[shortPath] = {
            "depList": testDepList,
            "thirdPartyDep": thirdPartyDep,
            "group": groupDestination,
            "module": moduleDestination
        }
        outFD.write(shortPath + "," + op.basename(op.dirname(shortPath)) +
                    "," + groupDestination + "," + moduleDestination +
                    ",test,\n")

    outFD.close()

    # sum all test dependencies in every module
    allTestDepends = gatherTestDepends(testCxx, fullDepList)

    # clean the test depends (i.e. ImageIO is dragged by TestKernel)
    cleanTestDepends = {}
    for mod in allTestDepends:
        cleanTestDepends[mod] = {}
        for dep1 in allTestDepends[mod]:
            isClean = True
            for dep2 in allTestDepends[mod]:
                if dep1 == dep2:
                    continue
                if dep1 in fullDepList[dep2]:
                    isClean = False
                    break
            if isClean:
                cleanTestDepends[mod][dep1] = 1

    if csvTestDepends:
        manifestParser.outputCSVEdgeList(cleanTestDepends, csvTestDepends)
def main(argv):
    manifestPath = op.expanduser(argv[1])
    moduleDepPath = op.expanduser(argv[2])
    otbDir = op.expanduser(argv[3])
    outManifest = argv[4]

    if len(argv) >= 6:
        csvTestDepends = argv[5]
    else:
        csvTestDepends = None

    testing_dir = op.join(otbDir, "Testing")

    # Standard Manifest parsing, extract simple and full dependencies
    [groups, moduleList, sourceList] = manifestParser.parseManifest(manifestPath)
    depList = manifestParser.parseDependList(moduleDepPath)
    fullDepList = manifestParser.buildFullDep(depList)
    # make sure every module is in depList and fullDepList (even if it has no dependencies)
    for mod in moduleList:
        if not depList.has_key(mod):
            depList[mod] = {}
        if not fullDepList.has_key(mod):
            fullDepList[mod] = {}

    OldFolderPartition = buildOldFolderPartition(moduleList)

    testCxx = {}

    outFD = open(outManifest, "wb")
    outFD.write("# Monolithic path, Current dir, group name, module name, subDir name, comment\n")

    # parse all cxx test files : analyse them and extract their dependencies
    for (d, f) in codeParser.FindBinaries(testing_dir):
        fullPath = op.join(d, f)
        shortPath = fullPath.replace(otbDir, ".")

        # skip Testing/Utilities , will not be used anymore
        if shortPath.startswith("./Testing/Utilities/"):
            continue

        moduleDestination = "TBD"
        groupDestination = "TBD"

        res = parseTestCxx(fullPath)

        if res["isTestDriver"]:
            # no need to dispatch test drivers, they can be generated again
            continue

        [testDepList, thirdPartyDep] = getTestDependencies(res["includes"], sourceList)

        # try to clean the dependency list (remove inherited modules)
        ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"]
        cleanTestDepList = []
        depListToRemove = []
        for dep1 in testDepList:
            # register the "from" field
            testDepList[dep1]["from"] = shortPath

            for dep2 in testDepList:
                if dep2 == dep1:
                    continue
                # avoid IO modules to 'eat' usefull dependencies
                if dep1 in ignoreModules:
                    continue
                if (dep2 in fullDepList[dep1]) and (not dep2 in depListToRemove):
                    depListToRemove.append(dep2)
        for dep in testDepList:
            if not dep in depListToRemove:
                cleanTestDepList.append(dep)

        # build all dependencies of the test
        testFullDepList = []
        for dep in testDepList:
            for subDep in fullDepList[dep]:
                if not subDep in testFullDepList:
                    testFullDepList.append(subDep)

        # start guessing
        luckyGuess = None
        guessStep = 1

        # try to get the list of module used to partition the corresponding source directory
        guessModules = []
        guessSourceDir = op.split(shortPath.replace("./Testing", "."))[0]
        if OldFolderPartition.has_key(guessSourceDir):
            guessModules = OldFolderPartition[guessSourceDir].keys()

        # special case for Testing/Application  -> ApplicationEngine
        if guessSourceDir == "./Applications":
            guessModules.append("ApplicationEngine")

        # first filter : find modules that appear in cleanTestDepList and in guessModules
        overlappingModules = []
        for dep in cleanTestDepList:
            if dep in guessModules:
                overlappingModules.append(dep)
        if len(overlappingModules) == 1:
            luckyGuess = overlappingModules[0]

        # second filter : find the source file with the closest name
        if not luckyGuess:
            guessStep += 1
            [matchFile, matchPercent] = findClosestSourceName(f, sourceList)
            if (sourceList[matchFile] in testDepList) and (matchPercent > 50.0):
                luckyGuess = sourceList[matchFile]
            elif (sourceList[matchFile] in testFullDepList) and (matchPercent > 70.0):
                luckyGuess = sourceList[matchFile]

        # third filter : ThirdParty
        if not luckyGuess:
            guessStep += 1
            if guessSourceDir == "./Utilities" or len(testDepList) == 0:
                groupDestination = "ThirdParty"
                if len(thirdPartyDep) == 1:
                    luckyGuess = thirdPartyDep[0]

        # fourth filter : if there is only one dependency in cleanTestDepList : take it
        if not luckyGuess:
            guessStep += 1
            if len(cleanTestDepList) == 1:
                luckyGuess = cleanTestDepList[0]

        # fifth filter : separate IO test from non-IO test
        if not luckyGuess:
            guessStep += 1
            if (
                (f.find("Reader") >= 0)
                or (f.find("Reading") >= 0)
                or (f.find("Write") >= 0)
                or (f.find("Writing") >= 0)
                or (f.find("ImageIO") >= 0)
                or (guessSourceDir == "./Code/IO")
            ):
                # remove non-IO deps from cleanTestDepList and look what's left
                ioCleanDep = []
                for dep in cleanTestDepList:
                    if manifestParser.getGroup(dep, groups) == "IO":
                        ioCleanDep.append(dep)
                # ImageIO should be low priority compared to other IO modules
                if (len(ioCleanDep) == 2) and ("ImageIO" in ioCleanDep):
                    ioCleanDep.remove("ImageIO")
                if len(ioCleanDep) == 1:
                    luckyGuess = ioCleanDep[0]
            else:
                # remove non-IO deps from cleanTestDepList and look what's left
                nonIOcleanDep = []
                for dep in cleanTestDepList:
                    if manifestParser.getGroup(dep, groups) != "IO":
                        nonIOcleanDep.append(dep)
                if len(nonIOcleanDep) == 1:
                    luckyGuess = nonIOcleanDep[0]
                elif len(nonIOcleanDep) == 2:
                    # compare the 2 possible modules based on their group
                    groupAandB = [
                        manifestParser.getGroup(nonIOcleanDep[0], groups),
                        manifestParser.getGroup(nonIOcleanDep[1], groups),
                    ]
                    levelAandB = [0, 0]
                    for idx in [0, 1]:
                        if groupAandB[idx] == "Core":
                            levelAandB[idx] = 1
                        elif groupAandB[idx] == "Filtering":
                            levelAandB[idx] = 2
                        else:
                            levelAandB[idx] = 3
                    if levelAandB[0] > levelAandB[1]:
                        luckyGuess = nonIOcleanDep[0]
                    if levelAandB[0] < levelAandB[1]:
                        luckyGuess = nonIOcleanDep[1]

        if luckyGuess:
            moduleDestination = luckyGuess
        else:
            pass
            # print f + " -> " + str(testDepList)
            # print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )"

        # if module is found and not group, deduce group
        if groupDestination == "TBD" and moduleDestination != "TBD":
            groupDestination = manifestParser.getGroup(moduleDestination, groups)

        if not res["hasMain"]:
            # manually add dependency to TestKernel for cxx using a test driver
            # the include to otbTestMain.h header is not located in the cxx
            testDepList["TestKernel"] = {"from": shortPath, "to": "./Code/Testing/otbTestMain.h"}

        testCxx[shortPath] = {
            "depList": testDepList,
            "thirdPartyDep": thirdPartyDep,
            "group": groupDestination,
            "module": moduleDestination,
        }
        outFD.write(
            shortPath
            + ","
            + op.basename(op.dirname(shortPath))
            + ","
            + groupDestination
            + ","
            + moduleDestination
            + ",test,\n"
        )

    outFD.close()

    # sum all test dependencies in every module
    allTestDepends = gatherTestDepends(testCxx, fullDepList)

    # clean the test depends (i.e. ImageIO is dragged by TestKernel)
    cleanTestDepends = {}
    for mod in allTestDepends:
        cleanTestDepends[mod] = {}
        for dep1 in allTestDepends[mod]:
            isClean = True
            for dep2 in allTestDepends[mod]:
                if dep1 == dep2:
                    continue
                if dep1 in fullDepList[dep2]:
                    isClean = False
                    break
            if isClean:
                cleanTestDepends[mod][dep1] = 1

    if csvTestDepends:
        manifestParser.outputCSVEdgeList(cleanTestDepends, csvTestDepends)
Beispiel #6
0
def main(argv):
  otbDir = op.abspath(argv[1])
  targetModule = argv[2]
  targetGroup = "TBD"
  if targetModule.count('/') == 1:
    pos = targetModule.find('/')
    targetGroup = targetModule[0:pos]
    targetModule = targetModule[pos+1:]
  if targetModule == "":
    print "Wrong module name, check input argument : "+argv[2]
  if targetGroup == "":
    print "Wrong group name, check input argument : "+argv[2]

  print "Target module: "+targetGroup+"/"+targetModule

  srcFiles = []
  for item in argv[3:]:
    src = item
    if not src.startswith("Modules/"):
      src = "Modules/"+src
    if op.isfile(op.join(otbDir,src)):
      srcFiles.append(src.strip('./'))
    else:
      print "Input source file not found, will be skipped : "+src
  
  # First, analyse current OTB tree, retrieve :
  #  - module list
  #  - group vs module association
  modulesRoot = op.join(otbDir,"Modules")
  
  [depList, optDepList, testDepList] = sourceAPI.parseOTBModuleCmake(modulesRoot)
  #fullDepList = manifestParser.buildFullDep(depList)
  #oldCleanDepList = sourceAPI.cleanDepList(depList,fullDepList)
  
  [groups,moduleList,sourceList,testList] = sourceAPI.parseModuleRoot(modulesRoot)
  
  # DEBUG
  #manifestParser.printDepList(depList)
  #print str(moduleList)
  
  
  # Second, operate the move
  if targetModule in moduleList:
    targetGroup = manifestParser.getGroup(targetModule,groups)
  else:
    if targetGroup == 'TBD':
      print "Error : group name must be specified for new modules (use group/module syntax)"
      return 1
  
  destinationPrefix = op.join(targetGroup,targetModule)
  for srcFile in srcFiles:
    cleanFile = srcFile.strip('./')
    words = cleanFile.split('/')
    srcMod = words[2]
    srcGrp = words[1]
    targetFile = cleanFile.replace(srcGrp+'/'+srcMod,destinationPrefix,1)
    targetPath = op.join(otbDir,op.dirname(targetFile))
    if not op.isdir(targetPath):
      os.makedirs(targetPath)
    shutil.move(op.join(otbDir,cleanFile),targetPath)
  
  # Compute new modules dependencies
  [newGroups,newModuleList,newSourceList,newTestList] = sourceAPI.parseModuleRoot(modulesRoot)
  newDepList = sourceAPI.buildModularDep(otbDir,newModuleList,newSourceList)
  newTestDepList = sourceAPI.buildModularDep(otbDir,newTestList,newSourceList)
  
  # compute full dependencies
  newFullDepList = manifestParser.buildFullDep(newDepList)
  
  # detect cyclic dependencies
  cyclicDependentModules = []
  for mod in newFullDepList.keys():
    if mod in newFullDepList[mod]:
      if not mod in cyclicDependentModules:
        cyclicDependentModules.append(mod)
  if len(cyclicDependentModules) > 0:
    print "Check for cyclic dependency : Failed"
    for m in cyclicDependentModules:
      cycle = sourceAPI.getCyclicDep(m,newDepList)
      print "Cycle for module "+m+": "+str(cycle)
      if len(cycle) > 0:
        a = m
        for c in cycle:
          b = c
          print a+" -> "+b+": "
          for dep in newDepList[a][b]:
            print "\t"+dep["from"]+" -> "+dep["to"]
          a = c
    
    #manifestParser.printDepList(newDepList,cyclicDependentModules)
    return 1
  else:
    print "Check for cyclic dependency : Passed"
  
  # fix srcList and test declaration
  os.chdir(otbDir)
  for srcFile in srcFiles:
    cleanFile = srcFile.strip('./')
    words = cleanFile.split('/')
    srcMod = words[2]
    srcGrp = words[1]
    srcSub = words[3]
    targetFile = cleanFile.replace(srcGrp+"/"+srcMod,destinationPrefix,1)
    # call hg rename -A
    call(["hg","rename","-A",cleanFile,targetFile])
    #  - for input files in 'src' : adapt OTBModule_SRC
    if srcSub == "src" and len(words) == 5:
      # remove entry in previous module
      removed = [op.basename(srcFile)]
      added = []
      cmakelistPath = op.join(modulesRoot,srcGrp,srcMod,srcSub,"CMakeLists.txt")
      # Check that there are some source file remaining in src dir
      source_files = filter(os.path.isfile, glob.glob(op.join(modulesRoot,srcGrp,srcMod,srcSub)+'/*.c*'))
      if len(source_files) >0 :
        sourceAPI.updateSourceList(cmakelistPath,"OTB"+srcMod+"_SRC",added,removed)
      else:
        # There are no more sources here, the whole src module can be removed
        print "Removing "+srcSub+" dir in "+srcGrp+"/"+srcMod+", since it does not contain source files anymore"
        call(["hg","remove",op.join(modulesRoot,srcGrp,srcMod,srcSub)+"/*"])
      # add entry in target module
      removed = []
      added = [op.basename(srcFile)]
      cmakelistPath = op.join(modulesRoot,destinationPrefix,srcSub,"CMakeLists.txt")
      if not op.exists(cmakelistPath):
        sourceAPI.initializeSrcCMakeLists(cmakelistPath,targetModule)
        call(["hg","add",cmakelistPath.replace(otbDir,".")])
      sourceAPI.updateSourceList(cmakelistPath,"OTB"+targetModule+"_SRC",added,removed)

    #  -for input files in 'app'
    if srcSub == "app" and len(words) == 5:
      cmakelistPath = op.join(modulesRoot,srcGrp,srcMod,srcSub,"CMakeLists.txt")
      nextCmakelistPath = op.join(modulesRoot,destinationPrefix,srcSub,"CMakeLists.txt")
      if not op.exists(nextCmakelistPath):
        sourceAPI.initializeAppCMakeLists(nextCmakelistPath,targetModule)
        call(["hg","add",nextCmakelistPath.replace(otbDir,".")])
      # move the 'otb_create_application' bloc
      appName = sourceAPI.moveAppBloc(cmakelistPath,nextCmakelistPath,op.basename(srcFile))
      
      source_files = filter(os.path.isfile, glob.glob(op.join(modulesRoot,srcGrp,srcMod,srcSub)+'/*.c*'))
      if len(source_files) == 0 :
        # There are no more sources here, the whole src module can be removed
        print "Removing "+srcSub+" dir in "+srcGrp+"/"+srcMod+", since it does not contain source files anymore"
        call(["hg","remove",op.join(modulesRoot,srcGrp,srcMod,srcSub)+"/*"])
      
      # move tests
      testCmakelistPath = op.join(modulesRoot,srcGrp,srcMod,"test","CMakeLists.txt")
      nextTestCmakelistPath = op.join(modulesRoot,destinationPrefix,"test","CMakeLists.txt")
      appTestBlocs = {}
      if op.exists(testCmakelistPath):
        appTestBlocs = sourceAPI.extractAppTestBlocs(testCmakelistPath,appName)
      
      if len(appTestBlocs) > 0:
        if not op.exists(nextTestCmakelistPath):
          if not op.exists(op.dirname(nextTestCmakelistPath)):
            os.makedirs(op.dirname(nextTestCmakelistPath))
          fd = open(nextTestCmakelistPath,'w')
          fd.write("otb_module_test()\n")
          fd.write("\n")
          fd.close()
          call(["hg","add",nextTestCmakelistPath.replace(otbDir,".")])
        
        fd = open(nextTestCmakelistPath,'a')
        fd.write("#----------- "+appName+" TESTS ----------------\n")
        for tname in appTestBlocs:
          fd.writelines(appTestBlocs[tname])
          fd.write("\n")
        fd.close()
    
    if srcSub == "test" and len(words) == 5:
      # analyse test
      res = createTestManifest.parseTestCxx(op.join(otbDir,targetFile))
      
      if res["hasMain"]:
        print "Test with main ("+targetFile+") : not handled for now"
      else:
        # remove entry in previous module source list
        removed = [op.basename(srcFile)]
        added = []
        oldCmakelistPath = op.join(modulesRoot,op.join(srcGrp,op.join(srcMod,"test/CMakeLists.txt")))
        oldTestDriverPath = op.join(modulesRoot,op.join(srcGrp,op.join(srcMod,"test/otb"+srcMod+"TestDriver.cxx")))
        sourceAPI.updateSourceList(oldCmakelistPath,"OTB"+srcMod+"Tests",added,removed)
        
        # add entry in target module source list
        removed = []
        added = [op.basename(srcFile)]
        cmakelistPath = op.join(modulesRoot,op.join(destinationPrefix,"test/CMakeLists.txt"))
        testDriverPath = op.join(modulesRoot,op.join(destinationPrefix,"test/otb"+targetModule+"TestDriver.cxx"))
        if not op.exists(cmakelistPath):
          # there was no test before : initialize CMakeLists.txt
          sourceAPI.initializeTestCMakeLists(cmakelistPath,targetModule)
          call(["hg","add",cmakelistPath.replace(otbDir,".")])
        if not op.exists(testDriverPath):
          # there was no test before : initialize test driver
          sourceAPI.initializeTestDriver(testDriverPath)
          call(["hg","add",testDriverPath.replace(otbDir,".")])
        sourceAPI.updateSourceList(cmakelistPath,"OTB"+targetModule+"Tests",added,removed)
        
        #  - move test declaration
        testCode = dispatchTests.findTestFromExe(oldCmakelistPath,"otb"+srcMod+"TestDriver","",res["testFunctions"],'otb_')
        
        # get set_test_properties()
        testProp = dispatchTests.findTestProperties(oldCmakelistPath,testCode.keys())
        
        print "Found "+str(len(testCode))+" tests to move"

        sourceAPI.moveTestCode(oldCmakelistPath,cmakelistPath,testCode,testProp)

  for mod in newModuleList:
    curGroup = manifestParser.getGroup(mod,newGroups)
    
    # fix the otb-module.cmake
    cmake_module_path = op.join(modulesRoot,op.join(curGroup,op.join(mod,"otb-module.cmake")))
    cmakel_module_path = op.join(modulesRoot,op.join(curGroup,op.join(mod,"CMakeLists.txt")))
    if (not op.exists(cmake_module_path)) and (mod == targetModule):
      # initialize new otb-module.cmake if new module
      sourceAPI.initializeOTBModuleCmake(cmake_module_path,mod)
      with_lib = False
      if op.exists(op.join(curGroup,op.join(mod,"src"))):
        with_lib = True
      sourceAPI.initializeOTBModuleCMakeLists(cmakel_module_path,mod,with_lib)
      call(["hg","add",cmakel_module_path.replace(otbDir,".")])
      
      call(["hg","add",cmake_module_path.replace(otbDir,".")])
    
    #  - fix the target_link_libraries
    src_dir = op.join(modulesRoot,curGroup,mod,"src")

    sub_src_CMakeList = op.join(modulesRoot,op.join(curGroup,op.join(mod,"src/CMakeLists.txt")))

  # Update the module dependencies
  print "Updating module dependencies ..."
  count = updateModuleDependencies.update(otbDir)
  print str(count)+" modules were updated"
  
  # Update the module test driver
  print "Updating test drivers ..."
  count = updateTestDrivers.update(otbDir)
  print str(count)+" files updated"

  # Update the group declaration in doxygen
  print "Updating doxygen group declaration ..."
  updateDoxyGroup.update(otbDir)

  # TODO : hg commit by the user
  print "\nTo commit those changes, run: hg commit -m \"ENH: Automatic move of files to module "+targetGroup+"/"+targetModule+"\"\n"
  print "It is advised to run updateModuleDependencies.py script beforehand.\n"
Beispiel #7
0
def main(argv):
    otbDir = op.abspath(argv[1])
    targetModule = argv[2]
    targetGroup = "TBD"
    if targetModule.count('/') == 1:
        pos = targetModule.find('/')
        targetGroup = targetModule[0:pos]
        targetModule = targetModule[pos + 1:]
    if targetModule == "":
        print "Wrong module name, check input argument : " + argv[2]
    if targetGroup == "":
        print "Wrong group name, check input argument : " + argv[2]

    print "Target module: " + targetGroup + "/" + targetModule

    srcFiles = []
    for item in argv[3:]:
        src = item
        if not src.startswith("Modules/"):
            src = "Modules/" + src
        if op.isfile(op.join(otbDir, src)):
            srcFiles.append(src.strip('./'))
        else:
            print "Input source file not found, will be skipped : " + src

    # First, analyse current OTB tree, retrieve :
    #  - module list
    #  - group vs module association
    modulesRoot = op.join(otbDir, "Modules")

    [depList, optDepList,
     testDepList] = sourceAPI.parseOTBModuleCmake(modulesRoot)
    #fullDepList = manifestParser.buildFullDep(depList)
    #oldCleanDepList = sourceAPI.cleanDepList(depList,fullDepList)

    [groups, moduleList, sourceList,
     testList] = sourceAPI.parseModuleRoot(modulesRoot)

    # DEBUG
    #manifestParser.printDepList(depList)
    #print str(moduleList)

    # Second, operate the move
    if targetModule in moduleList:
        targetGroup = manifestParser.getGroup(targetModule, groups)
    else:
        if targetGroup == 'TBD':
            print "Error : group name must be specified for new modules (use group/module syntax)"
            return 1

    destinationPrefix = op.join(targetGroup, targetModule)
    for srcFile in srcFiles:
        cleanFile = srcFile.strip('./')
        words = cleanFile.split('/')
        srcMod = words[2]
        srcGrp = words[1]
        targetFile = cleanFile.replace(srcGrp + '/' + srcMod,
                                       destinationPrefix, 1)
        targetPath = op.join(otbDir, op.dirname(targetFile))
        if not op.isdir(targetPath):
            os.makedirs(targetPath)
        shutil.move(op.join(otbDir, cleanFile), targetPath)

    # Compute new modules dependencies
    [newGroups, newModuleList, newSourceList,
     newTestList] = sourceAPI.parseModuleRoot(modulesRoot)
    newDepList = sourceAPI.buildModularDep(otbDir, newModuleList,
                                           newSourceList)
    newTestDepList = sourceAPI.buildModularDep(otbDir, newTestList,
                                               newSourceList)

    # compute full dependencies
    newFullDepList = manifestParser.buildFullDep(newDepList)

    # detect cyclic dependencies
    cyclicDependentModules = []
    for mod in newFullDepList.keys():
        if mod in newFullDepList[mod]:
            if not mod in cyclicDependentModules:
                cyclicDependentModules.append(mod)
    if len(cyclicDependentModules) > 0:
        print "Check for cyclic dependency : Failed"
        for m in cyclicDependentModules:
            cycle = sourceAPI.getCyclicDep(m, newDepList)
            print "Cycle for module " + m + ": " + str(cycle)
            if len(cycle) > 0:
                a = m
                for c in cycle:
                    b = c
                    print a + " -> " + b + ": "
                    for dep in newDepList[a][b]:
                        print "\t" + dep["from"] + " -> " + dep["to"]
                    a = c

        #manifestParser.printDepList(newDepList,cyclicDependentModules)
        return 1
    else:
        print "Check for cyclic dependency : Passed"

    # fix srcList and test declaration
    os.chdir(otbDir)
    for srcFile in srcFiles:
        cleanFile = srcFile.strip('./')
        words = cleanFile.split('/')
        srcMod = words[2]
        srcGrp = words[1]
        srcSub = words[3]
        targetFile = cleanFile.replace(srcGrp + "/" + srcMod,
                                       destinationPrefix, 1)
        # call hg rename -A
        call(["hg", "rename", "-A", cleanFile, targetFile])
        #  - for input files in 'src' : adapt OTBModule_SRC
        if srcSub == "src" and len(words) == 5:
            # remove entry in previous module
            removed = [op.basename(srcFile)]
            added = []
            cmakelistPath = op.join(modulesRoot, srcGrp, srcMod, srcSub,
                                    "CMakeLists.txt")
            # Check that there are some source file remaining in src dir
            source_files = filter(
                os.path.isfile,
                glob.glob(
                    op.join(modulesRoot, srcGrp, srcMod, srcSub) + '/*.c*'))
            if len(source_files) > 0:
                sourceAPI.updateSourceList(cmakelistPath,
                                           "OTB" + srcMod + "_SRC", added,
                                           removed)
            else:
                # There are no more sources here, the whole src module can be removed
                print "Removing " + srcSub + " dir in " + srcGrp + "/" + srcMod + ", since it does not contain source files anymore"
                call([
                    "hg", "remove",
                    op.join(modulesRoot, srcGrp, srcMod, srcSub) + "/*"
                ])
            # add entry in target module
            removed = []
            added = [op.basename(srcFile)]
            cmakelistPath = op.join(modulesRoot, destinationPrefix, srcSub,
                                    "CMakeLists.txt")
            if not op.exists(cmakelistPath):
                sourceAPI.initializeSrcCMakeLists(cmakelistPath, targetModule)
                call(["hg", "add", cmakelistPath.replace(otbDir, ".")])
            sourceAPI.updateSourceList(cmakelistPath,
                                       "OTB" + targetModule + "_SRC", added,
                                       removed)

        #  -for input files in 'app'
        if srcSub == "app" and len(words) == 5:
            cmakelistPath = op.join(modulesRoot, srcGrp, srcMod, srcSub,
                                    "CMakeLists.txt")
            nextCmakelistPath = op.join(modulesRoot, destinationPrefix, srcSub,
                                        "CMakeLists.txt")
            if not op.exists(nextCmakelistPath):
                sourceAPI.initializeAppCMakeLists(nextCmakelistPath,
                                                  targetModule)
                call(["hg", "add", nextCmakelistPath.replace(otbDir, ".")])
            # move the 'otb_create_application' bloc
            appName = sourceAPI.moveAppBloc(cmakelistPath, nextCmakelistPath,
                                            op.basename(srcFile))

            source_files = filter(
                os.path.isfile,
                glob.glob(
                    op.join(modulesRoot, srcGrp, srcMod, srcSub) + '/*.c*'))
            if len(source_files) == 0:
                # There are no more sources here, the whole src module can be removed
                print "Removing " + srcSub + " dir in " + srcGrp + "/" + srcMod + ", since it does not contain source files anymore"
                call([
                    "hg", "remove",
                    op.join(modulesRoot, srcGrp, srcMod, srcSub) + "/*"
                ])

            # move tests
            testCmakelistPath = op.join(modulesRoot, srcGrp, srcMod, "test",
                                        "CMakeLists.txt")
            nextTestCmakelistPath = op.join(modulesRoot, destinationPrefix,
                                            "test", "CMakeLists.txt")
            appTestBlocs = {}
            if op.exists(testCmakelistPath):
                appTestBlocs = sourceAPI.extractAppTestBlocs(
                    testCmakelistPath, appName)

            if len(appTestBlocs) > 0:
                if not op.exists(nextTestCmakelistPath):
                    if not op.exists(op.dirname(nextTestCmakelistPath)):
                        os.makedirs(op.dirname(nextTestCmakelistPath))
                    fd = open(nextTestCmakelistPath, 'w')
                    fd.write("otb_module_test()\n")
                    fd.write("\n")
                    fd.close()
                    call([
                        "hg", "add",
                        nextTestCmakelistPath.replace(otbDir, ".")
                    ])

                fd = open(nextTestCmakelistPath, 'a')
                fd.write("#----------- " + appName +
                         " TESTS ----------------\n")
                for tname in appTestBlocs:
                    fd.writelines(appTestBlocs[tname])
                    fd.write("\n")
                fd.close()

        if srcSub == "test" and len(words) == 5:
            # analyse test
            res = createTestManifest.parseTestCxx(op.join(otbDir, targetFile))

            if res["hasMain"]:
                print "Test with main (" + targetFile + ") : not handled for now"
            else:
                # remove entry in previous module source list
                removed = [op.basename(srcFile)]
                added = []
                oldCmakelistPath = op.join(
                    modulesRoot,
                    op.join(srcGrp, op.join(srcMod, "test/CMakeLists.txt")))
                oldTestDriverPath = op.join(
                    modulesRoot,
                    op.join(
                        srcGrp,
                        op.join(srcMod,
                                "test/otb" + srcMod + "TestDriver.cxx")))
                sourceAPI.updateSourceList(oldCmakelistPath,
                                           "OTB" + srcMod + "Tests", added,
                                           removed)

                # add entry in target module source list
                removed = []
                added = [op.basename(srcFile)]
                cmakelistPath = op.join(
                    modulesRoot,
                    op.join(destinationPrefix, "test/CMakeLists.txt"))
                testDriverPath = op.join(
                    modulesRoot,
                    op.join(destinationPrefix,
                            "test/otb" + targetModule + "TestDriver.cxx"))
                if not op.exists(cmakelistPath):
                    # there was no test before : initialize CMakeLists.txt
                    sourceAPI.initializeTestCMakeLists(cmakelistPath,
                                                       targetModule)
                    call(["hg", "add", cmakelistPath.replace(otbDir, ".")])
                if not op.exists(testDriverPath):
                    # there was no test before : initialize test driver
                    sourceAPI.initializeTestDriver(testDriverPath)
                    call(["hg", "add", testDriverPath.replace(otbDir, ".")])
                sourceAPI.updateSourceList(cmakelistPath,
                                           "OTB" + targetModule + "Tests",
                                           added, removed)

                #  - move test declaration
                testCode = dispatchTests.findTestFromExe(
                    oldCmakelistPath, "otb" + srcMod + "TestDriver", "",
                    res["testFunctions"], 'otb_')

                # get set_test_properties()
                testProp = dispatchTests.findTestProperties(
                    oldCmakelistPath, testCode.keys())

                print "Found " + str(len(testCode)) + " tests to move"

                sourceAPI.moveTestCode(oldCmakelistPath, cmakelistPath,
                                       testCode, testProp)

    for mod in newModuleList:
        curGroup = manifestParser.getGroup(mod, newGroups)

        # fix the otb-module.cmake
        cmake_module_path = op.join(
            modulesRoot, op.join(curGroup, op.join(mod, "otb-module.cmake")))
        cmakel_module_path = op.join(
            modulesRoot, op.join(curGroup, op.join(mod, "CMakeLists.txt")))
        if (not op.exists(cmake_module_path)) and (mod == targetModule):
            # initialize new otb-module.cmake if new module
            sourceAPI.initializeOTBModuleCmake(cmake_module_path, mod)
            with_lib = False
            if op.exists(op.join(curGroup, op.join(mod, "src"))):
                with_lib = True
            sourceAPI.initializeOTBModuleCMakeLists(cmakel_module_path, mod,
                                                    with_lib)
            call(["hg", "add", cmakel_module_path.replace(otbDir, ".")])

            call(["hg", "add", cmake_module_path.replace(otbDir, ".")])

        #  - fix the target_link_libraries
        src_dir = op.join(modulesRoot, curGroup, mod, "src")

        sub_src_CMakeList = op.join(
            modulesRoot, op.join(curGroup, op.join(mod, "src/CMakeLists.txt")))

    # Update the module dependencies
    print "Updating module dependencies ..."
    count = updateModuleDependencies.update(otbDir)
    print str(count) + " modules were updated"

    # Update the module test driver
    print "Updating test drivers ..."
    count = updateTestDrivers.update(otbDir)
    print str(count) + " files updated"

    # Update the group declaration in doxygen
    print "Updating doxygen group declaration ..."
    updateDoxyGroup.update(otbDir)

    # TODO : hg commit by the user
    print "\nTo commit those changes, run: hg commit -m \"ENH: Automatic move of files to module " + targetGroup + "/" + targetModule + "\"\n"
    print "It is advised to run updateModuleDependencies.py script beforehand.\n"
def main(argv):
    manifestPath = op.expanduser(argv[1])
    moduleDepPath = op.expanduser(argv[2])
    otbDir = op.expanduser(argv[3])
    outManifest = argv[4]

    example_dir = op.join(otbDir, "Examples")

    # Standard Manifest parsing, extract simple and full dependencies
    [groups, moduleList,
     sourceList] = manifestParser.parseManifest(manifestPath)
    depList = manifestParser.parseDependList(moduleDepPath)
    fullDepList = manifestParser.buildFullDep(depList)
    # make sure every module is in depList and fullDepList (even if it has no dependencies)
    for mod in moduleList:
        if not depList.has_key(mod):
            depList[mod] = {}
        if not fullDepList.has_key(mod):
            fullDepList[mod] = {}

    depListPerGroup = manifestParser.findGroupDeps(groups, depList)

    OldFolderPartition = createTestManifest.buildOldFolderPartition(moduleList)

    exampleCxx = {}

    outFD = open(outManifest, 'wb')
    outFD.write(
        "# Monolithic path, Current dir, group name, module name, subDir name, comment\n"
    )

    # parse all cxx test files : analyse them and extract their dependencies
    for (d, f) in codeParser.FindBinaries(example_dir):
        fullPath = op.join(d, f)
        shortPath = fullPath.replace(otbDir, '.')

        moduleDestination = "TBD"
        groupDestination = "TBD"

        res = createTestManifest.parseTestCxx(fullPath)

        if res["isTestDriver"]:
            # no need to dispatch test drivers, they can be generated again
            continue

        [exampleDepList, thirdPartyDep
         ] = createTestManifest.getTestDependencies(res["includes"],
                                                    sourceList)

        # if no dependency found, at least put Common
        if len(exampleDepList) == 0:
            exampleDepList["Common"] = {"to": "unkown_source"}

        # try to clean the dependency list (remove inherited modules)
        ignoreModules = ["ImageIO", "VectorDataIO", "TestKernel"]
        cleanExampleDepList = []
        depListToRemove = []
        for dep1 in exampleDepList:
            # register the "from" field
            exampleDepList[dep1]["from"] = shortPath

            for dep2 in exampleDepList:
                if dep2 == dep1:
                    continue
                # avoid IO modules to 'eat' usefull dependencies
                if dep1 in ignoreModules:
                    continue
                if (dep2 in fullDepList[dep1]) and \
                   (not dep2 in depListToRemove):
                    depListToRemove.append(dep2)
        for dep in exampleDepList:
            if not dep in depListToRemove:
                cleanExampleDepList.append(dep)

        # build all dependencies of the test
        exampleFullDepList = []
        for dep in exampleDepList:
            for subDep in fullDepList[dep]:
                if not subDep in exampleFullDepList:
                    exampleFullDepList.append(subDep)

        # start guessing
        luckyGuess = None
        guessStep = 1

        # try to get the list of module used to partition the corresponding source directory
        guessModules = []
        guessSourceDir = op.split(shortPath.replace("./Examples", "./Code"))[0]
        if OldFolderPartition.has_key(guessSourceDir):
            guessModules = OldFolderPartition[guessSourceDir].keys()

        # special case for Examples/Application  -> ApplicationEngine
        if guessSourceDir == "./Application":
            guessModules.append("ApplicationEngine")

        # first filter : find modules that appear in cleanExampleDepList and in guessModules
        overlappingModules = []
        for dep in cleanExampleDepList:
            if dep in guessModules:
                overlappingModules.append(dep)
        if len(overlappingModules) == 1:
            luckyGuess = overlappingModules[0]

        # second filter : find the source file with the closest name
        if not luckyGuess:
            guessStep += 1
            [matchFile, matchPercent
             ] = createTestManifest.findClosestSourceName(f, sourceList)
            if (sourceList[matchFile]
                    in exampleDepList) and (matchPercent > 50.0):
                luckyGuess = sourceList[matchFile]
            elif (sourceList[matchFile]
                  in exampleFullDepList) and (matchPercent > 70.0):
                luckyGuess = sourceList[matchFile]

        # third guess :
        # Constrain the search : if the folder containing the test corresponds
        # to a group name, limit the search to the modules in this group
        # Also, separate IO examples from non-IO examples
        if not luckyGuess:
            folderName = op.basename(d)
            if folderName == "Classification":
                folderName = "Learning"
            if folderName in groups:
                groupDestination = folderName
            exampleSmallerDepList = {}
            for dep in exampleDepList:
                if groupDestination != "TBD":
                    if dep in groups[groupDestination]:
                        exampleSmallerDepList[dep] = 1
                else:
                    if not dep in groups["IO"]:
                        exampleSmallerDepList[dep] = 1
            if len(exampleSmallerDepList) == 1:
                luckyGuess = exampleSmallerDepList.keys()[0]
            elif len(exampleSmallerDepList) > 1:
                # filter again to get top-level dependencies
                doubleCleanDepList = []
                depListToRemove = []
                for dep1 in exampleSmallerDepList:
                    for dep2 in exampleSmallerDepList:
                        if dep2 == dep1:
                            continue
                        if (dep2 in fullDepList[dep1]) and \
                           (not dep2 in depListToRemove):
                            depListToRemove.append(dep2)
                for dep in exampleSmallerDepList:
                    if not dep in depListToRemove:
                        doubleCleanDepList.append(dep)
                if len(doubleCleanDepList) == 1:
                    luckyGuess = doubleCleanDepList[0]
            elif len(exampleSmallerDepList) == 0:
                # No dependence in guessed group
                # choose the most probable module in that group
                for mod in moduleList:
                    if mod.startswith(folderName) and (
                            mod in groups[groupDestination]):
                        luckyGuess = mod
                        break

        # fourth filter : if there is only one dependency in cleanExampleDepList : take it
        if not luckyGuess:
            guessStep += 1
            if len(cleanExampleDepList) == 1:
                luckyGuess = cleanExampleDepList[0]

        # DEBUG
        if not luckyGuess:
            print shortPath + " : " + str(exampleDepList.keys())
            print shortPath + " : " + str(exampleSmallerDepList.keys())
            luckyGuess = "TBD"

        if luckyGuess:
            moduleDestination = luckyGuess
        else:
            pass
            #print f + " -> " + str(exampleDepList)
            #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )"

        # if module is found and not group, deduce group
        if groupDestination == "TBD" and moduleDestination != "TBD":
            groupDestination = manifestParser.getGroup(moduleDestination,
                                                       groups)

        exampleCxx[shortPath] = {
            "depList": exampleDepList,
            "thirdPartyDep": thirdPartyDep,
            "group": groupDestination,
            "module": moduleDestination
        }
        outFD.write(shortPath + "," + op.basename(op.dirname(shortPath)) +
                    "," + groupDestination + "," + moduleDestination +
                    ",example,\n")

    outFD.close()
def main(argv):
  manifestPath = op.expanduser(argv[1])
  moduleDepPath = op.expanduser(argv[2])
  otbDir = op.expanduser(argv[3])
  outManifest = argv[4]
  
  example_dir = op.join(otbDir,"Examples")
  
  # Standard Manifest parsing, extract simple and full dependencies
  [groups,moduleList,sourceList] = manifestParser.parseManifest(manifestPath)
  depList = manifestParser.parseDependList(moduleDepPath)
  fullDepList = manifestParser.buildFullDep(depList)
  # make sure every module is in depList and fullDepList (even if it has no dependencies)
  for mod in moduleList:
    if not depList.has_key(mod):
      depList[mod] = {}
    if not fullDepList.has_key(mod):
      fullDepList[mod] = {}
  
  depListPerGroup = manifestParser.findGroupDeps(groups,depList)
  
  OldFolderPartition = createTestManifest.buildOldFolderPartition(moduleList)
  
  exampleCxx = {}
  
  outFD = open(outManifest,'wb')
  outFD.write("# Monolithic path, Current dir, group name, module name, subDir name, comment\n")
  
  # parse all cxx test files : analyse them and extract their dependencies
  for (d,f) in codeParser.FindBinaries(example_dir):
    fullPath = op.join(d,f)
    shortPath = fullPath.replace(otbDir,'.')
    
    moduleDestination = "TBD"
    groupDestination = "TBD"
    
    res = createTestManifest.parseTestCxx(fullPath)
    
    if res["isTestDriver"]:
      # no need to dispatch test drivers, they can be generated again
      continue
    
    [exampleDepList,thirdPartyDep] = createTestManifest.getTestDependencies(res["includes"],sourceList)
    
    # if no dependency found, at least put Common
    if len(exampleDepList) == 0:
      exampleDepList["Common"] = {"to":"unkown_source"}
    
    # try to clean the dependency list (remove inherited modules)
    ignoreModules = ["ImageIO","VectorDataIO","TestKernel"]
    cleanExampleDepList = []
    depListToRemove = []
    for dep1 in exampleDepList:
      # register the "from" field
      exampleDepList[dep1]["from"] = shortPath
      
      for dep2 in exampleDepList:
        if dep2 == dep1:
          continue
        # avoid IO modules to 'eat' usefull dependencies
        if dep1 in ignoreModules:
          continue
        if (dep2 in fullDepList[dep1]) and \
           (not dep2 in depListToRemove):
          depListToRemove.append(dep2)
    for dep in exampleDepList:
      if not dep in depListToRemove:
        cleanExampleDepList.append(dep)
    
    # build all dependencies of the test
    exampleFullDepList = []
    for dep in exampleDepList:
      for subDep in fullDepList[dep]:
        if not subDep in exampleFullDepList:
          exampleFullDepList.append(subDep)
    
    # start guessing
    luckyGuess = None
    guessStep = 1
    
    # try to get the list of module used to partition the corresponding source directory
    guessModules = []
    guessSourceDir = op.split(shortPath.replace("./Examples","./Code"))[0]
    if OldFolderPartition.has_key(guessSourceDir):
      guessModules = OldFolderPartition[guessSourceDir].keys()
    
    # special case for Examples/Application  -> ApplicationEngine
    if guessSourceDir == "./Application":
      guessModules.append("ApplicationEngine")
    
    # first filter : find modules that appear in cleanExampleDepList and in guessModules
    overlappingModules = []
    for dep in cleanExampleDepList:
      if dep in guessModules:
        overlappingModules.append(dep)
    if len(overlappingModules) == 1:
      luckyGuess = overlappingModules[0]
    
    # second filter : find the source file with the closest name
    if not luckyGuess:
      guessStep += 1
      [matchFile, matchPercent] = createTestManifest.findClosestSourceName(f,sourceList)
      if (sourceList[matchFile] in exampleDepList) and (matchPercent > 50.0):
        luckyGuess = sourceList[matchFile]
      elif (sourceList[matchFile] in exampleFullDepList) and (matchPercent > 70.0):
        luckyGuess = sourceList[matchFile]
    
    # third guess :
    # Constrain the search : if the folder containing the test corresponds
    # to a group name, limit the search to the modules in this group
    # Also, separate IO examples from non-IO examples
    if not luckyGuess:
      folderName = op.basename(d)
      if folderName == "Classification":
        folderName = "Learning"
      if folderName in groups:
        groupDestination = folderName
      exampleSmallerDepList = {}
      for dep in exampleDepList:
        if groupDestination != "TBD":
          if dep in groups[groupDestination]:
            exampleSmallerDepList[dep] = 1
        else:
          if not dep in groups["IO"]:
            exampleSmallerDepList[dep] = 1
      if len(exampleSmallerDepList) == 1:
        luckyGuess = exampleSmallerDepList.keys()[0]
      elif len(exampleSmallerDepList) > 1:
        # filter again to get top-level dependencies
        doubleCleanDepList = []
        depListToRemove = []
        for dep1 in exampleSmallerDepList:
          for dep2 in exampleSmallerDepList:
            if dep2 == dep1:
              continue
            if (dep2 in fullDepList[dep1]) and \
               (not dep2 in depListToRemove):
              depListToRemove.append(dep2)
        for dep in exampleSmallerDepList:
          if not dep in depListToRemove:
            doubleCleanDepList.append(dep)
        if len(doubleCleanDepList) == 1:
          luckyGuess = doubleCleanDepList[0]
      elif len(exampleSmallerDepList) == 0:
        # No dependence in guessed group
        # choose the most probable module in that group
        for mod in moduleList:
          if mod.startswith(folderName) and (mod in groups[groupDestination]):
            luckyGuess = mod
            break
      
    
    # fourth filter : if there is only one dependency in cleanExampleDepList : take it
    if not luckyGuess:
      guessStep += 1
      if len(cleanExampleDepList) == 1:
        luckyGuess = cleanExampleDepList[0]
    
    
    # DEBUG
    if not luckyGuess:
      print shortPath+" : "+str(exampleDepList.keys())
      print shortPath+" : "+str(exampleSmallerDepList.keys()) 
      luckyGuess = "TBD"
    
    if luckyGuess:
      moduleDestination = luckyGuess
    else:
      pass
      #print f + " -> " + str(exampleDepList)
      #print f + " -> "+ matchFile + " ( " + str(matchPercent) + "% )"
    
    # if module is found and not group, deduce group
    if groupDestination == "TBD" and moduleDestination != "TBD":
      groupDestination = manifestParser.getGroup(moduleDestination,groups)
    
    
    exampleCxx[shortPath] = {"depList":exampleDepList , "thirdPartyDep":thirdPartyDep, "group":groupDestination, "module":moduleDestination}
    outFD.write(shortPath+","+op.basename(op.dirname(shortPath))+","+groupDestination+","+moduleDestination+",example,\n")
  
  outFD.close()