def generateStatModel(pathShapes, pathToTiles, pathToStats, pathToCmdStats, pathWd, pathConf): AllCmd = [] modTiles = GM.getModel(pathShapes) cfg = Config(pathConf) Stack_ind = fu.getFeatStackName(pathConf) classifier = Config(file(pathConf)).argTrain.classifier for mod, Tiles in modTiles: allpath = "" for tile in Tiles: pathToFeat = pathToTiles + "/" + tile + "/Final/" + Stack_ind allpath = allpath + " " + pathToFeat + " " # hpc case #if pathWd != None: # pathToStats = "$TMPDIR" if classifier == "svm": cmd = "otbcli_ComputeImagesStatistics -il " + allpath + " -out " + pathToStats + "/Model_" + str( mod) + ".xml" else: cmd = "echo 'random forest does not need stats'" AllCmd.append(cmd) fu.writeCmds(pathToCmdStats + "/stats.txt", AllCmd) return AllCmd
def fusion(pathClassif, pathConf, pathWd): f = file(pathConf) cfg = Config(f) classifMode = cfg.argClassification.classifMode N = int(cfg.chain.runs) allTiles = cfg.chain.listTile.split(" ") fusionOptions = cfg.argClassification.fusionOptions mode = cfg.chain.mode pixType = cfg.argClassification.pixType if mode == "outside": AllClassif = fu.fileSearchRegEx(pathClassif + "/Classif_*_model_*f*_seed_*.tif") allTiles = [] models = [] for classif in AllClassif: mod = classif.split("/")[-1].split("_")[3].split("f")[0] tile = classif.split("/")[-1].split("_")[1] if not mod in models: models.append(mod) if not tile in allTiles: allTiles.append(tile) AllCmd = [] for seed in range(N): for tile in allTiles: directoryOut = pathClassif if pathWd != None: directoryOut = "$TMPDIR" if mode != "outside": classifPath = fu.FileSearch_AND(pathClassif, True, "Classif_" + tile, "seed_" + str(seed) + ".tif") allPathFusion = " ".join(classifPath) cmd = "otbcli_FusionOfClassifications -il " + allPathFusion + " " + fusionOptions + " -out " + directoryOut + "/" + tile + "_FUSION_seed_" + str( seed) + ".tif" AllCmd.append(cmd) else: for mod in models: classifPath = fu.fileSearchRegEx(pathClassif + "/Classif_" + tile + "_model_" + mod + "f*_seed_" + str(seed) + ".tif") if len(classifPath) != 0: allPathFusion = " ".join(classifPath) cmd = "otbcli_FusionOfClassifications -il " + allPathFusion + " " + fusionOptions + " -out " + directoryOut + "/" + tile + "_FUSION_model_" + mod + "_seed_" + str( seed) + ".tif " + pixType AllCmd.append(cmd) tmp = pathClassif.split("/") if pathClassif[-1] == "/": del tmp[-1] tmp[-1] = "cmd/fusion" pathToCmdFusion = "/".join(tmp) fu.writeCmds(pathToCmdFusion + "/fusion.txt", AllCmd) return AllCmd
def launchFit(sensorName, cmdPath, workingDirectory, outDirectory, outGridPath, outputGrid_tileNameField, outPixRes, outputProjection, interpolator, inputGrid_tileNameField, inputGRID, arboMask, masksInitValues, masksPatterns, arboRaster, rasterInitValue, rasterPatterns, TileFolder): createOutGrid(TileFolder, rasterPatterns[0], outputProjection, outputGrid_tileNameField, outGridPath) intersections = getIntersections(inputGRID, outGridPath, inputGrid_tileNameField, outputGrid_tileNameField) allCmd = [] for refTile, tiles in intersections: outFolder_tile = outDirectory + "/" + refTile if not os.path.exists(outFolder_tile): os.mkdir(outFolder_tile) minX_ref, maxX_ref, minY_ref, maxY_ref = getTileEnvelope( inputGRID, inputGrid_tileNameField, refTile) for currentTile in tiles: datas, dates = getAllDatasByDate(currentTile, rasterPatterns, masksPatterns, arboRaster, arboMask, TileFolder, rasterInitValue, masksInitValues) for currentDatas, currentDate in zip(datas, dates): outFolderDate = outFolder_tile + "/" + sensorName + "_" + refTile + "_" + currentDate + "_" + currentTile outFolderDateMask = outFolderDate + "/MASKS" if not os.path.exists(outFolderDate): os.mkdir(outFolderDate) os.mkdir(outFolderDateMask) for currentRaster in currentDatas: folder = outFolderDate print currentRaster if checkMaskFromRaster(currentRaster[0], masksPatterns): folder = outFolderDateMask outFolder = folder if workingDirectory: folder = workingDirectory + "/" outName = currentRaster[0].split("/")[-1].replace( currentTile, refTile).replace(".tif", "_" + currentTile + ".tif") out = folder + "/" + outName cmd = "gdalwarp -t_srs EPSG:" + outputProjection + " -wo INIT_DEST=" + currentRaster[ 1] + " -te " + str(minX_ref) + " " + str(minY_ref) + " " + str(maxX_ref) + " " + str( maxY_ref) + " -tr " + outPixRes + " -" + outPixRes + " -r " + interpolator + " " + \ currentRaster[0] + " " + out if not os.path.exists(outFolder + "/" + outName): allCmd.append(cmd) print cmd os.system(cmd) if workingDirectory: shutil.copy(out, outFolder + "/" + outName) os.remove(out) fu.writeCmds(cmdPath, allCmd)
def genCmdSplitShape(config): f = file(config) cfg = Config(f) maxArea = float(cfg.chain.mode_outside_RegionSplit) outputpath = cfg.chain.outputPath dataField = cfg.chain.dataField execMode = cfg.chain.executionMode allShape = fu.fileSearchRegEx(outputpath + "/dataRegion/*.shp") allArea = getAreaByRegion(allShape) workingDir = " --wd $TMPDIR " if execMode == "sequential": workingDir = " " print "all area [square meter]:" print allArea shapeToSplit = [] dic = {} # {'region':Nsplits,..} for region, area in allArea: fold = math.ceil(area / (maxArea * 1e6)) dic[region] = fold TooBigRegions = [region for region in dic if dic[region] > 1] print "Too big regions" print TooBigRegions for bigR in TooBigRegions: tmp = fu.fileSearchRegEx(outputpath + "/dataAppVal/*_region_" + bigR + "*.shp") for shapeTmp in tmp: shapeToSplit.append(shapeTmp) print shapeToSplit # write cmds AllCmd = [] for currentShape in shapeToSplit: currentRegion = currentShape.split('/')[-1].split("_")[2].split("f")[0] cmd = "python splitShape.py -config " + config + " -path.shape " + currentShape + " -Nsplit " + str( int(dic[currentRegion])) + " " + workingDir AllCmd.append(cmd) fu.writeCmds(outputpath + "/cmd/splitShape/splitShape.txt", AllCmd) return AllCmd
def launchClassification(model, pathConf, stat, pathToRT, pathToImg, pathToRegion, fieldRegion, N, pathToCmdClassif, pathOut, pathWd): f = file(pathConf) cfg = Config(f) classif = cfg.argTrain.classifier mode = cfg.chain.mode outputPath = cfg.chain.outputPath classifMode = cfg.argClassification.classifMode regionMode = cfg.chain.mode pixType = cfg.argClassification.pixType bindingPy = cfg.GlobChain.bindingPython Stack_ind = fu.getFeatStackName(pathConf) AllCmd = [] allTiles_s = cfg.chain.listTile allTiles = allTiles_s.split(" ") maskFiles = pathOut + "/MASK" if not os.path.exists(maskFiles): os.system("mkdir " + maskFiles) shpRName = pathToRegion.split("/")[-1].replace(".shp", "") AllModel = fu.FileSearch_AND(model, True, "model", ".txt") AllModel = [f for f in AllModel if os.path.splitext(f)[1] == '.txt'] for path in AllModel: model = path.split("/")[-1].split("_")[1] tiles = fu.getListTileFromModel( model, outputPath + "/config_model/configModel.cfg") model_Mask = model if re.search('model_.*f.*_', path.split("/")[-1]): model_Mask = path.split("/")[-1].split("_")[1].split("f")[0] seed = path.split("/")[-1].split("_")[-1].replace(".txt", "") tilesToEvaluate = tiles if ("fusion" in classifMode and regionMode != "outside") or (regionMode == "one_region"): tilesToEvaluate = allTiles # construction du string de sortie for tile in tilesToEvaluate: pathToFeat = pathToImg + "/" + tile + "/Final/" + Stack_ind if bindingPy == "True": pathToFeat = fu.FileSearch_AND( pathToImg + "/" + tile + "/tmp/", True, ".tif")[0] maskSHP = pathToRT + "/" + shpRName + "_region_" + model_Mask + "_" + tile + ".shp" maskTif = shpRName + "_region_" + model_Mask + "_" + tile + ".tif" CmdConfidenceMap = "" confidenceMap = "" if "fusion" in classifMode: if mode != "outside": tmp = pathOut.split("/") if pathOut[-1] == "/": del tmp[-1] tmp[-1] = "envelope" pathToEnvelope = "/".join(tmp) maskSHP = pathToEnvelope + "/" + tile + ".shp" confidenceMap = tile + "_model_" + model + "_confidence_seed_" + seed + ".tif" CmdConfidenceMap = " -confmap " + pathOut + "/" + confidenceMap if not os.path.exists(maskFiles + "/" + maskTif): pathToMaskCommun = pathToImg + "/" + tile + "/tmp/MaskCommunSL.shp" # cas cluster if pathWd != None: pathToMaskCommun = pathToImg + "/" + tile + "/MaskCommunSL.shp" maskFiles = pathWd nameOut = fu.ClipVectorData(maskSHP, pathToMaskCommun, maskFiles, maskTif.replace(".tif", "")) cmdRaster = "otbcli_Rasterization -in " + nameOut + " -mode attribute -mode.attribute.field " + fieldRegion + " -im " + pathToFeat + " -out " + maskFiles + "/" + maskTif if "fusion" in classifMode: cmdRaster = "otbcli_Rasterization -in " + nameOut + " -mode binary -mode.binary.foreground 1 -im " + pathToFeat + " -out " + maskFiles + "/" + maskTif print cmdRaster os.system(cmdRaster) if pathWd != None: os.system("cp " + pathWd + "/" + maskTif + " " + pathOut + "/MASK") out = pathOut + "/Classif_" + tile + "_model_" + model + "_seed_" + seed + ".tif" # hpc case if pathWd != None: out = "$TMPDIR/Classif_" + tile + "_model_" + model + "_seed_" + seed + ".tif" CmdConfidenceMap = " -confmap $TMPDIR/" + confidenceMap appli = "otbcli_ImageClassifier " pixType_cmd = pixType if bindingPy == "True": appli = "python bPy_ImageClassifier.py -conf " + pathConf + " " pixType_cmd = " -pixType " + pixType cmd = appli + " -in " + pathToFeat + " -model " + path + " -mask " + pathOut + "/MASK/" + maskTif + " -out " + out + " " + pixType_cmd + " -ram 128 " + CmdConfidenceMap # Ajout des stats lors de la phase de classification if ("svm" in classif): cmd = cmd + " -imstat " + stat + "/Model_" + str( model) + ".xml" AllCmd.append(cmd) fu.writeCmds(pathToCmdClassif + "/class.txt", AllCmd) return AllCmd
def launchTraining(pathShapes, pathConf, pathToTiles, dataField, stat, N, pathToCmdTrain, out, pathWd, pathlog): """ OUT : les commandes pour l'app """ cmd_out = [] f = file(pathConf) cfg = Config(f) classif = cfg.argTrain.classifier options = cfg.argTrain.options outputPath = cfg.chain.outputPath samplesMode = Config(file(pathConf)).argTrain.shapeMode dataField = Config(file(pathConf)).chain.dataField binding = Config(file(pathConf)).GlobChain.bindingPython posModel = -3 # model's position, if training shape is split by "_" Stack_ind = fu.getFeatStackName(pathConf) pathToModelConfig = outputPath + "/config_model/configModel.cfg" configModel = open(pathToModelConfig, "w") configModel.write("AllModel:\n[\n") configModel.close() for seed in range(N): pathAppVal = fu.FileSearch_AND(pathShapes, True, "seed" + str(seed), ".shp", "learn") sort = [(path.split("/")[-1].split("_")[posModel], path) for path in pathAppVal] sort = fu.sortByFirstElem(sort) # get tiles by model names = [] for r, paths in sort: tmp = "" for i in range(len(paths)): if i < len(paths) - 1: tmp = tmp + paths[i].split("/")[-1].split("_")[0] + "_" else: tmp = tmp + paths[i].split("/")[-1].split("_")[0] names.append(tmp) cpt = 0 for r, paths in sort: writeConfigName(r, names[cpt], pathToModelConfig) cpt += 1 if samplesMode == "points": pathAppVal = fu.FileSearch_AND(outputPath + "/learningSamples", True, "seed" + str(seed), ".sqlite", "learn") sort = [(path.split("/")[-1].split("_")[posModel], path) for path in pathAppVal] for r, paths in sort: print r if samplesMode != "points": cmd = buildTrainCmd_poly(r, paths, pathToTiles, Stack_ind, classif, options, dataField, out, seed, stat, pathlog) else: if binding == "True" and classif == "svm": outStats = outputPath + "/stats/Model_" + r + ".xml" if os.path.exists(outStats): os.remove(outStats) writeStatsFromSample(paths, outStats) cmd = buildTrainCmd_points(r, paths, classif, options, dataField, out, seed, stat, pathlog) cmd_out.append(cmd) configModel = open(pathToModelConfig, "a") configModel.write("\n]\n") configModel.close() fu.writeCmds(pathToCmdTrain + "/train.txt", cmd_out) return cmd_out
def genConfMatrix(pathClassif, pathValid, N, dataField, pathToCmdConfusion, pathConf, pathWd): AllCmd = [] pathTMP = pathClassif + "/TMP" f = file(pathConf) cfg = Config(f) pathTest = cfg.chain.outputPath workingDirectory = pathClassif + "/TMP" if pathWd: workingDirectory = os.getenv('TMPDIR').replace(":", "") # AllTiles = cfg.chain.listTile.split(" ") AllTiles = [] validationFiles = fu.FileSearch_AND(pathValid, True, "_val.shp") for valid in validationFiles: currentTile = valid.split("/")[-1].split("_")[0] try: ind = AllTiles.index(currentTile) except ValueError: AllTiles.append(currentTile) for seed in range(N): # recherche de tout les shapeFiles par seed, par tuiles pour les fusionner for tile in AllTiles: valTile = fu.FileSearch_AND(pathValid, True, tile, "_seed" + str(seed) + "_val.shp") fu.mergeVectors("ShapeValidation_" + tile + "_seed_" + str(seed), pathTMP, valTile) learnTile = fu.FileSearch_AND(pathValid, True, tile, "_seed" + str(seed) + "_learn.shp") fu.mergeVectors("ShapeLearning_" + tile + "_seed_" + str(seed), pathTMP, learnTile) pathDirectory = pathTMP if pathWd != None: pathDirectory = "$TMPDIR" cmd = 'otbcli_ComputeConfusionMatrix -in ' + pathClassif + '/Classif_Seed_' + str( seed ) + '.tif -out ' + pathDirectory + '/' + tile + '_seed_' + str( seed ) + '.csv -ref.vector.field ' + dataField + ' -ref vector -ref.vector.in ' + pathTMP + '/ShapeValidation_' + tile + '_seed_' + str( seed) + '.shp' AllCmd.append(cmd) classif = pathTMP + "/" + tile + "_seed_" + str(seed) + ".tif" diff = pathTMP + "/" + tile + "_seed_" + str(seed) + "_CompRef.tif" footprint = pathTest + "/final/Classif_Seed_0.tif" compareRef( pathTMP + '/ShapeValidation_' + tile + '_seed_' + str(seed) + '.shp', pathTMP + '/ShapeLearning_' + tile + '_seed_' + str(seed) + '.shp', classif, diff, footprint, workingDirectory, pathConf) fu.writeCmds(pathToCmdConfusion + "/confusion.txt", AllCmd) spatialRes = cfg.chain.spatialResolution for seed in range(N): AllDiff = fu.FileSearch_AND(pathTMP, True, "_seed_" + str(seed) + "_CompRef.tif") diff_seed = pathTest + "/final/diff_seed_" + str(seed) + ".tif" if pathWd: diff_seed = workingDirectory + "/diff_seed_" + str(seed) + ".tif" fu.assembleTile_Merge(AllDiff, spatialRes, diff_seed) if pathWd: shutil.copy(workingDirectory + "/diff_seed_" + str(seed) + ".tif", pathTest + "/final/diff_seed_" + str(seed) + ".tif") return (AllCmd)