def generateStatModel(pathShapes, pathToTiles, pathToStats, pathToCmdStats,
pathWd, pathConf):
AllCmd = []
modTiles = GM.getModel(pathShapes)
cfg = Config(pathConf)
Stack_ind = fu.getFeatStackName(pathConf)
classifier = Config(file(pathConf)).argTrain.classifier
for mod, Tiles in modTiles:
allpath = ""
for tile in Tiles:
pathToFeat = pathToTiles + "/" + tile + "/Final/" + Stack_ind
allpath = allpath + " " + pathToFeat + " "
# hpc case
#if pathWd != None:
# pathToStats = "$TMPDIR"
if classifier == "svm":
cmd = "otbcli_ComputeImagesStatistics -il " + allpath + " -out " + pathToStats + "/Model_" + str(
mod) + ".xml"
else:
cmd = "echo 'random forest does not need stats'"
AllCmd.append(cmd)
fu.writeCmds(pathToCmdStats + "/stats.txt", AllCmd)
return AllCmd
def fusion(pathClassif, pathConf, pathWd):
f = file(pathConf)
cfg = Config(f)
classifMode = cfg.argClassification.classifMode
N = int(cfg.chain.runs)
allTiles = cfg.chain.listTile.split(" ")
fusionOptions = cfg.argClassification.fusionOptions
mode = cfg.chain.mode
pixType = cfg.argClassification.pixType
if mode == "outside":
AllClassif = fu.fileSearchRegEx(pathClassif +
"/Classif_*_model_*f*_seed_*.tif")
allTiles = []
models = []
for classif in AllClassif:
mod = classif.split("/")[-1].split("_")[3].split("f")[0]
tile = classif.split("/")[-1].split("_")[1]
if not mod in models:
models.append(mod)
if not tile in allTiles:
allTiles.append(tile)
AllCmd = []
for seed in range(N):
for tile in allTiles:
directoryOut = pathClassif
if pathWd != None:
directoryOut = "$TMPDIR"
if mode != "outside":
classifPath = fu.FileSearch_AND(pathClassif, True,
"Classif_" + tile,
"seed_" + str(seed) + ".tif")
allPathFusion = " ".join(classifPath)
cmd = "otbcli_FusionOfClassifications -il " + allPathFusion + " " + fusionOptions + " -out " + directoryOut + "/" + tile + "_FUSION_seed_" + str(
seed) + ".tif"
AllCmd.append(cmd)
else:
for mod in models:
classifPath = fu.fileSearchRegEx(pathClassif +
"/Classif_" + tile +
"_model_" + mod +
"f*_seed_" + str(seed) +
".tif")
if len(classifPath) != 0:
allPathFusion = " ".join(classifPath)
cmd = "otbcli_FusionOfClassifications -il " + allPathFusion + " " + fusionOptions + " -out " + directoryOut + "/" + tile + "_FUSION_model_" + mod + "_seed_" + str(
seed) + ".tif " + pixType
AllCmd.append(cmd)
tmp = pathClassif.split("/")
if pathClassif[-1] == "/":
del tmp[-1]
tmp[-1] = "cmd/fusion"
pathToCmdFusion = "/".join(tmp)
fu.writeCmds(pathToCmdFusion + "/fusion.txt", AllCmd)
return AllCmd
def launchFit(sensorName, cmdPath, workingDirectory, outDirectory, outGridPath,
outputGrid_tileNameField, outPixRes, outputProjection,
interpolator, inputGrid_tileNameField, inputGRID, arboMask,
masksInitValues, masksPatterns, arboRaster, rasterInitValue,
rasterPatterns, TileFolder):
createOutGrid(TileFolder, rasterPatterns[0], outputProjection,
outputGrid_tileNameField, outGridPath)
intersections = getIntersections(inputGRID, outGridPath,
inputGrid_tileNameField,
outputGrid_tileNameField)
allCmd = []
for refTile, tiles in intersections:
outFolder_tile = outDirectory + "/" + refTile
if not os.path.exists(outFolder_tile):
os.mkdir(outFolder_tile)
minX_ref, maxX_ref, minY_ref, maxY_ref = getTileEnvelope(
inputGRID, inputGrid_tileNameField, refTile)
for currentTile in tiles:
datas, dates = getAllDatasByDate(currentTile, rasterPatterns,
masksPatterns, arboRaster,
arboMask, TileFolder,
rasterInitValue, masksInitValues)
for currentDatas, currentDate in zip(datas, dates):
outFolderDate = outFolder_tile + "/" + sensorName + "_" + refTile + "_" + currentDate + "_" + currentTile
outFolderDateMask = outFolderDate + "/MASKS"
if not os.path.exists(outFolderDate):
os.mkdir(outFolderDate)
os.mkdir(outFolderDateMask)
for currentRaster in currentDatas:
folder = outFolderDate
print currentRaster
if checkMaskFromRaster(currentRaster[0], masksPatterns):
folder = outFolderDateMask
outFolder = folder
if workingDirectory: folder = workingDirectory + "/"
outName = currentRaster[0].split("/")[-1].replace(
currentTile,
refTile).replace(".tif", "_" + currentTile + ".tif")
out = folder + "/" + outName
cmd = "gdalwarp -t_srs EPSG:" + outputProjection + " -wo INIT_DEST=" + currentRaster[
1] + " -te " + str(minX_ref) + " " + str(minY_ref) + " " + str(maxX_ref) + " " + str(
maxY_ref) + " -tr " + outPixRes + " -" + outPixRes + " -r " + interpolator + " " + \
currentRaster[0] + " " + out
if not os.path.exists(outFolder + "/" + outName):
allCmd.append(cmd)
print cmd
os.system(cmd)
if workingDirectory:
shutil.copy(out, outFolder + "/" + outName)
os.remove(out)
fu.writeCmds(cmdPath, allCmd)
def genCmdSplitShape(config):
f = file(config)
cfg = Config(f)
maxArea = float(cfg.chain.mode_outside_RegionSplit)
outputpath = cfg.chain.outputPath
dataField = cfg.chain.dataField
execMode = cfg.chain.executionMode
allShape = fu.fileSearchRegEx(outputpath + "/dataRegion/*.shp")
allArea = getAreaByRegion(allShape)
workingDir = " --wd $TMPDIR "
if execMode == "sequential":
workingDir = " "
print "all area [square meter]:"
print allArea
shapeToSplit = []
dic = {} # {'region':Nsplits,..}
for region, area in allArea:
fold = math.ceil(area / (maxArea * 1e6))
dic[region] = fold
TooBigRegions = [region for region in dic if dic[region] > 1]
print "Too big regions"
print TooBigRegions
for bigR in TooBigRegions:
tmp = fu.fileSearchRegEx(outputpath + "/dataAppVal/*_region_" + bigR + "*.shp")
for shapeTmp in tmp:
shapeToSplit.append(shapeTmp)
print shapeToSplit
# write cmds
AllCmd = []
for currentShape in shapeToSplit:
currentRegion = currentShape.split('/')[-1].split("_")[2].split("f")[0]
cmd = "python splitShape.py -config " + config + " -path.shape " + currentShape + " -Nsplit " + str(
int(dic[currentRegion])) + " " + workingDir
AllCmd.append(cmd)
fu.writeCmds(outputpath + "/cmd/splitShape/splitShape.txt", AllCmd)
return AllCmd
def launchClassification(model, pathConf, stat, pathToRT, pathToImg,
pathToRegion, fieldRegion, N, pathToCmdClassif,
pathOut, pathWd):
f = file(pathConf)
cfg = Config(f)
classif = cfg.argTrain.classifier
mode = cfg.chain.mode
outputPath = cfg.chain.outputPath
classifMode = cfg.argClassification.classifMode
regionMode = cfg.chain.mode
pixType = cfg.argClassification.pixType
bindingPy = cfg.GlobChain.bindingPython
Stack_ind = fu.getFeatStackName(pathConf)
AllCmd = []
allTiles_s = cfg.chain.listTile
allTiles = allTiles_s.split(" ")
maskFiles = pathOut + "/MASK"
if not os.path.exists(maskFiles):
os.system("mkdir " + maskFiles)
shpRName = pathToRegion.split("/")[-1].replace(".shp", "")
AllModel = fu.FileSearch_AND(model, True, "model", ".txt")
AllModel = [f for f in AllModel if os.path.splitext(f)[1] == '.txt']
for path in AllModel:
model = path.split("/")[-1].split("_")[1]
tiles = fu.getListTileFromModel(
model, outputPath + "/config_model/configModel.cfg")
model_Mask = model
if re.search('model_.*f.*_', path.split("/")[-1]):
model_Mask = path.split("/")[-1].split("_")[1].split("f")[0]
seed = path.split("/")[-1].split("_")[-1].replace(".txt", "")
tilesToEvaluate = tiles
if ("fusion" in classifMode
and regionMode != "outside") or (regionMode == "one_region"):
tilesToEvaluate = allTiles
# construction du string de sortie
for tile in tilesToEvaluate:
pathToFeat = pathToImg + "/" + tile + "/Final/" + Stack_ind
if bindingPy == "True":
pathToFeat = fu.FileSearch_AND(
pathToImg + "/" + tile + "/tmp/", True, ".tif")[0]
maskSHP = pathToRT + "/" + shpRName + "_region_" + model_Mask + "_" + tile + ".shp"
maskTif = shpRName + "_region_" + model_Mask + "_" + tile + ".tif"
CmdConfidenceMap = ""
confidenceMap = ""
if "fusion" in classifMode:
if mode != "outside":
tmp = pathOut.split("/")
if pathOut[-1] == "/":
del tmp[-1]
tmp[-1] = "envelope"
pathToEnvelope = "/".join(tmp)
maskSHP = pathToEnvelope + "/" + tile + ".shp"
confidenceMap = tile + "_model_" + model + "_confidence_seed_" + seed + ".tif"
CmdConfidenceMap = " -confmap " + pathOut + "/" + confidenceMap
if not os.path.exists(maskFiles + "/" + maskTif):
pathToMaskCommun = pathToImg + "/" + tile + "/tmp/MaskCommunSL.shp"
# cas cluster
if pathWd != None:
pathToMaskCommun = pathToImg + "/" + tile + "/MaskCommunSL.shp"
maskFiles = pathWd
nameOut = fu.ClipVectorData(maskSHP, pathToMaskCommun,
maskFiles,
maskTif.replace(".tif", ""))
cmdRaster = "otbcli_Rasterization -in " + nameOut + " -mode attribute -mode.attribute.field " + fieldRegion + " -im " + pathToFeat + " -out " + maskFiles + "/" + maskTif
if "fusion" in classifMode:
cmdRaster = "otbcli_Rasterization -in " + nameOut + " -mode binary -mode.binary.foreground 1 -im " + pathToFeat + " -out " + maskFiles + "/" + maskTif
print cmdRaster
os.system(cmdRaster)
if pathWd != None:
os.system("cp " + pathWd + "/" + maskTif + " " + pathOut +
"/MASK")
out = pathOut + "/Classif_" + tile + "_model_" + model + "_seed_" + seed + ".tif"
# hpc case
if pathWd != None:
out = "$TMPDIR/Classif_" + tile + "_model_" + model + "_seed_" + seed + ".tif"
CmdConfidenceMap = " -confmap $TMPDIR/" + confidenceMap
appli = "otbcli_ImageClassifier "
pixType_cmd = pixType
if bindingPy == "True":
appli = "python bPy_ImageClassifier.py -conf " + pathConf + " "
pixType_cmd = " -pixType " + pixType
cmd = appli + " -in " + pathToFeat + " -model " + path + " -mask " + pathOut + "/MASK/" + maskTif + " -out " + out + " " + pixType_cmd + " -ram 128 " + CmdConfidenceMap
# Ajout des stats lors de la phase de classification
if ("svm" in classif):
cmd = cmd + " -imstat " + stat + "/Model_" + str(
model) + ".xml"
AllCmd.append(cmd)
fu.writeCmds(pathToCmdClassif + "/class.txt", AllCmd)
return AllCmd
def launchTraining(pathShapes, pathConf, pathToTiles, dataField, stat, N,
pathToCmdTrain, out, pathWd, pathlog):
"""
OUT : les commandes pour l'app
"""
cmd_out = []
f = file(pathConf)
cfg = Config(f)
classif = cfg.argTrain.classifier
options = cfg.argTrain.options
outputPath = cfg.chain.outputPath
samplesMode = Config(file(pathConf)).argTrain.shapeMode
dataField = Config(file(pathConf)).chain.dataField
binding = Config(file(pathConf)).GlobChain.bindingPython
posModel = -3 # model's position, if training shape is split by "_"
Stack_ind = fu.getFeatStackName(pathConf)
pathToModelConfig = outputPath + "/config_model/configModel.cfg"
configModel = open(pathToModelConfig, "w")
configModel.write("AllModel:\n[\n")
configModel.close()
for seed in range(N):
pathAppVal = fu.FileSearch_AND(pathShapes, True, "seed" + str(seed),
".shp", "learn")
sort = [(path.split("/")[-1].split("_")[posModel], path)
for path in pathAppVal]
sort = fu.sortByFirstElem(sort)
# get tiles by model
names = []
for r, paths in sort:
tmp = ""
for i in range(len(paths)):
if i < len(paths) - 1:
tmp = tmp + paths[i].split("/")[-1].split("_")[0] + "_"
else:
tmp = tmp + paths[i].split("/")[-1].split("_")[0]
names.append(tmp)
cpt = 0
for r, paths in sort:
writeConfigName(r, names[cpt], pathToModelConfig)
cpt += 1
if samplesMode == "points":
pathAppVal = fu.FileSearch_AND(outputPath + "/learningSamples",
True, "seed" + str(seed), ".sqlite",
"learn")
sort = [(path.split("/")[-1].split("_")[posModel], path)
for path in pathAppVal]
for r, paths in sort:
print r
if samplesMode != "points":
cmd = buildTrainCmd_poly(r, paths, pathToTiles, Stack_ind,
classif, options, dataField, out,
seed, stat, pathlog)
else:
if binding == "True" and classif == "svm":
outStats = outputPath + "/stats/Model_" + r + ".xml"
if os.path.exists(outStats):
os.remove(outStats)
writeStatsFromSample(paths, outStats)
cmd = buildTrainCmd_points(r, paths, classif, options,
dataField, out, seed, stat, pathlog)
cmd_out.append(cmd)
configModel = open(pathToModelConfig, "a")
configModel.write("\n]\n")
configModel.close()
fu.writeCmds(pathToCmdTrain + "/train.txt", cmd_out)
return cmd_out
def genConfMatrix(pathClassif, pathValid, N, dataField, pathToCmdConfusion,
pathConf, pathWd):
AllCmd = []
pathTMP = pathClassif + "/TMP"
f = file(pathConf)
cfg = Config(f)
pathTest = cfg.chain.outputPath
workingDirectory = pathClassif + "/TMP"
if pathWd:
workingDirectory = os.getenv('TMPDIR').replace(":", "")
# AllTiles = cfg.chain.listTile.split(" ")
AllTiles = []
validationFiles = fu.FileSearch_AND(pathValid, True, "_val.shp")
for valid in validationFiles:
currentTile = valid.split("/")[-1].split("_")[0]
try:
ind = AllTiles.index(currentTile)
except ValueError:
AllTiles.append(currentTile)
for seed in range(N):
# recherche de tout les shapeFiles par seed, par tuiles pour les fusionner
for tile in AllTiles:
valTile = fu.FileSearch_AND(pathValid, True, tile,
"_seed" + str(seed) + "_val.shp")
fu.mergeVectors("ShapeValidation_" + tile + "_seed_" + str(seed),
pathTMP, valTile)
learnTile = fu.FileSearch_AND(pathValid, True, tile,
"_seed" + str(seed) + "_learn.shp")
fu.mergeVectors("ShapeLearning_" + tile + "_seed_" + str(seed),
pathTMP, learnTile)
pathDirectory = pathTMP
if pathWd != None: pathDirectory = "$TMPDIR"
cmd = 'otbcli_ComputeConfusionMatrix -in ' + pathClassif + '/Classif_Seed_' + str(
seed
) + '.tif -out ' + pathDirectory + '/' + tile + '_seed_' + str(
seed
) + '.csv -ref.vector.field ' + dataField + ' -ref vector -ref.vector.in ' + pathTMP + '/ShapeValidation_' + tile + '_seed_' + str(
seed) + '.shp'
AllCmd.append(cmd)
classif = pathTMP + "/" + tile + "_seed_" + str(seed) + ".tif"
diff = pathTMP + "/" + tile + "_seed_" + str(seed) + "_CompRef.tif"
footprint = pathTest + "/final/Classif_Seed_0.tif"
compareRef(
pathTMP + '/ShapeValidation_' + tile + '_seed_' + str(seed) +
'.shp', pathTMP + '/ShapeLearning_' + tile + '_seed_' +
str(seed) + '.shp', classif, diff, footprint, workingDirectory,
pathConf)
fu.writeCmds(pathToCmdConfusion + "/confusion.txt", AllCmd)
spatialRes = cfg.chain.spatialResolution
for seed in range(N):
AllDiff = fu.FileSearch_AND(pathTMP, True,
"_seed_" + str(seed) + "_CompRef.tif")
diff_seed = pathTest + "/final/diff_seed_" + str(seed) + ".tif"
if pathWd:
diff_seed = workingDirectory + "/diff_seed_" + str(seed) + ".tif"
fu.assembleTile_Merge(AllDiff, spatialRes, diff_seed)
if pathWd:
shutil.copy(workingDirectory + "/diff_seed_" + str(seed) + ".tif",
pathTest + "/final/diff_seed_" + str(seed) + ".tif")
return (AllCmd)