def rastToVectRecode(path, classif, vector, outputName, ram = "10000", dtype = "uint8", valvect = 255, valrastout = 255):
"""
Convert vector in raster file and change background value
Parameters
----------
path : string
working directory
classif : string
path to landcover classification
vector : string
vector file to rasterize
outputName : string
output filename and path
ram : string
ram for OTB applications
dtype : string
pixel type of the output raster
valvect : integer
value of vector to search
valrastout : integer
value to use to recode
"""
# Empty raster
bmapp = OtbAppBank.CreateBandMathApplication({"il": classif,
"exp": "im1b1*0",
"ram": str(1 * float(ram)),
"pixType": dtype,
"out": os.path.join(path, 'temp.tif')})
#bandMathAppli.ExecuteAndWriteOutput()
p = mp.Process(target=executeApp, args=[bmapp])
p.start()
p.join()
# Burn
tifMasqueMerRecode = os.path.join(path, 'masque_mer_recode.tif')
rastApp = OtbAppBank.CreateRasterizationApplication({"in" : vector,
"im" : os.path.join(path, 'temp.tif'),
"background": 1,
"out": tifMasqueMerRecode})
#bandMathAppli.ExecuteAndWriteOutput()
p = mp.Process(target=executeApp, args=[rastApp])
p.start()
p.join()
# Differenciate inland water and sea water
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": [classif, tifMasqueMerRecode],
"exp": "(im2b1=={})?im1b1:{}".format(valvect, valrastout),
"ram": str(1 * float(ram)),
"pixType": dtype,
"out": outputName})
#bandMathAppli.ExecuteAndWriteOutput()
p = mp.Process(target=executeApp, args=[bandMathAppli])
p.start()
p.join()
os.remove(tifMasqueMerRecode)
def maskSampleSelection(path, raster, maskmer, ram):
tifMasqueMer = os.path.join(path, 'masque_mer.tif')
bmapp = OtbAppBank.CreateBandMathApplication({
"il": raster,
"exp": "im1b1*0",
"ram": ram,
"pixType": 'uint8',
"out": tifMasqueMer
})
bmapp.ExecuteAndWriteOutput()
maskmerbuff = os.path.join(
path,
os.path.splitext(os.path.basename(maskmer))[0] + '_buff.shp')
BufferOgr.bufferPoly(maskmer, maskmerbuff, 500)
tifMasqueMerRecode = os.path.join(path, 'masque_mer_recode.tif')
rastApp = OtbAppBank.CreateRasterizationApplication(
maskmerbuff, tifMasqueMer, 1, tifMasqueMerRecode)
rastApp.Execute()
#command = "gdal_rasterize -burn 1 %s %s"%(maskmerbuff, tifMasqueMer)
#os.system(command)
out = os.path.join(path, 'mask.tif')
bmapp = OtbAppBank.CreateBandMathApplication({
"il": [raster, rastApp],
"exp": "((im1b1==0) || (im1b1==51)) && (im2b1==0)?0:1",
"ram": ram,
"pixType": 'uint8',
"out": out
})
bmapp.ExecuteAndWriteOutput()
return out
def generateBorderMask(data_img, out_mask, RAMPerProcess=4000):
"""
"""
threshold = 0.0011
mask = OtbAppBank.CreateBandMathApplication({
"il":
data_img,
"exp":
"im1b1<{}?1:0".format(threshold),
"ram":
str(RAMPerProcess),
"pixType":
'uint8'
})
mask.Execute()
borderMask = OtbAppBank.CreateBinaryMorphologicalOperation({
"in":
mask,
"out":
out_mask,
"ram":
str(RAMPerProcess),
"pixType":
"uint8",
"filter":
"opening",
"ballxradius":
5,
"ballyradius":
5
})
dep = mask
return borderMask, dep
def RastersToSqlitePoint(path,
vecteur,
field,
outname,
ram,
rtype,
rasters,
maskmer=None,
split=None):
timeinit = time.time()
# Rasters concatenation
if len(rasters) > 1:
concatApp = OtbAppBank.CreateConcatenateImagesApplication({
"il":
rasters,
"ram":
ram,
"pixType":
rtype
})
concatApp.Execute()
classif = OtbAppBank.CreateBandMathApplication({
"il": rasters[0],
"exp": "im1b1",
"ram": ram,
"pixType": rtype
})
classif.Execute()
else:
concatApp = OtbAppBank.CreateBandMathApplication({
"il": rasters,
"exp": "im1b1",
"ram": ram,
"pixType": rtype
})
concatApp.Execute()
timeconcat = time.time()
print " ".join([
" : ".join(["Raster concatenation",
str(timeconcat - timeinit)]), "seconds"
])
# Stats and sample selection
if len(rasters) == 1:
classif = concatApp
outsqlite = sampleSelection(path, classif, vecteur, field, ram, split,
maskmer)
# Stats extraction
outtmp = os.path.join(path, os.path.basename(outname))
sampleExtraction(concatApp, outsqlite, field, outtmp, split, ram)
shutil.copyfile(outtmp, outname)
def sampleSelection(path,
raster,
vecteur,
field,
ram='128',
split=None,
mask=None):
timeinit = time.time()
# polygon class stats (stats.xml)
outxml = os.path.join(path, 'stats' + str(split) + '.xml')
otbParams = {
'in': raster,
'vec': vecteur,
'field': field,
'out': outxml,
'ram': ram
}
statsApp = OtbAppBank.CreatePolygonClassStatisticsApplication(otbParams)
statsApp.ExecuteAndWriteOutput()
shutil.copy(os.path.join(path, 'stats.xml'),
'/work/OT/theia/oso/vincent/vectorisation/')
timestats = time.time()
print " ".join([
" : ".join(["Stats calculation",
str(timestats - timeinit)]), "seconds"
])
if mask is not None:
mask = maskSampleSelection(path, raster, mask, ram)
else:
mask = None
# Sample selection
outsqlite = os.path.join(path, 'sample_selection' + str(split) + '.sqlite')
if mask is None:
otbParams = {'in':raster, 'vec':vecteur, 'field':field, 'instats': outxml, \
'out':outsqlite, 'ram':ram, 'strategy':'all', 'sampler':'random'}
else:
otbParams = {'in':raster, 'vec':vecteur, 'field':field, 'instats': outxml, \
'out':outsqlite, 'mask':mask, 'ram':ram, 'strategy':'all', 'sampler':'random'}
sampleApp = OtbAppBank.CreateSampleSelectionApplication(otbParams)
sampleApp.ExecuteAndWriteOutput()
shutil.copy(os.path.join(path, outsqlite),
'/work/OT/theia/oso/vincent/vectorisation/')
timesample = time.time()
print " ".join([
" : ".join(["Sample selection",
str(timesample - timestats)]), "seconds"
])
return outsqlite
def generateBorderMask(self, AllOrtho):
print("Generate Mask ...")
masks = []
for currentOrtho, _ in AllOrtho:
outputParameter = OtbAppBank.getInputParameterOutput(currentOrtho)
if "vv" not in currentOrtho.GetParameterValue(outputParameter):
continue
workingDirectory = os.path.split(
currentOrtho.GetParameterValue(outputParameter))[0]
nameBorderMask = os.path.split(
currentOrtho.GetParameterValue(outputParameter))[1].replace(
".tif", "_BorderMask.tif")
nameBorderMaskTMP = os.path.split(
currentOrtho.GetParameterValue(outputParameter))[1].replace(
".tif", "_BorderMask_TMP.tif")
bandMathMask = os.path.join(workingDirectory, nameBorderMaskTMP)
currentOrtho_out = currentOrtho
if self.wMode: currentOrtho_out.GetParameterValue(outputParameter)
maskBM = OtbAppBank.CreateBandMathApplication({
"il":
currentOrtho_out,
"exp":
"im1b1<0.0011?1:0",
"ram":
str(self.RAMPerProcess),
"pixType":
'uint8',
"out":
bandMathMask
})
if self.wMode: maskBM.ExecuteAndWriteOutput()
else: maskBM.Execute()
borderMaskOut = os.path.join(workingDirectory, nameBorderMask)
maskBM_out = maskBM
if self.wMode: maskBM_out.GetParameterValue("out")
borderMask = OtbAppBank.CreateBinaryMorphologicalOperation({
"in":
maskBM,
"out":
borderMaskOut,
"ram":
str(self.RAMPerProcess),
"pixType":
"uint8",
"filter":
"opening",
"ballxradius":
5,
"ballyradius":
5
})
masks.append((borderMask, maskBM))
return masks
def doCalibrationCmd(self, rawRaster):
"""
OUT :
allCmdOrho [list of otb application list to Execute or ExecuteAndWriteOutput]
allCmdCalib [all dependence to run ortho]
"""
allCmdCalib = []
allCmdOrtho = []
for i in range(len(rawRaster)):
for image in rawRaster[i].GetImageList():
calibrate = image.replace(".tiff", "_calibrate.tiff")
image_OK = image.replace(".tiff", "_OrthoReady.tiff")
if os.path.exists(image_OK) == True:
continue
calib = OtbAppBank.CreateSarCalibration({
"in":
image,
"out":
calibrate,
"lut":
"gamma",
"ram":
str(self.RAMPerProcess)
})
if self.wMode: calib.ExecuteAndWriteOutput()
else: calib.Execute()
allCmdCalib.append(calib)
calib_out = calib
if self.wMode: calib_out = calib.GetParameterValue("out")
expression = 'im1b1<' + str(self.borderThreshold) + '?' + str(
self.borderThreshold) + ':im1b1 '
orthoRdy = OtbAppBank.CreateBandMathApplication({
"il":
calib_out,
"exp":
expression,
"ram":
str(self.RAMPerProcess),
"pixType":
"float",
"out":
image_OK
})
allCmdOrtho.append(orthoRdy)
return allCmdOrtho, allCmdCalib
def writeOutputRaster(OTB_App,
overwrite=True,
workingDirectory=None,
logger=logger):
"""
"""
import shutil
from Common import OtbAppBank as otbApp
out_param = otbApp.getInputParameterOutput(OTB_App)
out_raster = OTB_App.GetParameterValue(out_param)
launch_write = True
if os.path.exists(out_raster.split("?")[0]) and not overwrite:
launch_write = False
if workingDirectory is None and launch_write:
OTB_App.ExecuteAndWriteOutput()
elif launch_write:
out_raster_dir, out_raster_name = os.path.split(out_raster)
out_workingDir = os.path.join(workingDirectory, out_raster_name)
out_workingDir = out_workingDir.split("?")[0]
OTB_App.SetParameterString(out_param, out_workingDir)
OTB_App.ExecuteAndWriteOutput()
shutil.copy(out_workingDir, out_raster.split("?")[0])
if os.path.exists(out_workingDir.replace(".tif", ".geom")):
shutil.copy(out_workingDir.replace(".tif", ".geom"),
out_raster.replace(".tif", ".geom").split("?")[0])
if not launch_write:
logger.info(
"{} already exists and will not be overwrited".format(out_raster))
OTB_App = None
return out_raster
def findTilesToConcatenate(applicationList):
"""
OUT:
listOfList:
Example [[r1,r2],[r3,r4],... mean
r1 and r2 must be concatenates together
same for r3,r4
"""
concatenate = []
names = [(currentName.split("_")[-1].split("t")[0], currentName)
for currentName in OtbAppBank.unPackFirst(applicationList)
]
names = sortByFirstElem(names)
toConcat = [
rasterList for currentDate, rasterList in names
if len(rasterList) > 2
]
for dateToConcat in toConcat:
tmp = [(currentRaster.split("_")[2], currentRaster)
for currentRaster in dateToConcat]
tmp = sortByFirstElem(tmp)[::-1] #VV first then VH
for pol, rasters in tmp:
concatenate.append(rasters)
Filter = []
for ToConcat in concatenate:
sat = [CToConcat.split("_")[0] for CToConcat in ToConcat]
if not sat.count(sat[0]) == len(sat):
continue
Filter.append(ToConcat)
return Filter
def getDatesInOtbOutputName(otbObj):
if isinstance(otbObj,str):
return int(otbObj.split("/")[-1].split("_")[4].split("t")[0])
elif type(otbObj)==otb.Application:
outputParameter = OtbAppBank.getInputParameterOutput(otbObj)
return int(otbObj.GetParameterValue(outputParameter).split("/")[-1].split("_")[4].split("t")[0])
def SAR_floatToInt(filterApplication,
nb_bands,
RAMPerProcess,
outputFormat="uint16",
db_min=-25,
db_max=3):
"""transform float SAR values to integer
"""
import math
from Common import OtbAppBank as otbApp
min_val = str(10.0**(db_min / 10.0))
max_val = str(10.0**(db_max / 10.0))
min_val_scale = 0
max_val_scale = "((2^16)-1)"
if outputFormat == "uint8":
max_val_scale = "((2^8)-1)"
#build expression
to_db_expression = "10*log10(im1bX)"
scale_expression = "(({}-{})/({}-{}))*({})+({}-(({}-{})*{})/({}-{}))".format(
max_val_scale, min_val_scale, db_max, db_min, to_db_expression,
min_val_scale, max_val_scale, min_val_scale, db_min, db_max, db_min)
scale_max_val = (2**16) - 1
scale_min_val = 0
threshold_expression = "{0}>{1}?{3}:{0}<{2}?{4}:{5}".format(
to_db_expression, db_max, db_min, scale_max_val, scale_min_val,
scale_expression)
expression = ";".join([
threshold_expression.replace("X", str(i + 1)) for i in range(nb_bands)
])
outputPath = filterApplication.GetParameterValue(
otbApp.getInputParameterOutput(filterApplication))
convert = OtbAppBank.CreateBandMathXApplication({
"il": filterApplication,
"out": outputPath,
"exp": expression,
"ram": str(RAMPerProcess),
"pixType": outputFormat
})
return convert
def maskOTBBandMathOutput(path, raster, exp, ram, output, dstnodata=0):
outbm = os.path.join(path, "mask.tif")
bandMathAppli = OtbAppBank.CreateBandMathApplication({
"il": raster,
"exp": exp,
"ram": str(ram),
"pixType": "uint8",
"out": outbm
})
p = mp.Process(target=executeApp, args=[bandMathAppli])
p.start()
p.join()
gdal.Warp(output, outbm, dstNodata=dstnodata, multithread=True, format="GTiff", \
warpOptions=[["NUM_THREADS=ALL_CPUS"],["OVERWRITE=TRUE"]])
os.remove(outbm)
return output
def sampleExtraction(raster, sample, field, outname, split, ram='128'):
timesample = time.time()
# Sample extraction
outfile = os.path.splitext(str(outname))[0] + split + os.path.splitext(
str(outname))[1]
otbParams = {
'in': raster,
'vec': sample,
'field': field.lower(),
'out': outfile,
'ram': ram
}
extractApp = OtbAppBank.CreateSampleExtractionApplication(otbParams)
extractApp.ExecuteAndWriteOutput()
timeextract = time.time()
print " ".join([
" : ".join(["Sample extraction",
str(timeextract - timesample)]), "seconds"
])
def extractStats(vectorIn, pathConf, wD=None):
dataField = Config(open(pathConf)).chain.dataField
iota2Folder = Config(open(pathConf)).chain.outputPath
tileToCompute = vectorIn.split("/")[-1].split("_")[0]
modelToCompute = vectorIn.split("/")[-1].split("_")[2].split("f")[0]
seed = vectorIn.split("/")[-1].split("_")[3].replace("seed", "")
workingDirectory = iota2Folder + "/final/TMP"
shapeMode = vectorIn.split("/")[-1].split("_")[-1].split(".")[
0] #'learn' or 'val'
if wD:
workingDirectory = wD
try:
refImg = fut.FileSearch_AND(iota2Folder + "/final/TMP", True,
tileToCompute, ".tif")[0]
except:
raise Exception("reference image can not be found in " + iota2Folder +
"/final/TMP")
statsFile = workingDirectory + "/" + tileToCompute + "_stats_model_" + modelToCompute + ".xml"
stats = otbApp.CreatePolygonClassStatisticsApplication({"in":refImg, "vec":vectorIn,\
"out":statsFile, "field":dataField})
stats.ExecuteAndWriteOutput()
selVector = workingDirectory + "/" + tileToCompute + "_selection_model_" + modelToCompute + ".sqlite"
sampleS = otbApp.CreateSampleSelectionApplication({"in":refImg, "vec":vectorIn, "out":selVector,\
"instats":statsFile, "strategy":"all",\
"field":dataField})
sampleS.ExecuteAndWriteOutput()
classificationRaster = fut.FileSearch_AND(
iota2Folder + "/final/TMP", True,
tileToCompute + "_seed_" + seed + ".tif")[0]
validity = fut.FileSearch_AND(iota2Folder + "/final/TMP", True,
tileToCompute + "_Cloud.tif")[0]
confiance = fut.FileSearch_AND(
iota2Folder + "/final/TMP", True,
tileToCompute + "_GlobalConfidence_seed_" + seed + ".tif")[0]
stack = [classificationRaster, validity, confiance]
dataStack = otbApp.CreateConcatenateImagesApplication({
"il": stack,
"ram": '1000',
"pixType": "uint8",
"out": ""
})
dataStack.Execute()
outSampleExtraction = workingDirectory + "/" + tileToCompute + "_extraction_model_" + modelToCompute + "_" + shapeMode + ".sqlite"
extraction = otbApp.CreateSampleExtractionApplication({"in":dataStack, "vec":selVector,\
"field":dataField, " out":outSampleExtraction,\
"outfield":"list",\
"outfield.list.names":["predictedClass", "validity", "confidence"]})
extraction.ExecuteAndWriteOutput()
conn = lite.connect(outSampleExtraction)
cursor = conn.cursor()
SQL = "alter table output add column TILE TEXT"
cursor.execute(SQL)
SQL = "update output set TILE='" + tileToCompute + "'"
cursor.execute(SQL)
SQL = "alter table output add column MODEL TEXT"
cursor.execute(SQL)
SQL = "update output set MODEL='" + modelToCompute + "'"
cursor.execute(SQL)
conn.commit()
os.remove(statsFile)
os.remove(selVector)
if wD:
shutil.copy(outSampleExtraction, iota2Folder + "/final/TMP")
def LaunchSARreprojection(rasterList,
refRaster=None,
tileName=None,
SRTM=None,
geoid=None,
output_directory=None,
RAMPerProcess=None,
workingDirectory=None):
"""must be use with multiprocessing.Pool
"""
def writeOutputRaster_2(OTB_App,
overwrite=True,
workingDirectory=None,
dep=None,
logger=logger):
"""
"""
import shutil
from Common import OtbAppBank as otbApp
out_param = otbApp.getInputParameterOutput(OTB_App)
out_raster = OTB_App.GetParameterValue(out_param)
launch_write = True
if os.path.exists(out_raster.split("?")[0]) and not overwrite:
launch_write = False
if workingDirectory is None and launch_write:
OTB_App.ExecuteAndWriteOutput()
elif launch_write:
out_raster_dir, out_raster_name = os.path.split(out_raster)
out_workingDir = os.path.join(workingDirectory, out_raster_name)
out_workingDir = out_workingDir.split("?")[0]
OTB_App.SetParameterString(out_param, out_workingDir)
OTB_App.ExecuteAndWriteOutput()
shutil.copy(out_workingDir, out_raster.split("?")[0])
if os.path.exists(out_workingDir.replace(".tif", ".geom")):
shutil.copy(out_workingDir.replace(".tif", ".geom"),
out_raster.replace(".tif", ".geom").split("?")[0])
if not launch_write:
logger.info("{} already exists and will not be overwrited".format(
out_raster))
OTB_App = None
return out_raster
date_position = 4
all_superI_vv = []
all_superI_vh = []
all_acquisition_date_vv = []
all_acquisition_date_vh = []
dates = []
all_dep = []
for date_to_Concatenate in rasterList:
#Calibration + Superimpose
vv, vh = date_to_Concatenate.GetImageList()
SAR_directory, SAR_name = os.path.split(vv)
currentPlatform = getPlatformFromS1Raster(vv)
manifest = date_to_Concatenate.getManifest()
currentOrbitDirection = getOrbitDirection(manifest)
acquisition_date = SAR_name.split("-")[date_position]
dates.append(acquisition_date)
calib_vv = OtbAppBank.CreateSarCalibration({
"in": vv,
"lut": "gamma",
"ram": str(RAMPerProcess)
})
calib_vh = OtbAppBank.CreateSarCalibration({
"in": vh,
"lut": "gamma",
"ram": str(RAMPerProcess)
})
calib_vv.Execute()
calib_vh.Execute()
all_dep.append(calib_vv)
all_dep.append(calib_vh)
orthoImageName_vv = "{}_{}_{}_{}_{}".format(currentPlatform, tileName,
"vv",
currentOrbitDirection,
acquisition_date)
super_vv, super_vv_dep = OtbAppBank.CreateSuperimposeApplication({
"inr":
refRaster,
"inm":
calib_vv,
"pixType":
"float",
"interpolator":
"bco",
"ram":
RAMPerProcess,
"elev.dem":
SRTM,
"elev.geoid":
geoid
})
orthoImageName_vh = "{}_{}_{}_{}_{}".format(currentPlatform, tileName,
"vh",
currentOrbitDirection,
acquisition_date)
super_vh, super_vh_dep = OtbAppBank.CreateSuperimposeApplication({
"inr":
refRaster,
"inm":
calib_vh,
"pixType":
"float",
"interpolator":
"bco",
"ram":
RAMPerProcess,
"elev.dem":
SRTM,
"elev.geoid":
geoid
})
super_vv.Execute()
super_vh.Execute()
all_dep.append(super_vv)
all_dep.append(super_vh)
all_superI_vv.append(super_vv)
all_superI_vh.append(super_vh)
all_acquisition_date_vv.append(orthoImageName_vv)
all_acquisition_date_vh.append(orthoImageName_vh)
all_acquisition_date_vv = "_".join(sorted(all_acquisition_date_vv))
all_acquisition_date_vh = "_".join(sorted(all_acquisition_date_vh))
#Concatenate thanks to a BandMath
vv_exp = ",".join(
["im{}b1".format(i + 1) for i in range(len(all_superI_vv))])
vv_exp = "max({})".format(vv_exp)
SAR_vv = os.path.join(output_directory, all_acquisition_date_vv + ".tif")
concatAppli_vv = OtbAppBank.CreateBandMathApplication({
"il":
all_superI_vv,
"exp":
vv_exp,
"out":
SAR_vv,
"ram":
str(RAMPerProcess),
"pixType":
"float"
})
vh_exp = ",".join(
["im{}b1".format(i + 1) for i in range(len(all_superI_vh))])
vh_exp = "max({})".format(vh_exp)
SAR_vh = os.path.join(output_directory, all_acquisition_date_vh + ".tif")
concatAppli_vh = OtbAppBank.CreateBandMathApplication({
"il":
all_superI_vh,
"exp":
vh_exp,
"out":
SAR_vh,
"ram":
str(RAMPerProcess),
"pixType":
"float"
})
ortho_path = writeOutputRaster_2(concatAppli_vv,
overwrite=False,
workingDirectory=workingDirectory,
dep=all_dep)
ortho_path = writeOutputRaster_2(concatAppli_vh,
overwrite=False,
workingDirectory=workingDirectory,
dep=all_dep)
#from the results generate a mask
super_vv = os.path.join(output_directory, all_acquisition_date_vv + ".tif")
border_mask = super_vv.replace(".tif", "_BorderMask.tif")
mask_app, _ = generateBorderMask(super_vv,
border_mask,
RAMPerProcess=RAMPerProcess)
mask_path = writeOutputRaster(mask_app,
overwrite=False,
workingDirectory=workingDirectory)
mask_path_geom = mask_path.replace(".tif", ".geom")
if os.path.exists(mask_path_geom):
os.remove(mask_path_geom)
return (SAR_vv, SAR_vh)
def concatenateImage(self, orthoList, maskList, tile):
"""
Concatenate Ortho at the same date
"""
def sortByFirstElem(MyList):
from collections import defaultdict
"""
Example 1:
MyList = [(1,2),(1,1),(6,1),(1,4),(6,7)]
print sortByElem(MyList)
>> [(1, [2, 1, 4]), (6, [1, 7])]
Example 2:
MyList = [((1,6),2),((1,6),1),((1,2),1),((1,6),4),((1,2),7)]
print sortByElem(MyList)
>> [((1, 2), [1, 7]), ((1, 6), [2, 1, 4])]
"""
d = defaultdict(list)
for k, v in MyList:
d[k].append(v)
return list(d.items())
def findTilesToConcatenate(applicationList):
"""
OUT:
listOfList:
Example [[r1,r2],[r3,r4],... mean
r1 and r2 must be concatenates together
same for r3,r4
"""
concatenate = []
names = [(currentName.split("_")[-1].split("t")[0], currentName)
for currentName in OtbAppBank.unPackFirst(applicationList)
]
names = sortByFirstElem(names)
toConcat = [
rasterList for currentDate, rasterList in names
if len(rasterList) > 2
]
for dateToConcat in toConcat:
tmp = [(currentRaster.split("_")[2], currentRaster)
for currentRaster in dateToConcat]
tmp = sortByFirstElem(tmp)[::-1] #VV first then VH
for pol, rasters in tmp:
concatenate.append(rasters)
Filter = []
for ToConcat in concatenate:
sat = [CToConcat.split("_")[0] for CToConcat in ToConcat]
if not sat.count(sat[0]) == len(sat):
continue
Filter.append(ToConcat)
return Filter
def findMasksToConcatenate(maskList):
concatenate = []
names = [
os.path.split(mask.GetParameterValue("out"))[-1].split("?")[0]
for mask, dep in maskList
]
nameDate = [(name.split("_")[4].split("t")[0], name)
for name in names]
nameDate = sortByFirstElem(nameDate)
for date, maskList in nameDate:
if len(maskList) > 1:
concatenate.append(maskList)
#check if all masks comes from the same satellite
maskFilter = []
for masksToConcat in concatenate:
sat = [
CmasksToConcat.split("_")[0]
for CmasksToConcat in masksToConcat
]
if not sat.count(sat[0]) == len(sat):
continue
maskFilter.append(masksToConcat)
return maskFilter
print("concatenate")
allOrtho = []
allMasks = []
imageList = [(os.path.split(
currentOrtho.GetParameterValue(
OtbAppBank.getInputParameterOutput(currentOrtho)))[-1].split(
"?")[0], currentOrtho, _) for currentOrtho, _ in orthoList]
imageList.sort()
rastersToConcat = findTilesToConcatenate(imageList)
#fill ortho
for rasters in rastersToConcat:
tmp = []
name = []
for pol in rasters:
name.append(pol.replace(".tif", ""))
for currentOrtho, _ in orthoList:
outputParameter = OtbAppBank.getInputParameterOutput(
currentOrtho)
if pol in currentOrtho.GetParameterValue(outputParameter):
if self.wMode == False: tmp.append((currentOrtho, _))
else:
tmp.append(
currentOrtho.GetParameterValue(
OtbAppBank.getInputParameterOutput(
currentOrtho)))
name = "_".join(name) + ".tif"
outputImage = os.path.join(self.outputPreProcess, tile,
name + "?&writegeom=false")
concatAppli = OtbAppBank.CreateBandMathApplication({
"il":
tmp,
"exp":
"max(im1b1,im2b1)",
"ram":
str(self.RAMPerProcess),
"pixType":
"float",
"out":
outputImage
})
allOrtho.append(concatAppli)
for currentOrtho, _ in orthoList:
outputParameter = OtbAppBank.getInputParameterOutput(currentOrtho)
currentName = os.path.split(
currentOrtho.GetParameterValue(outputParameter))[-1].split(
"?")[0]
if not currentName in [
currentRaster for currentRasters in rastersToConcat
for currentRaster in currentRasters
]:
allOrtho.append(currentOrtho)
#fill masks
if not maskList:
return allOrtho, []
masksToConcat = findMasksToConcatenate(maskList)
for mask in masksToConcat:
tmp_m = []
maskName = []
for dateMask in mask:
maskName.append(dateMask)
for currentMask, _ in maskList:
if dateMask in currentMask.GetParameterValue("out"):
if self.wMode == False:
tmp_m.append((currentMask, _))
else:
tmp_m.append(currentMask.GetParameterValue("out"))
maskName = "_".join([
elem.replace(".tif", "").replace("_BorderMask", "")
for elem in maskName
]) + "_BorderMask.tif"
outputImage = os.path.join(self.outputPreProcess, tile, maskName)
concatAppliM = OtbAppBank.CreateBandMathApplication({
"il":
tmp_m,
"exp":
"max(im1b1,im2b1)",
"ram":
str(self.RAMPerProcess),
"pixType":
"uint8",
"out":
outputImage + "?&writegeom=false"
})
allMasks.append((concatAppliM, ""))
for currentMask, _ in maskList:
currentName = os.path.split(
currentMask.GetParameterValue("out"))[-1].split("?")[0]
if not currentName in [
currentRaster for currentRasters in masksToConcat
for currentRaster in currentRasters
]:
allMasks.append((currentMask, _))
return allOrtho, allMasks
def DoAugmentation(samples,
class_augmentation,
strategy,
field,
excluded_fields=[],
Jstdfactor=None,
Sneighbors=None,
workingDirectory=None,
logger=logger):
"""perform data augmentation according to input parameters
Parameters
----------
samples : string
path to the set of samples to augment (OGR geometries must be 'POINT')
class_augmentation : dict
number of new samples to compute by class
strategy : string
which method to use in order to perform data augmentation (replicate/jitter/smote)
field : string
data's field
excluded_fields : list
do not consider these fields to perform data augmentation
Jstdfactor : float
Factor for dividing the standard deviation of each feature
Sneighbors : int
Number of nearest neighbors (smote's method)
workingDirectory : string
path to a working directory
Note
----
This function use the OTB's application **SampleAugmentation**,
more documentation
`here <http://www.orfeo-toolbox.org/Applications/SampleAugmentation.html>`_
"""
from Common import OtbAppBank
samples_dir_o, samples_name = os.path.split(samples)
samples_dir = samples_dir_o
if workingDirectory:
samples_dir = workingDirectory
shutil.copy(samples, samples_dir)
samples = os.path.join(samples_dir, samples_name)
augmented_files = []
for class_name, class_samples_augmentation in list(
class_augmentation.items()):
logger.info(
"{} samples of class {} will be generated by data augmentation ({} method) in {}"
.format(class_samples_augmentation, class_name, strategy, samples))
sample_name_augmented = "_".join([
os.path.splitext(samples_name)[0],
"aug_class_{}.sqlite".format(class_name)
])
output_sample_augmented = os.path.join(samples_dir,
sample_name_augmented)
parameters = {
"in": samples,
"field": field,
"out": output_sample_augmented,
"label": class_name,
"strategy": strategy,
"samples": class_samples_augmentation
}
if excluded_fields:
parameters["exclude"] = excluded_fields
if strategy.lower() == "jitter":
parameters["strategy.jitter.stdfactor"] = Jstdfactor
elif strategy.lower() == "smote":
parameters["strategy.smote.neighbors"] = Sneighbors
augmentation_application = OtbAppBank.CreateSampleAugmentationApplication(
parameters)
augmentation_application.ExecuteAndWriteOutput()
logger.debug("{} samples of class {} were added in {}".format(
class_samples_augmentation, class_name, samples))
augmented_files.append(output_sample_augmented)
outputVector = os.path.join(
samples_dir,
"_".join([os.path.splitext(samples_name)[0], "augmented.sqlite"]))
fut.mergeSQLite("_".join([os.path.splitext(samples_name)[0], "augmented"]),
samples_dir, [samples] + augmented_files)
logger.info("Every data augmentation done in {}".format(samples))
shutil.move(outputVector, os.path.join(samples_dir_o, samples_name))
#clean-up
for augmented_file in augmented_files:
os.remove(augmented_file)
def regularisation(raster, threshold, nbcores, path, ram = "128"):
filetodelete = []
# First regularisation in connection 8, second in connection 4
init_regul = time.time()
# A mask for each regularization rule
# Agricultuture
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==11 || im1b1==12)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_1.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_1.tif'))
# Forest
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==31 || im1b1==32)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_2.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_2.tif'))
# Urban
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==41 || im1b1==42 || im1b1==43)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_3.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_3.tif'))
# Open natural areas
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==34 || im1b1==36 || im1b1==211)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_4.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_4.tif'))
# Bare soil
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==45 || im1b1==46)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_5.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_5.tif'))
# Perennial agriculture
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==221 || im1b1==222)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_6.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_6.tif'))
# Road
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==44)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_7.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_7.tif'))
# Water
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==51)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_8.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_8.tif'))
# Snow and glacier
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster,
"exp": '(im1b1==53)?im1b1:0',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_9.tif')})
bandMathAppli.ExecuteAndWriteOutput()
filetodelete.append(os.path.join(path, 'mask_9.tif'))
for i in range(9):
command = "gdalwarp -q -multi -wo NUM_THREADS=%s -dstnodata 0 %s/mask_%s.tif %s/mask_nd_%s.tif"%(nbcores, \
path, \
str(i + 1), \
path, \
str(i + 1))
Utils.run(command)
filetodelete.append("%s/mask_nd_%s.tif"%(path, str(i + 1)))
masktime = time.time()
print(" ".join([" : ".join(["Masks generation for adaptive rules", str(masktime - init_regul)]), "seconds"]))
# Two successive regularisation (8 neighbors then 4 neighbors)
for i in range(2):
if i == 0:
connexion = 8
else :
connexion = 4
# Tiles number to treat in parralel
pool = Pool(processes = 6)
iterable = (np.arange(6)).tolist()
function = partial(gdal_sieve, threshold, connexion, path)
pool.map(function, iterable)
pool.close()
pool.join()
for j in range(6):
command = "gdalwarp -q -multi -wo NUM_THREADS=%s -dstnodata 0 %s/mask_%s_%s.tif %s/mask_nd_%s_%s.tif"%(nbcores, \
path, \
str(j + 1), \
str(connexion), \
path, \
str(j + 1), \
str(connexion))
Utils.run(command)
for j in range(6):
os.remove(path + "/mask_%s_%s.tif"%(str(j + 1),str(connexion)))
for j in range(6):
os.remove(path + "/mask_nd_%s_8.tif"%(str(j + 1)))
adaptativetime = time.time()
print(" ".join([" : ".join(["Adaptative regularizations", str(adaptativetime - masktime)]), "seconds"]))
# Fusion of rule-based regularisation
rastersList = [os.path.join(path, "mask_nd_1_4.tif"), os.path.join(path, "mask_nd_2_4.tif"), os.path.join(path, "mask_nd_3_4.tif"), \
os.path.join(path, "mask_nd_4_4.tif"), os.path.join(path, "mask_nd_5_4.tif"), os.path.join(path, "mask_nd_6_4.tif"), \
os.path.join(path, "mask_nd_7.tif"), os.path.join(path, "mask_nd_8.tif"), os.path.join(path, "mask_nd_9.tif")]
bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": rastersList,
"exp": 'im1b1+im2b1+\
im3b1+im4b1+\
im5b1+im6b1+\
im7b1+im8b1+\
im9b1',
"ram": str(0.2 * float(ram)),
"pixType": "uint8",
"out": os.path.join(path, 'mask_regul_adapt.tif')})
bandMathAppli.ExecuteAndWriteOutput()
for filemask in rastersList:
os.remove(filemask)
command = "gdalwarp -q -multi -wo NUM_THREADS="
command += "%s -dstnodata 0 %s/mask_regul_adapt.tif %s/mask_nd_regul_adapt.tif"%(nbcores, \
path, \
path)
Utils.run(command)
filetodelete.append("%s/mask_regul_adapt.tif"%(path))
# Regularisation based on majority voting
# 8 neighbors
command = "gdal_sieve.py -q -8 -st "
command += "%s %s/mask_nd_regul_adapt.tif %s/mask_regul_adapt_0.tif" %(threshold, \
path, \
path)
Utils.run(command)
filetodelete.append("%s/mask_nd_regul_adapt.tif"%(path))
command = "gdalwarp -q -multi -wo NUM_THREADS="
command += "%s -dstnodata 0 %s/mask_regul_adapt_0.tif %s/mask_nd_regul_adapt_0.tif"%(nbcores, \
path, \
path)
Utils.run(command)
filetodelete.append("%s/mask_regul_adapt_0.tif"%(path))
# 4 neighbors
command = "gdal_sieve.py -q -4 -st "
command += "%s %s/mask_nd_regul_adapt_0.tif %s/regul_adapt_maj.tif" %(threshold, \
path, \
path)
Utils.run(command)
filetodelete.append("%s/mask_nd_regul_adapt_0.tif"%(path))
out_classif_sieve = "%s/regul_adapt_maj.tif"%(path)
majoritytime = time.time()
print(" ".join([" : ".join(["Majority voting regularization", str(majoritytime - adaptativetime)]), "seconds"]))
for filetodel in filetodelete:
if os.path.exists(filetodel):
os.remove(filetodel)
end_regul = time.time() - init_regul
return out_classif_sieve, end_regul
def mergeFinalClassifications(iota2_dir,
dataField,
nom_path,
colorFile,
runs=1,
pixType='uint8',
method="majorityvoting",
undecidedlabel=255,
dempstershafer_mob="precision",
keep_runs_results=True,
enableCrossValidation=False,
validationShape=None,
workingDirectory=None,
logger=logger):
"""function use to merge classifications by majorityvoting or dempstershafer's method and evaluate it.
get all classifications Classif_Seed_*.tif in the /final directory and fusion them
under the raster call Classifications_fusion.tif. Then compute statistics using the
results_utils library
Parameters
----------
iota2_dir : string
path to the iota2's output path
dataField : string
data's field name
nom_path : string
path to the nomenclature file
colorFile : string
path to the color file description
runs : int
number of iota2 runs (random learning splits)
pixType : string
output pixel format (available in OTB)
method : string
fusion's method (majorityvoting/dempstershafer)
undecidedlabel : int
label for label for un-decisions
dempstershafer_mob : string
mass of belief measurement (precision/recall/accuracy/kappa)
keep_runs_results : bool
flag to inform if seeds results could be overwritten
enableCrossValidation : bool
flag to inform if cross validation is enable
validationShape : string
path to a shape dedicated to validate fusion of classifications
workingDirectory : string
path to a working directory
See Also
--------
results_utils.gen_confusion_matrix_fig
results_utils.stats_report
"""
import shutil
from Common import OtbAppBank as otbApp
from Validation import ResultsUtils as ru
from Common import CreateIndexedColorImage as color
fusion_name = "Classifications_fusion.tif"
new_results_seed_file = "RESULTS_seeds.txt"
fusion_vec_name = "fusion_validation" #without extension
confusion_matrix_name = "fusionConfusion.png"
if not method in ["majorityvoting", "dempstershafer"]:
err_msg = "the fusion method must be 'majorityvoting' or 'dempstershafer'"
logger.error(err_msg)
raise Exception(err_msg)
if not dempstershafer_mob in ["precision", "recall", "accuracy", "kappa"]:
err_msg = "the dempstershafer MoB must be 'precision' or 'recall' or 'accuracy' or 'kappa'"
logger.error(err_msg)
raise Exception(err_msg)
iota2_dir_final = os.path.join(iota2_dir, "final")
wd = iota2_dir_final
wd_merge = os.path.join(iota2_dir_final, "merge_final_classifications")
if workingDirectory:
wd = workingDirectory
wd_merge = workingDirectory
final_classifications = [
fut.FileSearch_AND(iota2_dir_final, True,
"Classif_Seed_{}.tif".format(run))[0]
for run in range(runs)
]
fusion_path = os.path.join(wd, fusion_name)
fusion_options = compute_fusion_options(iota2_dir_final,
final_classifications, method,
undecidedlabel, dempstershafer_mob,
pixType, fusion_path)
logger.debug("fusion options:")
logger.debug(fusion_options)
fusion_app = otbApp.CreateFusionOfClassificationsApplication(
fusion_options)
logger.debug("START fusion of final classifications")
fusion_app.ExecuteAndWriteOutput()
logger.debug("END fusion of final classifications")
fusion_color_index = color.CreateIndexedColorImage(
fusion_path,
colorFile,
co_option=["COMPRESS=LZW"],
output_pix_type=gdal.GDT_Byte
if pixType == "uint8" else gdal.GDT_UInt16)
confusion_matrix = os.path.join(iota2_dir_final,
"merge_final_classifications",
"confusion_mat_maj_vote.csv")
if enableCrossValidation is False:
vector_val = fut.FileSearch_AND(
os.path.join(iota2_dir_final, "merge_final_classifications"), True,
"_majvote.sqlite")
else:
vector_val = fut.FileSearch_AND(os.path.join(iota2_dir, "dataAppVal"),
True, "val.sqlite")
if validationShape:
validation_vector = validationShape
else:
fut.mergeSQLite(fusion_vec_name, wd_merge, vector_val)
validation_vector = os.path.join(wd_merge, fusion_vec_name + ".sqlite")
confusion = otbApp.CreateComputeConfusionMatrixApplication({
"in":
fusion_path,
"out":
confusion_matrix,
"ref":
"vector",
"ref.vector.nodata":
"0",
"ref.vector.in":
validation_vector,
"ref.vector.field":
dataField.lower(),
"nodatalabel":
"0",
"ram":
"5000"
})
confusion.ExecuteAndWriteOutput()
maj_vote_conf_mat = os.path.join(iota2_dir_final, confusion_matrix_name)
ru.gen_confusion_matrix_fig(csv_in=confusion_matrix,
out_png=maj_vote_conf_mat,
nomenclature_path=nom_path,
undecidedlabel=undecidedlabel,
dpi=900)
if keep_runs_results:
seed_results = fut.FileSearch_AND(iota2_dir_final, True,
"RESULTS.txt")[0]
shutil.copy(seed_results,
os.path.join(iota2_dir_final, new_results_seed_file))
maj_vote_report = os.path.join(iota2_dir_final, "RESULTS.txt")
ru.stats_report(csv_in=[confusion_matrix],
nomenclature_path=nom_path,
out_report=maj_vote_report,
undecidedlabel=undecidedlabel)
if workingDirectory:
shutil.copy(fusion_path, iota2_dir_final)
shutil.copy(fusion_color_index, iota2_dir_final)
os.remove(fusion_path)
def clumpAndStackClassif(path,
raster,
outpath,
ram,
float64=False,
exe64="",
logger=logger):
begin_clump = time.time()
# split path and file name of outfilename
out = os.path.dirname(outpath)
outfilename = os.path.basename(outpath)
# Clump Classif with OTB segmentation algorithm
clumpAppli = OtbAppBank.CreateClumpApplication({
"in":
raster,
"filter.cc.expr":
'distance<1',
"ram":
str(0.2 * float(ram)),
"pixType":
'uint32',
"mode":
"raster",
"filter":
"cc",
"mode.raster.out":
os.path.join(path, 'clump.tif')
})
if not float64:
clumpAppli.Execute()
clumptime = time.time()
logger.info(" ".join([
" : ".join(
["Input raster well clumped : ",
str(clumptime - begin_clump)]), "seconds"
]))
# Add 300 to all clump ID
bandMathAppli = OtbAppBank.CreateBandMathApplication({
"il":
clumpAppli,
"exp":
'im1b1+300',
"ram":
str(0.2 * float(ram)),
"pixType":
'uint32',
"out":
os.path.join(path, 'clump300.tif')
})
bandMathAppli.Execute()
dataRamAppli = OtbAppBank.CreateBandMathApplication({
"il":
raster,
"exp":
'im1b1',
"ram":
str(0.2 * float(ram)),
"pixType":
'uint8'
})
dataRamAppli.Execute()
concatImages = OtbAppBank.CreateConcatenateImagesApplication({
"il": [dataRamAppli, bandMathAppli],
"ram":
str(0.2 * float(ram)),
"pixType":
'uint32',
"out":
os.path.join(path, outfilename)
})
concatImages.ExecuteAndWriteOutput()
concattime = time.time()
logger.info(" ".join([
" : ".join([
"Regularized and Clumped rasters concatenation : ",
str(concattime - clumptime)
]), "seconds"
]))
shutil.copyfile(os.path.join(path, outfilename),
os.path.join(out, outfilename))
else:
clumpAppli.ExecuteAndWriteOutput()
command = '%s/iota2BandMath %s "%s" %s %s'%(exe64, \
os.path.join(path, 'clump.tif'), \
"im1b1+300", \
os.path.join(path, 'clump300.tif'), \
10)
try:
Utils.run(command)
clumptime = time.time()
logger.info(" ".join([
" : ".join([
"Input raster well clumped : ",
str(clumptime - begin_clump)
]), "seconds"
]))
except:
logger.error(
"Application 'iota2BandMath' for 64 bits does not exist, please change 64 bits binaries path"
)
sys.exit()
command = '%s/iota2ConcatenateImages %s %s %s %s'%((exe64,
raster, \
os.path.join(path, 'clump300.tif'), \
os.path.join(path, outfilename),
10))
try:
Utils.run(command)
concattime = time.time()
logger.info(" ".join([" : ".join(["Regularized and Clumped rasters concatenation : ", \
str(concattime - clumptime)]), "seconds"]))
shutil.copyfile(os.path.join(path, outfilename),
os.path.join(out, outfilename))
os.remove(os.path.join(path, 'clump.tif'))
os.remove(os.path.join(path, 'clump300.tif'))
except:
logger.error(
"Application 'iota2ConcatenateImages' for 64 bits does not exist, please change 64 bits binaries path"
)
sys.exit()
command = "gdal_translate -q -b 2 -ot Uint32 %s %s" % (os.path.join(
path, outfilename), os.path.join(path, "clump32bits.tif"))
Utils.run(command)
shutil.copy(os.path.join(path, "clump32bits.tif"), out)
os.remove(os.path.join(path, "clump32bits.tif"))
if os.path.exists(os.path.join(path, outfilename)):
os.remove(os.path.join(path, outfilename))
clumptime = time.time()
logger.info(" ".join(
[" : ".join(["Clump : ", str(clumptime - begin_clump)]), "seconds"]))
def main(ortho=None, configFile=None, dates=None, tileName=None, logger=logger):
import ast
from Common import FileUtils
config = configparser.ConfigParser()
config.read(configFile)
wMode = ast.literal_eval(config.get('Processing','writeTemporaryFiles'))
stackFlag = ast.literal_eval(config.get('Processing','outputStack'))
stdoutfile=None
stderrfile=open("S1ProcessorErr.log",'a')
RAMPerProcess=int(config.get('Processing','RAMPerProcess'))
if "logging" in config.get('Processing','Mode').lower():
stdoutfile=open("S1ProcessorOut.log",'a')
stderrfile=open("S1ProcessorErr.log",'a')
if "debug" in config.get('Processing','Mode').lower():
stdoutfile=None
stderrfile=None
outputPreProcess = config.get('Paths','Output')
wr = config.get('Filtering','Window_radius')
directories=next(os.walk(outputPreProcess))
SARFilter = []
#OUTCOREs
for d in directories[1]:
s1_vv_DES_scene = sorted([currentOrtho for currentOrtho in getOrtho(ortho,"s1(.*)"+d+"(.*)vv(.*)DES(.*)tif")],key=getDatesInOtbOutputName)
if s1_vv_DES_scene:
outcore_s1_vv_DES = os.path.join(directories[0],d,"outcore_S1_vv_DES.tif")
outcore_s1_vv_DES_dates = os.path.join(directories[0],d,"S1_vv_DES_dates.txt")
new_outcore_s1_vv_DES_dates = compareDates(outcore_s1_vv_DES_dates, dates["s1_DES"])
if new_outcore_s1_vv_DES_dates:
FileUtils.WriteNewFile(outcore_s1_vv_DES_dates,
"\n".join(dates["s1_DES"]))
s1_vv_DES_outcore = remove_old_dates(s1_vv_DES_scene,
new_outcore_s1_vv_DES_dates)
if s1_vv_DES_outcore or not os.path.exists(outcore_s1_vv_DES):
s1_vv_DES_outcore = OtbAppBank.CreateMultitempFilteringOutcore({"inl" : s1_vv_DES_outcore,
"oc" : outcore_s1_vv_DES,
"wr" : str(wr),
"ram" : str(RAMPerProcess),
"pixType" : "float"})
logger.info("writing : {}".format(outcore_s1_vv_DES))
s1_vv_DES_outcore.ExecuteAndWriteOutput()
logger.info("{} : done".format(outcore_s1_vv_DES))
s1_vh_DES_scene = sorted([currentOrtho for currentOrtho in getOrtho(ortho,"s1(.*)"+d+"(.*)vh(.*)DES(.*)tif")],key=getDatesInOtbOutputName)
if s1_vh_DES_scene:
outcore_s1_vh_DES = os.path.join(directories[0],d,"outcore_S1_vh_DES.tif")
outcore_s1_vh_DES_dates = os.path.join(directories[0],d,"S1_vh_DES_dates.txt")
new_outcore_s1_vh_DES_dates = compareDates(outcore_s1_vh_DES_dates, dates["s1_DES"])
if new_outcore_s1_vh_DES_dates:
FileUtils.WriteNewFile(outcore_s1_vh_DES_dates,
"\n".join(dates["s1_DES"]))
s1_vh_DES_outcore = remove_old_dates(s1_vh_DES_scene,
new_outcore_s1_vh_DES_dates)
if s1_vh_DES_outcore or not os.path.exists(outcore_s1_vh_DES):
s1_vh_DES_outcore = OtbAppBank.CreateMultitempFilteringOutcore({"inl" : s1_vh_DES_outcore,
"oc" : outcore_s1_vh_DES,
"wr" : str(wr),
"ram" : str(RAMPerProcess),
"pixType" : "float"})
logger.info("writing : {}".format(outcore_s1_vh_DES))
s1_vh_DES_outcore.ExecuteAndWriteOutput()
logger.info("{} : done".format(outcore_s1_vh_DES))
s1_vv_ASC_scene = sorted([currentOrtho for currentOrtho in getOrtho(ortho,"s1(.*)"+d+"(.*)vv(.*)ASC(.*)tif")], key=getDatesInOtbOutputName)
if s1_vv_ASC_scene:
outcore_s1_vv_ASC = os.path.join(directories[0],d,"outcore_S1_vv_ASC.tif")
outcore_s1_vv_ASC_dates = os.path.join(directories[0],d,"S1_vv_ASC_dates.txt")
new_outcore_s1_vv_ASC_dates = compareDates(outcore_s1_vv_ASC_dates, dates["s1_ASC"])
if new_outcore_s1_vv_ASC_dates:
FileUtils.WriteNewFile(outcore_s1_vv_ASC_dates,
"\n".join(dates["s1_ASC"]))
s1_vv_ASC_outcore = remove_old_dates(s1_vv_ASC_scene,
new_outcore_s1_vv_ASC_dates)
if s1_vv_ASC_outcore or not os.path.exists(outcore_s1_vv_ASC):
s1_vv_ASC_outcore = OtbAppBank.CreateMultitempFilteringOutcore({"inl" : s1_vv_ASC_outcore,
"oc" : outcore_s1_vv_ASC,
"wr" : str(wr),
"ram" : str(RAMPerProcess),
"pixType" : "float"})
logger.info("writing : {}".format(outcore_s1_vv_ASC))
s1_vv_ASC_outcore.ExecuteAndWriteOutput()
logger.info("{} : done".format(outcore_s1_vv_ASC))
s1_vh_ASC_scene = sorted([currentOrtho for currentOrtho in getOrtho(ortho,"s1(.*)"+d+"(.*)vh(.*)ASC(.*)tif")], key=getDatesInOtbOutputName)
if s1_vh_ASC_scene:
outcore_s1_vh_ASC = os.path.join(directories[0],d,"outcore_S1_vh_ASC.tif")
outcore_s1_vh_ASC_dates = os.path.join(directories[0],d,"S1_vh_ASC_dates.txt")
new_outcore_s1_vh_ASC_dates = compareDates(outcore_s1_vh_ASC_dates, dates["s1_ASC"])
if new_outcore_s1_vh_ASC_dates:
FileUtils.WriteNewFile(outcore_s1_vh_ASC_dates,
"\n".join(dates["s1_ASC"]))
s1_vh_ASC_outcore = remove_old_dates(s1_vh_ASC_scene,
new_outcore_s1_vh_ASC_dates)
if s1_vh_ASC_outcore or not os.path.exists(outcore_s1_vh_ASC):
s1_vh_ASC_outcore = OtbAppBank.CreateMultitempFilteringOutcore({"inl" : s1_vh_ASC_outcore,
"oc" : outcore_s1_vh_ASC,
"wr" : str(wr),
"ram" : str(RAMPerProcess),
"pixType" : "float"})
logger.info("writing : {}".format(outcore_s1_vh_ASC))
s1_vh_ASC_outcore.ExecuteAndWriteOutput()
logger.info("{} : done".format(outcore_s1_vh_ASC))
try:
os.makedirs(os.path.join(directories[0],d,"filtered"))
except:
pass
#FILTERING
if s1_vv_DES_scene:
enl = os.path.join(directories[0],d,"filtered/enl_S1_vv_DES.tif")
s1_vv_DES_filtered = os.path.join(directories[0],d,"filtered/S1_vv_DES_Filtered.tif")
s1_vv_DES_filtered_app, a, b = OtbAppBank.CreateMultitempFilteringFilter({"inl" : s1_vv_DES_scene,
"oc" : outcore_s1_vv_DES,
"wr" : str(wr),
"enl" : enl,
"ram" : str(RAMPerProcess),
"pixType" : "float",
"outputstack" : s1_vv_DES_filtered})
SARFilter.append((s1_vv_DES_filtered_app, a, b))
if s1_vh_DES_scene:
enl = os.path.join(directories[0],d,"filtered/enl_S1_vh_DES.tif")
s1_vh_DES_filtered = os.path.join(directories[0],d,"filtered/S1_vh_DES_Filtered.tif")
s1_vh_DES_filtered_app, a, b = OtbAppBank.CreateMultitempFilteringFilter({"inl" : s1_vh_DES_scene,
"oc" : outcore_s1_vh_DES,
"wr" : str(wr),
"enl" : enl,
"ram" : str(RAMPerProcess),
"pixType" : "float",
"outputstack" : s1_vh_DES_filtered})
SARFilter.append((s1_vh_DES_filtered_app, a, b))
if s1_vv_ASC_scene:
enl = os.path.join(directories[0],d,"filtered/enl_S1_vv_ASC.tif")
s1_vv_ASC_filtered = os.path.join(directories[0],d,"filtered/S1_vv_ASC_Filtered.tif")
s1_vv_ASC_filtered_app, a, b = OtbAppBank.CreateMultitempFilteringFilter({"inl" : s1_vv_ASC_scene,
"oc" : outcore_s1_vv_ASC,
"wr" : str(wr),
"enl" : enl,
"ram" : str(RAMPerProcess),
"pixType" : "float",
"outputstack" : s1_vv_ASC_filtered})
SARFilter.append((s1_vv_ASC_filtered_app, a, b))
if s1_vh_ASC_scene:
enl = os.path.join(directories[0],d,"filtered/enl_S1_vh_ASC.tif")
s1_vh_ASC_filtered = os.path.join(directories[0],d,"filtered/S1_vh_ASC_Filtered.tif")
s1_vh_ASC_filtered_app, a, b = OtbAppBank.CreateMultitempFilteringFilter({"inl" : s1_vh_ASC_scene,
"oc" : outcore_s1_vh_ASC,
"wr" : str(wr),
"enl" : enl,
"ram" : str(RAMPerProcess),
"pixType" : "float",
"outputstack" : s1_vh_ASC_filtered})
SARFilter.append((s1_vh_ASC_filtered_app, a, b))
return SARFilter
def doOrthoByTile(self, rasterList, tileName):
allOrtho = []
for i in range(len(rasterList)):
raster, tileOrigin = rasterList[i]
manifest = raster.getManifest()
calibrationApplication = rasterList[i][
0].GetCalibrationApplication()
for calib, dep in calibrationApplication:
image = calib.GetParameterValue("out")
currentDate = getDateFromS1Raster(image)
currentPolar = getPolarFromS1Raster(image)
currentPlatform = getPlatformFromS1Raster(image)
currentOrbitDirection = getOrbitDirection(manifest)
outUTMZone = tileName[0:2]
outUTMNorthern = str(int(tileName[2] >= 'N'))
workingDirectory = os.path.join(self.outputPreProcess,
tileName)
if not os.path.exists(workingDirectory):
os.makedirs(workingDirectory)
inEPSG = 4326
outEPSG = 32600 + int(outUTMZone)
if not outUTMNorthern == "0":
outEPSG = outEPSG + 100
if self.ManyProjection:
[(x, y, dummy)] = converCoord([tileOrigin[0]], inEPSG,
outEPSG)
[(lrx, lry, dummy)] = converCoord([tileOrigin[2]], inEPSG,
outEPSG)
uniqueProj = None
refRaster = None
else:
refRaster = FileSearch_AND(
self.referencesFolder + "/T" + tileName, True,
self.rasterPattern)[0]
print("reference raster found : " + refRaster)
if self.ManyProjection and outUTMNorthern == "1" and y > 0:
y = y - 10000000.
lry = lry - 10000000.
if self.ManyProjection and outUTMNorthern == "0" and y < 0:
y = y + 10000000.
lry = lry + 10000000.
orthoImageName = currentPlatform+"_"+\
tileName+"_"+\
currentPolar+"_"+\
currentOrbitDirection+"_"+\
currentDate+".tif"
inputImage = (calib, dep)
if self.wMode: inputImage = calib.GetParameterValue("out")
orthoRaster = os.path.join(workingDirectory, orthoImageName)
if self.ManyProjection:
sizeX = abs(lrx - x) / self.outSpacialRes
sizeY = abs(lry - y) / self.outSpacialRes
ortho, ortho_dep = OtbAppBank.CreateOrthoRectification({
"in":
inputImage,
"out":
orthoRaster,
"ram":
self.RAMPerProcess,
"outputs.spacingx":
self.outSpacialRes,
"outputs.spacingy":
-self.outSpacialRes,
"outputs.sizex":
sizeX,
"outputs.sizey":
sizeY,
"opt.gridspacing":
self.gridSpacing,
"map.utm.zone":
outUTMZone,
"map.utm.northhem":
outUTMNorthern,
"outputs.ulx":
x,
"outputs.uly":
y,
"elev.dem":
self.SRTM,
"elev.geoid":
self.geoid,
"map":
"utm"
})
else:
ortho, ortho_dep = OtbAppBank.CreateSuperimposeApplication(
{
"inr": refRaster,
"inm": inputImage,
"pixType": "float",
"interpolator": "bco",
"ram": self.RAMPerProcess,
"io.out": orthoRaster,
"elev.dem": self.SRTM,
"elev.geoid": self.geoid
})
allOrtho.append((ortho, ortho_dep))
return allOrtho
def coregister(insrc,
inref,
band,
bandref,
resample=1,
step=256,
minstep=16,
minsiftpoints=40,
iterate=1,
prec=3,
mode=2,
datadir=None,
pattern='*STACK.tif',
datatype='S2',
writeFeatures=False,
workingDirectory=None):
""" register an image / a time series on a reference image
Parameters
----------
insrc : string
source raster
inref : string
reference raster
band : int
band number for the source raster
bandref : int
band number for the raster reference raster
resample : boolean
resample to reference raster resolution
step : int
initial step between the geobins
minstep : int
minimal step between the geobins when iterates
minsiftpoints : int
minimal number of sift points to perform the registration
iterate : boolean
argument to iterate with smaller geobin step to find more sift points
prec : int
precision between the source and reference image (in source pixel unit)
mode : int
registration mode,
1 : simple registration ;
2 : time series registration ;
3 : time series cascade registration (to do)
datadir : string
path to the data directory
pattern : string
pattern of the STACK files to register
writeFeatures : boolean
argument to keep temporary files
Note
------
This function use the OTB's application **OrthoRectification** and **SuperImpose**
more documentation for
`OrthoRectification <https://www.orfeo-toolbox.org/Applications/OrthoRectification.html>`_
and
`SuperImpose <https://www.orfeo-toolbox.org/Applications/Superimpose.html>`_
"""
from Common.FileUtils import ensure_dir
pathWd = os.path.dirname(
insrc) if not workingDirectory else workingDirectory
if os.path.exists(pathWd) == False:
ensure_dir(pathWd)
srcClip = os.path.join(pathWd, 'tempSrcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication({
"in": insrc,
"mode": "fit",
"mode.fit.im": inref,
"out": srcClip,
"pixType": "uint16"
})
extractROIApp.ExecuteAndWriteOutput()
# #SensorModel generation
SensorModel = os.path.join(pathWd, 'SensorModel.geom')
PMCMApp = OtbAppBank.CreatePointMatchCoregistrationModel({
"in":
srcClip,
"band1":
band,
"inref":
inref,
"bandref":
bandref,
"resample":
resample,
"precision":
str(prec),
"mfilter":
"1",
"backmatching":
"1",
"outgeom":
SensorModel,
"initgeobinstep":
str(step),
"mingeobinstep":
str(minstep),
"minsiftpoints":
str(minsiftpoints),
"iterate":
iterate
})
PMCMApp.ExecuteAndWriteOutput()
# mode 1 : application on the source image
if mode == 1 or mode == 3:
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' + SensorModel)
ds = gdal.Open(srcClip)
prj = ds.GetProjection()
gt = ds.GetGeoTransform()
srs = osr.SpatialReference()
srs.ImportFromWkt(prj)
code = srs.GetAuthorityCode(None)
gsp = str(int(2 * round(max(abs(gt[1]), abs(gt[5])))))
ds = None
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in": io_Src,
"io.out": outSrc,
"map": "epsg",
"map.epsg.code": code,
"opt.gridspacing": gsp,
"pixType": "uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalOutput = os.path.join(
pathWd,
os.path.basename(insrc.replace(ext, ext.replace('.', '_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication({
"inr":
srcClip,
"inm":
orthoRecApp[0],
"out":
finalOutput,
"pixType":
"uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(finalOutput, insrc.replace(ext,
ext.replace('.', '_COREG.')))
shutil.move(finalOutput.replace(ext, '.geom'),
insrc.replace(ext, '_COREG.geom'))
# Mask registration if exists
masks = glob.glob(
os.path.dirname(insrc) + os.sep + 'MASKS' + os.sep +
'*BINARY_MASK' + ext)
if len(masks) != 0:
for mask in masks:
srcClip = os.path.join(pathWd, 'tempSrcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication({
"in":
mask,
"mode":
"fit",
"mode.fit.im":
inref,
"out":
srcClip,
"pixType":
"uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(mask + '?&skipcarto=true&geom=' + SensorModel)
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalMask = os.path.join(
pathWd,
os.path.basename(
mask.replace(ext, ext.replace('.', '_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication({
"inr":
mask,
"inm":
orthoRecApp[0],
"out":
finalMask,
"pixType":
"uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
if finalMask != mask.replace(ext, ext.replace('.', '_COREG.')):
shutil.move(finalMask,
mask.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalMask.replace(ext, '.geom'),
mask.replace(ext, '_COREG.geom'))
if mode == 3:
folders = glob.glob(os.path.join(datadir, '*'))
vhr_ref = inref
if datatype in ['S2', 'S2_S2C']:
dates = [
os.path.basename(fld).split('_')[1].split("-")[0]
for fld in folders
]
ref_date = os.path.basename(insrc).split('_')[1].split("-")[0]
elif datatype in ['L5', 'L8']:
dates = [
os.path.basename(fld).split('_')[3] for fld in folders
]
ref_date = os.path.basename(insrc).split('_')[3]
dates.sort()
ref_date_ind = dates.index(ref_date)
bandref = band
clean_dates = [ref_date]
for date in reversed(dates[:ref_date_ind]):
inref = glob.glob(
os.path.join(datadir, '*' + clean_dates[-1] + '*',
pattern))[0]
insrc = glob.glob(
os.path.join(datadir, '*' + date + '*', pattern))[0]
srcClip = os.path.join(pathWd, 'srcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication({
"in":
insrc,
"mode":
"fit",
"mode.fit.im":
inref,
"out":
srcClip,
"pixType":
"uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSensorModel = os.path.join(pathWd,
'SensorModel_%s.geom' % date)
try:
PMCMApp = OtbAppBank.CreatePointMatchCoregistrationModel({
"in":
srcClip,
"band1":
band,
"inref":
inref,
"bandref":
bandref,
"resample":
resample,
"precision":
str(prec),
"mfilter":
"1",
"backmatching":
"1",
"outgeom":
outSensorModel,
"initgeobinstep":
str(step),
"mingeobinstep":
str(minstep),
"minsiftpoints":
str(minsiftpoints),
"iterate":
iterate
})
PMCMApp.ExecuteAndWriteOutput()
except RuntimeError:
shutil.copy(SensorModel, outSensorModel)
logger.warning(
'Coregistration failed, %s will be process with %s' %
(insrc, outSensorModel))
continue
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' +
outSensorModel)
ds = gdal.Open(srcClip)
prj = ds.GetProjection()
gt = ds.GetGeoTransform()
srs = osr.SpatialReference()
srs.ImportFromWkt(prj)
code = srs.GetAuthorityCode(None)
gsp = str(int(2 * round(max(abs(gt[1]), abs(gt[5])))))
ds = None
try:
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
except RuntimeError:
os.remove(outSensorModel)
shutil.copy(SensorModel, outSensorModel)
logger.warning(
'Coregistration failed, %s will be process with %s' %
(insrc, outSensorModel))
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
continue
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalOutput = os.path.join(
pathWd,
os.path.basename(
insrc.replace(ext, ext.replace('.', '_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication({
"inr":
srcClip,
"inm":
orthoRecApp[0],
"out":
finalOutput,
"pixType":
"uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(finalOutput,
insrc.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalOutput.replace(ext, '.geom'),
insrc.replace(ext, '_COREG.geom'))
# Mask registration if exists
masks = glob.glob(
os.path.dirname(insrc) + os.sep + 'MASKS' + os.sep +
'*BINARY_MASK' + ext)
if len(masks) != 0:
for mask in masks:
srcClip = os.path.join(pathWd, 'srcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication(
{
"in": mask,
"mode": "fit",
"mode.fit.im": inref,
"out": srcClip,
"pixType": "uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' +
outSensorModel)
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalMask = os.path.join(
pathWd,
os.path.basename(
mask.replace(ext, ext.replace('.',
'_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication(
{
"inr": srcClip,
"inm": orthoRecApp[0],
"out": finalMask,
"pixType": "uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(
finalMask,
mask.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalMask.replace(ext, '.geom'),
mask.replace(ext, '_COREG.geom'))
if not writeFeatures and os.path.exists(outSensorModel):
os.remove(outSensorModel)
if datatype in ['S2', 'S2_S2C']:
mtd_file = glob.glob(
os.path.join(os.path.dirname(insrc), '*_MTD_ALL*'))[0]
cloud_clear = get_S2_Tile_Cloud_Cover(mtd_file)
cover = get_S2_Tile_Coverage(mtd_file)
if cloud_clear > 0.6 and cover > 0.8:
clean_dates.append(date)
elif datatype in ['L5', 'L8']:
mlt_file = glob.glob(
os.path.join(os.path.dirname(insrc), '*_MTL*'))[0]
cloud_clear = get_L8_Tile_Cloud_Cover(mlt_file)
if cloud_clear > 0.6:
clean_dates.append(date)
clean_dates = [ref_date]
for date in dates[ref_date_ind + 1:]:
inref = glob.glob(
os.path.join(datadir, '*' + clean_dates[-1] + '*',
pattern))[0]
insrc = glob.glob(
os.path.join(datadir, '*' + date + '*', pattern))[0]
srcClip = os.path.join(pathWd, 'srcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication({
"in":
insrc,
"mode":
"fit",
"mode.fit.im":
inref,
"out":
srcClip,
"pixType":
"uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSensorModel = os.path.join(pathWd,
'SensorModel_%s.geom' % date)
try:
PMCMApp = OtbAppBank.CreatePointMatchCoregistrationModel({
"in":
srcClip,
"band1":
band,
"inref":
inref,
"bandref":
bandref,
"resample":
resample,
"precision":
str(prec),
"mfilter":
"1",
"backmatching":
"1",
"outgeom":
outSensorModel,
"initgeobinstep":
str(step),
"mingeobinstep":
str(minstep),
"minsiftpoints":
str(minsiftpoints),
"iterate":
iterate
})
PMCMApp.ExecuteAndWriteOutput()
except RuntimeError:
shutil.copy(SensorModel, outSensorModel)
logger.warning(
'Coregistration failed, %s will be process with %s' %
(insrc, outSensorModel))
continue
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' +
outSensorModel)
ds = gdal.Open(srcClip)
prj = ds.GetProjection()
gt = ds.GetGeoTransform()
srs = osr.SpatialReference()
srs.ImportFromWkt(prj)
code = srs.GetAuthorityCode(None)
gsp = str(int(2 * round(max(abs(gt[1]), abs(gt[5])))))
ds = None
try:
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
except RuntimeError:
os.remove(outSensorModel)
shutil.copy(SensorModel, outSensorModel)
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
continue
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalOutput = os.path.join(
pathWd,
os.path.basename(
insrc.replace(ext, ext.replace('.', '_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication({
"inr":
srcClip,
"inm":
orthoRecApp[0],
"out":
finalOutput,
"pixType":
"uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(finalOutput,
insrc.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalOutput.replace(ext, '.geom'),
insrc.replace(ext, '_COREG.geom'))
# Mask registration if exists
masks = glob.glob(
os.path.dirname(insrc) + os.sep + 'MASKS' + os.sep +
'*BINARY_MASK' + ext)
if len(masks) != 0:
for mask in masks:
srcClip = os.path.join(pathWd, 'srcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication(
{
"in": mask,
"mode": "fit",
"mode.fit.im": inref,
"out": srcClip,
"pixType": "uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' +
outSensorModel)
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalMask = os.path.join(
pathWd,
os.basename(
mask.replace(ext, ext.replace('.',
'_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication(
{
"inr": srcClip,
"inm": orthoRecApp[0],
"out": finalMask,
"pixType": "uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(
finalMask,
mask.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalMask.replace(ext, '.geom'),
mask.replace(ext, '_COREG.geom'))
if writeFeatures == False and os.path.exists(outSensorModel):
os.remove(outSensorModel)
if datatype in ['S2', 'S2_S2C']:
mtd_file = glob.glob(
os.path.join(os.path.dirname(insrc), '*_MTD_ALL*'))[0]
cloud_clear = get_S2_Tile_Cloud_Cover(mtd_file)
cover = get_S2_Tile_Coverage(mtd_file)
if cloud_clear > 0.6 and cover > 0.8:
clean_dates.append(date)
elif datatype in ['L5', 'L8']:
mlt_file = glob.glob(
os.path.join(os.path.dirname(insrc), '*_MTL*'))[0]
cloud_clear = get_L8_Tile_Cloud_Cover(mlt_file)
if cloud_clear > 0.6:
clean_dates.append(date)
if not writeFeatures and os.path.exists(SensorModel):
os.remove(SensorModel)
# mode 2 : application on the time series
elif mode == 2:
ext = os.path.splitext(insrc)[1]
file_list = glob.glob(datadir + os.sep + '*' + os.sep + pattern)
for insrc in file_list:
srcClip = os.path.join(pathWd, 'tempSrcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication({
"in":
insrc,
"mode":
"fit",
"mode.fit.im":
inref,
"out":
srcClip,
"pixType":
"uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' + SensorModel)
ds = gdal.Open(srcClip)
prj = ds.GetProjection()
gt = ds.GetGeoTransform()
srs = osr.SpatialReference()
srs.ImportFromWkt(prj)
code = srs.GetAuthorityCode(None)
gsp = str(int(2 * round(max(abs(gt[1]), abs(gt[5])))))
ds = None
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalOutput = os.path.join(
pathWd,
os.path.basename(
insrc.replace(ext, ext.replace('.', '_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication({
"inr":
srcClip,
"inm":
orthoRecApp[0],
"out":
finalOutput,
"pixType":
"uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(finalOutput,
insrc.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalOutput.replace(ext, '.geom'),
insrc.replace(ext, '_COREG.geom'))
# Mask registration if exists
masks = glob.glob(
os.path.dirname(insrc) + os.sep + 'MASKS' + os.sep +
'*BINARY_MASK*' + ext)
if len(masks) != 0:
for mask in masks:
srcClip = os.path.join(pathWd, 'tempSrcClip.tif')
extractROIApp = OtbAppBank.CreateExtractROIApplication({
"in":
mask,
"mode":
"fit",
"mode.fit.im":
inref,
"out":
srcClip,
"pixType":
"uint16"
})
extractROIApp.ExecuteAndWriteOutput()
outSrc = os.path.join(pathWd, 'temp_file.tif')
io_Src = str(srcClip + '?&skipcarto=true&geom=' +
SensorModel)
orthoRecApp = OtbAppBank.CreateOrthoRectification({
"in":
io_Src,
"io.out":
outSrc,
"map":
"epsg",
"map.epsg.code":
code,
"opt.gridspacing":
gsp,
"pixType":
"uint16"
})
if writeFeatures:
orthoRecApp[0].ExecuteAndWriteOutput()
else:
orthoRecApp[0].Execute()
ext = os.path.splitext(insrc)[1]
finalMask = os.path.join(
pathWd,
os.path.basename(
mask.replace(ext, ext.replace('.', '_COREG.'))))
superImposeApp = OtbAppBank.CreateSuperimposeApplication({
"inr":
srcClip,
"inm":
orthoRecApp[0],
"out":
finalMask,
"pixType":
"uint16"
})
superImposeApp[0].ExecuteAndWriteOutput()
shutil.move(finalMask,
mask.replace(ext, ext.replace('.', '_COREG.')))
shutil.move(finalMask.replace(ext, '.geom'),
mask.replace(ext, '_COREG.geom'))
os.remove(srcClip)
if not writeFeatures and os.path.exists(SensorModel):
os.remove(SensorModel)
return None