def readROIFromVector(vector, roiprefix, *args):
"""
**********
Parameters
----------
vector : str
Vector path ('myFolder/class.shp',str).
roiprefix : str
Common suffixof the training shpfile (i.e. 'band_',str).
*args : str
Field name containing your class number (i.e. 'class', str).
Output
----------
ROIvalues : array
*args : array
"""
file = ogr.Open(vector)
lyr = file.GetLayer()
roiFields = []
# get all fields and save only roiFields
ldefn = lyr.GetLayerDefn()
listFields = []
for n in range(ldefn.GetFieldCount()):
fdefn = ldefn.GetFieldDefn(n)
if not fdefn.name is listFields:
listFields.append(fdefn.name)
if fdefn.name.startswith(roiprefix):
roiFields.append(fdefn.name)
if len(roiFields) > 0:
# fill ROI and level
ROIvalues = np.zeros([lyr.GetFeatureCount(), len(roiFields)])
if len(args) > 0:
ROIlevels = np.zeros([lyr.GetFeatureCount(), len(args)])
for i, feature in enumerate(lyr):
for j, band in enumerate(roiFields):
ROIvalues[i, j] = feature.GetField(band)
if len(args) > 0:
for a in range(len(args)):
ROIlevels[i, a] = feature.GetField(args[a])
if len(args) > 0:
return ROIvalues, ROIlevels
else:
return ROIvalues
else:
from mainfunction import pushFeedback
pushFeedback(
'ROI field "{}" do not exists. These fields are available : '.
format(roiprefix))
pushFeedback(listFields)
def resampleWithSameDateAsSource(sourceImage,
targetImage,
sourceDates,
targetDates,
nSpectralBands,
resampledImage,
feedback=None):
pushFeedback(1, feedback=feedback)
tempDir = os.path.dirname(resampledImage) + '/tmp'
if not os.path.exists(tempDir):
os.makedirs(tempDir)
#RefDates = sp.loadtxt("/mnt/Data_2/Formosat/RAW/formosat_SudouestKalideos_2010/sample_time.csv")
#ChangeDates = sp.loadtxt("/mnt/Data_2/Formosat/RAW/formosat_SudouestKalideos_2012/sample_time.csv")
sourceDatesArr = np.loadtxt(sourceDates)
targetDatesArr = np.loadtxt(targetDates)
nTargetBands = len(targetDatesArr)
nSourceBands = len(sourceDatesArr)
DOY = []
for i in (sourceDatesArr, targetDatesArr):
DOY.append(convertToDateTime(i, '%Y%m%d', DOY=True))
sourceDOY = np.asarray(DOY[0])
targetDOY = np.asarray(DOY[1])
# createEmptyMaskDates = [d for d in sourceDOY if(d not in(targetDOY))]
combineDOY = np.unique(np.sort(np.concatenate((sourceDOY, targetDOY))))
sourceDOYidx = np.searchsorted(combineDOY, sourceDOY)
# needmask if 1, 0 means image is in source Image
needMask = np.ones(combineDOY.shape)
needMask[sourceDOYidx] = 0
# get target dtype for OTB computation
dataTarget = gdal.Open(targetImage)
dataTargetBand = dataTarget.GetRasterBand(1)
targetDataType = dataraster.convertGdalDataTypeToOTB(
dataTargetBand.DataType)
# create doy vrt using false image
dataSource = gdal.Open(sourceImage)
GeoTransform = dataSource.GetGeoTransform()
Projection = dataSource.GetProjection()
#im = dataTarget.GetRasterBand(1).ReadAsArray()
im = np.zeros([dataSource.RasterYSize, dataSource.RasterXSize])
dataraster.create_uniquevalue_tiff(tempDir + '/mask1.tif', im, 1,
GeoTransform, Projection, 1,
dataTargetBand.DataType)
dataraster.create_uniquevalue_tiff(tempDir + '/mask0.tif', im, 1,
GeoTransform, Projection, 0,
dataTargetBand.DataType)
for i in combineDOY[needMask == 1]:
for spectral in range(nSpectralBands):
bashCommand = "gdalbuildvrt " + str(tempDir) + '/temp_' + str(
spectral +
1) + "_" + str(i) + ".tif " + str(tempDir) + "/mask1.tif"
bashCommandMask = "gdalbuildvrt " + str(tempDir) + '/temp_' + str(
spectral +
1) + "_" + str(i) + "_mask.tif " + str(tempDir) + "/mask1.tif"
os.system(bashCommand)
os.system(bashCommandMask)
# create doy vrt using real image
for i, j in enumerate(sourceDOY):
for spectral in range(nSpectralBands):
bandToKeep = (spectral * nSourceBands) + int(i) + 1
bashCommand = ("gdalbuildvrt -b " + str(bandToKeep) + ' ' +
str(tempDir) + '/temp_' + str(spectral + 1) + '_' +
str(j) + ".tif " + sourceImage)
bashCommandMask = ("gdalbuildvrt " + str(tempDir) + '/temp_' +
str(spectral + 1) + '_' + str(j) +
"_mask.tif " + str(tempDir) + '/mask0.tif')
os.system(bashCommand)
os.system(bashCommandMask)
pushFeedback(10, feedback=feedback)
"""
for i in convertToDateTime(createEmptyMaskDates):
date = i.strftime('%Y%m%d')
#bashCommand = "gdalbuildvrt "+str(WDIR)+"/mask1.tif"
bashCommand = ("gdalbuildvrt "+str(WDIR)+'temp_'+str(date)+".tif "+str(WDIR)+"mask1.tif ")*4+' -separate'
bashCommandMask = "gdalbuildvrt "+str(WDIR)+'temp_'+str(date)+".nuages.tif "+str(WDIR)+"mask1.tif "
os.system(bashCommand)
os.system(bashCommandMask)
"""
# os.remove(WDIR+'mask1.tif')
#nDOY = (datetime.date(2013,1,1) - datetime.date(2012, 1, 1)).days
srcDate = convertToDateTime(sourceDatesArr)[0]
sourceYEAR = srcDate.year
def DOYtoDates(doys, sourceYEAR, convertToInt=False):
dates = []
for i in doys:
currentDate = datetime.date(sourceYEAR, 1,
1) + datetime.timedelta(int(i) - 1)
if convertToInt:
currentDate = currentDate.strftime('%Y%m%d')
dates.append(currentDate)
return dates
newDates = np.unique(DOYtoDates(combineDOY, sourceYEAR, convertToInt=True))
np.savetxt(tempDir + '/sampleTime.csv',
np.asarray(newDates, dtype=int),
fmt='%d')
"""
tokeep=''
initDOY=0
for k in range(4):
for i in DOY[0]:
tokeep=tokeep+'-b '+str(initDOY+i)+' '
initDOY+=nDOY
bashCommand='gdalbuildvrt '+tokeep+"SITS_2012_temp.tif "+WDIR+"SITS_2010.tif "
os.system(bashCommand)
bashCommand='gdal_translate SITS_2012_temp.tif '+WDIR+'NODTW_SameDatesAsRef/SITS_2012.tif && rm SITS_2012_temp.tif'
os.system(bashCommand)
bashCommand='cp /mnt/Data_2/Formosat/SITS_2/SITS_2010.tif '+WDIR+'NODTW_SameDatesAsRef/SITS_2010.tif'
os.system(bashCommand)
"""
# os.remove(tempDir+'/mask0.tif')
# os.remove(tempDir+'/mask1.tif')
#print DOY
sourceDOYidx = np.unique(np.searchsorted(combineDOY, targetDOY))
#bandsToKeepInVrt = listToStr(sourceDOYidx+1,sep=' -b ')
total = 60 / nSpectralBands
for spectral in range(nSpectralBands):
try:
if feedback.isCanceled():
break
except BaseException:
pass
pushFeedback(10 + total * (spectral + 1), feedback=feedback)
pushFeedback('Gap-filling timeseries (' + str(spectral + 1) + '/' +
str(nSpectralBands) + ')',
feedback=feedback)
toGapFill = glob.glob(tempDir + '/*_' + str(spectral + 1) +
'_*[0-9].tif')
toGapFillMask = glob.glob(tempDir + '/*_' + str(spectral + 1) +
'_*[0-9]_mask.tif')
vrt = 'gdalbuildvrt -separate ' + tempDir + \
'/temp.vrt' + listToStr(sorted(toGapFill))
os.system(vrt)
vrtmask = 'gdalbuildvrt -separate ' + tempDir + \
'/temp_mask.vrt' + listToStr(sorted(toGapFillMask))
os.system(vrtmask)
bashCommand = (
"otbcli_ImageTimeSeriesGapFilling -in {0} -mask {1} -out {2} {4} -comp 1 -it linear -id {3}"
).format(tempDir + '/temp.vrt', tempDir + '/temp_mask.vrt',
tempDir + '/temp_' + str(spectral + 1) + '.tif',
tempDir + '/sampleTime.csv', targetDataType)
pushFeedback('Executing gap filling : ' + bashCommand,
feedback=feedback)
os.system(bashCommand)
os.remove(tempDir + '/temp.vrt')
os.remove(tempDir + '/temp_mask.vrt')
tempList = []
for i, j in enumerate(sourceDOYidx):
currentVrt = (tempDir + '/band_temp_{}_{}.tif').format(
str(spectral + 1), i)
tempList.append(currentVrt)
vrt = ('gdalbuildvrt -b {0} {1} ' +
tempDir + '/temp_{2}.tif').format(j + 1, currentVrt,
str(spectral + 1))
os.system(vrt)
vrt = 'gdalbuildvrt -separate ' + tempDir + '/temp_' + \
str(spectral + 1) + '.vrt' + listToStr(tempList)
os.system(vrt)
bandList = [
tempDir + '/temp_' + str(x + 1) + '.vrt' for x in range(nSpectralBands)
]
# if outfolder not exists, create
conca = 'otbcli_ConcatenateImages -il {} -out {} {}'.format(
listToStr(bandList), resampledImage, targetDataType)
pushFeedback(80, feedback=feedback)
pushFeedback('Executing image concatenation : ' + conca, feedback=feedback)
os.system(conca)
files = glob.glob(tempDir + '/*')
for file in files:
os.remove(file)
os.removedirs(tempDir)
return resampledImage
def findBestParameters(self, Xs, ys, Xt, yt):
self.bestScore = None
for gridOT in self.generateParamForGridSearch():
self.transportModel = self.transportFunction(**gridOT)
self.transportModel.fit(Xs, ys, Xt, yt)
#XsTransformed = self.transportModel.transform(Xs)
#XsPredict = self.inverseTransform(XsTransformed)
from ot.da import BaseTransport
transp_Xt = BaseTransport.inverse_transform(
self.transportModel, Xs=Xs, ys=ys, Xt=Xt, yt=yt)
if self.feedback:
pushFeedback(
'Testing params : ' + str(gridOT),
feedback=self.feedback)
"""
#score = mean_squared_error(Xs,XsPredict)
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV
param_grid = dict(gamma=2.0**np.arange(-4,1), C=10.0**np.arange(-2,3))
classifier = SVC(probability=False)
cv = StratifiedKFold(n_splits=5)
grid = GridSearchCV(classifier,param_grid=param_grid, cv=cv,n_jobs=1)
# need to rescale for hyperparameter of svm
if self.scaler is False:
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(-1,1))
scaler.fit(Xs,ys)
Xs = scaler.transform(Xs)
XsPredict = scaler.transform(XsPredict)
#XsPredict = scaler.transform(XsPredict)
grid.fit(Xs,ys)
model = grid.best_estimator_
model.fit(Xs,ys)
yp = model.predict(XsPredict)
currentScore = dict(OA=accuracy_score(yp,ys),Kappa=cohen_kappa_score(yp,ys),F1=f1_score(yp,ys,average='micro'))
if self.feedback:
pushFeedback('Kappa is : '+str(currentScore.get('Kappa')))
if self.bestScore is None or self.bestScore.get('Kappa') < currentScore.get('Kappa'):
self.bestScore = currentScore.copy()
self.bestParam = gridOT.copy()
"""
currentScore = mean_squared_error(Xs, transp_Xt)
if self.feedback:
pushFeedback(
'RMSE is : ' + str(currentScore),
feedback=self.feedback)
if self.bestScore is None or self.bestScore > currentScore:
self.bestScore = currentScore
self.bestParam = gridOT.copy()
"""
del self.transportModel,yp
"""
if self.feedback:
pushFeedback('Best grid is ' +
str(self.bestParam), feedback=self.feedback)
pushFeedback('Best score is ' +
str(self.bestScore), feedback=self.feedback)
def learnTransfer(self, Xs, ys, Xt, yt=None):
"""
Learn domain adaptation model.
Parameters
----------
Xs : array_like, shape (n_source_samples, n_features)
Source domain array.
ys : array_like, shape (n_source_samples,)
Label source array (1d).
Xt: array_like, shape (n_source_samples, n_features)
Target domain array.
yt: array_like, shape (n_source_samples,)
Label target array (1d).
Returns
-------
transportmodel : object
The output model
"""
# save original samples
self.Xs_ = Xs
self.Xt_ = Xt
self.params = self.params_
if self.feedback:
pushFeedback(10, feedback=self.feedback)
pushFeedback('Learning Optimal Transport with ' +
str(self.transportAlgorithm) +
' algorithm.', feedback=self.feedback)
# check if label is 1d
if ys is not None:
if len(ys.shape) > 1:
ys = ys[:, 0]
if yt is not None:
if len(yt.shape) > 1:
yt = yt[:, 0]
# rescale Data
if self.scaler:
self.scaler.fit(Xs, ys)
self.scalerTarget.fit(Xt, yt)
Xs = self.scaler.transform(Xs)
Xt = self.scalerTarget.transform(Xt)
# import Domain Adaptation specific algorithm function from OT Library
self.transportFunction = getattr(
__import__("ot").da, self.transportAlgorithm)
if self.params is None:
self.transportModel = self.transportFunction()
else:
# order for reproductibility
self.params = sorted(self.params.items())
# if grid search
if self.isGridSearch():
# compute combinaison for each param
self.findBestParameters(Xs, ys=ys, Xt=Xt, yt=yt)
self.transportModel = self.transportFunction(**self.bestParam)
else:
# simply train with basic param
self.transportModel = self.transportFunction(**self.params_)
self.transportModel.fit(Xs, ys=ys, Xt=Xt, yt=yt)
if self.feedback:
pushFeedback(20, feedback=self.feedback)
return self.transportModel
def predictTransfer(self, imageSource, outRaster, mask=None,
NODATA=-9999, feedback=None, norm=False):
"""
Predict model using domain adaptation.
Parameters
----------
model : object
Model generated from learnTransfer function.
imageSource : str
Path of image to adapt (source image)
outRaster : str
Path of tiff image to save as.
mask: str, optional
Path of raster mask.
NODATA : int, optional
Default -9999
feedback : object, optional
For Qgis Processing. Default is None.
Returns
-------
outRaster : str
Return the path of the predicted image.
"""
if self.feedback:
pushFeedback('Now transporting ' +
str(os.path.basename(imageSource)))
dataSrc = gdal.Open(imageSource)
# Get the size of the image
d = dataSrc.RasterCount
nc = dataSrc.RasterXSize
nl = dataSrc.RasterYSize
# Get the geoinformation
GeoTransform = dataSrc.GetGeoTransform()
Projection = dataSrc.GetProjection()
# Get block size
band = dataSrc.GetRasterBand(1)
block_sizes = band.GetBlockSize()
x_block_size = block_sizes[0]
y_block_size = block_sizes[1]
#gdal_dt = band.DataType
# Initialize the output
driver = gdal.GetDriverByName('GTiff')
dst_ds = driver.Create(outRaster, nc, nl, d, 3)
dst_ds.SetGeoTransform(GeoTransform)
dst_ds.SetProjection(Projection)
del band
# Perform the classification
if mask is not None:
maskData = gdal.Open(mask, gdal.GA_ReadOnly)
total = nl * y_block_size
total = 80 / (int(nl / y_block_size))
for i in range(0, nl, y_block_size):
# feedback for Qgis
if self.feedback:
pushFeedback(int(i * total) + 20, feedback=self.feedback)
try:
if self.feedback.isCanceled():
break
except BaseException:
pass
if i + y_block_size < nl: # Check for size consistency in Y
lines = y_block_size
else:
lines = nl - i
for j in range(
0, nc, x_block_size): # Check for size consistency in X
if j + x_block_size < nc:
cols = x_block_size
else:
cols = nc - j
# Load the data and Do the prediction
X = np.empty((cols * lines, d))
for ind in range(d):
X[:, ind] = dataSrc.GetRasterBand(
int(ind + 1)).ReadAsArray(j, i, cols, lines).reshape(cols * lines)
# Do the prediction
if mask is None:
mask_temp = dataSrc.GetRasterBand(1).ReadAsArray(
j, i, cols, lines).reshape(cols * lines)
else:
mask_temp = maskData.GetRasterBand(1).ReadAsArray(
j, i, cols, lines).reshape(cols * lines)
# check if nodata
t = np.where((mask_temp != 0) & (X[:, 0] != NODATA))[0]
# transform array, default has nodata value
yp = np.empty((cols * lines, d))
yp[:, :] = NODATA
# yp = np.nan((cols*lines,d))
# K = np.zeros((cols*lines,))
# TODO: Change this part accorindgly ...
# if t.size > 0:
if t.size > 0:
tempOT = X[t, :]
yp[t, :] = self.transportModel.transform(tempOT)
for ind in range(d):
out = dst_ds.GetRasterBand(ind + 1)
# Write the data
ypTemp = yp[:, ind]
out.WriteArray(ypTemp.reshape(lines, cols), j, i)
out.SetNoDataValue(NODATA)
out.FlushCache()
del X, yp
return outRaster
def learnTransfer(self,Xs,ys,Xt,yt=None):
"""
Learn domain adaptation model.
Parameters
----------
Xs : array_like, shape (n_source_samples, n_features)
Source domain array.
ys : array_like, shape (n_source_samples,)
Label source array (1d).
Xt: array_like, shape (n_source_samples, n_features)
Target domain array.
yt: array_like, shape (n_source_samples,)
Label target array (1d).
Returns
-------
transportmodel : object
The output model
"""
# save original samples
self.Xs_ = Xs
self.Xt_ = Xt
self.params = self.params_
if self.feedback:
pushFeedback(10,feedback=self.feedback)
pushFeedback('Learning Optimal Transport with '+str(self.transportAlgorithm)+' algorithm.',feedback=self.feedback)
# check if label is 1d
if ys is not None:
if len(ys.shape)>1:
ys = ys[:,0]
if yt is not None:
if len(yt.shape)>1:
yt = yt[:,0]
# rescale Data
if self.scaler:
self.scaler.fit(Xs,ys)
self.scalerTarget.fit(Xt,yt)
Xs = self.scaler.transform(Xs)
Xt = self.scalerTarget.transform(Xt)
# import Domain Adaptation specific algorithm function from OT Library
self.transportFunction = getattr(__import__("ot").da,self.transportAlgorithm)
if self.params is None:
self.transportModel = self.transportFunction()
else:
# order for reproductibility
self.params = sorted(self.params.items())
# if grid search
if self.isGridSearch():
# compute combinaison for each param
self.findBestParameters(Xs,ys=ys,Xt=Xt,yt=yt)
self.transportModel = self.transportFunction(**self.bestParam)
else:
# simply train with basic param
self.transportModel = self.transportFunction(**self.params_)
self.transportModel.fit(Xs,ys=ys,Xt=Xt,yt=yt)
if self.feedback:
pushFeedback(20,feedback=self.feedback)
return self.transportModel
def findBestParameters(self,Xs,ys,Xt,yt):
self.bestScore = None
for gridOT in self.generateParamForGridSearch():
self.transportModel = self.transportFunction(**gridOT)
self.transportModel.fit(Xs,ys,Xt,yt)
#XsTransformed = self.transportModel.transform(Xs)
#XsPredict = self.inverseTransform(XsTransformed)
from ot.da import BaseTransport
transp_Xt = BaseTransport.inverse_transform(self.transportModel,Xs=Xs,ys=ys,Xt=Xt,yt=yt)
if self.feedback:
pushFeedback('Testing params : '+str(gridOT),feedback=self.feedback)
"""
#score = mean_squared_error(Xs,XsPredict)
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV
param_grid = dict(gamma=2.0**np.arange(-4,1), C=10.0**np.arange(-2,3))
classifier = SVC(probability=False)
cv = StratifiedKFold(n_splits=5)
grid = GridSearchCV(classifier,param_grid=param_grid, cv=cv,n_jobs=1)
# need to rescale for hyperparameter of svm
if self.scaler is False:
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(-1,1))
scaler.fit(Xs,ys)
Xs = scaler.transform(Xs)
XsPredict = scaler.transform(XsPredict)
#XsPredict = scaler.transform(XsPredict)
grid.fit(Xs,ys)
model = grid.best_estimator_
model.fit(Xs,ys)
yp = model.predict(XsPredict)
currentScore = dict(OA=accuracy_score(yp,ys),Kappa=cohen_kappa_score(yp,ys),F1=f1_score(yp,ys,average='micro'))
if self.feedback:
pushFeedback('Kappa is : '+str(currentScore.get('Kappa')))
if self.bestScore is None or self.bestScore.get('Kappa') < currentScore.get('Kappa'):
self.bestScore = currentScore.copy()
self.bestParam = gridOT.copy()
"""
currentScore = mean_squared_error(Xs,transp_Xt)
if self.feedback:
pushFeedback('RMSE is : '+str(currentScore),feedback=self.feedback)
if self.bestScore is None or self.bestScore > currentScore:
self.bestScore = currentScore
self.bestParam = gridOT.copy()
"""
del self.transportModel,yp
"""
if self.feedback:
pushFeedback('Best grid is '+str(self.bestParam),feedback=self.feedback)
pushFeedback('Best score is '+str(self.bestScore),feedback=self.feedback)
def predictTransfer(self,imageSource,outRaster,mask=None,NODATA=-9999,feedback=None,norm=False):
"""
Predict model using domain adaptation.
Parameters
----------
model : object
Model generated from learnTransfer function.
imageSource : str
Path of image to adapt (source image)
outRaster : str
Path of tiff image to save as.
mask: str, optional
Path of raster mask.
NODATA : int, optional
Default -9999
feedback : object, optional
For Qgis Processing. Default is None.
Returns
-------
outRaster : str
Return the path of the predicted image.
"""
if self.feedback:
pushFeedback('Now transporting '+str(os.path.basename(imageSource)))
dataSrc = gdal.Open(imageSource)
# Get the size of the image
d = dataSrc.RasterCount
nc = dataSrc.RasterXSize
nl = dataSrc.RasterYSize
# Get the geoinformation
GeoTransform = dataSrc.GetGeoTransform()
Projection = dataSrc.GetProjection()
# Get block size
band = dataSrc.GetRasterBand(1)
block_sizes = band.GetBlockSize()
x_block_size = block_sizes[0]
y_block_size = block_sizes[1]
#gdal_dt = band.DataType
## Initialize the output
driver = gdal.GetDriverByName('GTiff')
dst_ds = driver.Create(outRaster, nc,nl, d, 3)
dst_ds.SetGeoTransform(GeoTransform)
dst_ds.SetProjection(Projection)
del band
## Perform the classification
if mask is not None:
maskData = gdal.Open(mask,gdal.GA_ReadOnly)
total = nl*y_block_size
total = 80/(int(nl/y_block_size))
for i in range(0,nl,y_block_size):
# feedback for Qgis
if self.feedback:
pushFeedback(int(i * total)+20,feedback=self.feedback)
try:
if self.feedback.isCanceled():
break
except:
pass
if i + y_block_size < nl: # Check for size consistency in Y
lines = y_block_size
else:
lines = nl - i
for j in range(0,nc,x_block_size): # Check for size consistency in X
if j + x_block_size < nc:
cols = x_block_size
else:
cols = nc - j
# Load the data and Do the prediction
X = np.empty((cols*lines,d))
for ind in range(d):
X[:,ind] = dataSrc.GetRasterBand(int(ind+1)).ReadAsArray(j, i, cols, lines).reshape(cols*lines)
# Do the prediction
if mask is None:
mask_temp = dataSrc.GetRasterBand(1).ReadAsArray(j, i, cols, lines).reshape(cols*lines)
else:
mask_temp = maskData.GetRasterBand(1).ReadAsArray(j, i, cols, lines).reshape(cols*lines)
# check if nodata
t = np.where((mask_temp!=0) & (X[:,0]!=NODATA))[0]
# transform array, default has nodata value
yp = np.empty((cols*lines,d))
yp[:,:] = NODATA
# yp = np.nan((cols*lines,d))
# K = np.zeros((cols*lines,))
# TODO: Change this part accorindgly ...
#if t.size > 0:
if t.size > 0 :
tempOT = X[t,:]
yp[t,:] = self.transportModel.transform(tempOT)
for ind in range(d):
out = dst_ds.GetRasterBand(ind+1)
# Write the data
ypTemp = yp[:,ind]
out.WriteArray(ypTemp.reshape(lines,cols),j,i)
out.SetNoDataValue(NODATA)
out.FlushCache()
del X,yp
return outRaster
def resampleWithSameDateAsSource(sourceImage,targetImage,sourceDates,targetDates,nSpectralBands,resampledImage,feedback=None):
pushFeedback(1,feedback=feedback)
tempDir = os.path.dirname(resampledImage)+'/tmp'
if not os.path.exists(tempDir):
os.makedirs(tempDir)
#RefDates = sp.loadtxt("/mnt/Data_2/Formosat/RAW/formosat_SudouestKalideos_2010/sample_time.csv")
#ChangeDates = sp.loadtxt("/mnt/Data_2/Formosat/RAW/formosat_SudouestKalideos_2012/sample_time.csv")
sourceDatesArr = np.loadtxt(sourceDates)
targetDatesArr = np.loadtxt(targetDates)
nTargetBands = len(targetDatesArr)
nSourceBands = len(sourceDatesArr)
DOY = []
for i in (sourceDatesArr,targetDatesArr):
DOY.append(convertToDateTime(i,'%Y%m%d',DOY=True))
sourceDOY = np.asarray(DOY[0])
targetDOY = np.asarray(DOY[1])
# createEmptyMaskDates = [d for d in sourceDOY if(d not in(targetDOY))]
combineDOY = np.unique(np.sort(np.concatenate((sourceDOY,targetDOY))))
sourceDOYidx = np.searchsorted(combineDOY,sourceDOY)
# needmask if 1, 0 means image is in source Image
needMask = np.ones(combineDOY.shape)
needMask[sourceDOYidx] = 0
# get target dtype for OTB computation
dataTarget = gdal.Open(targetImage)
dataTargetBand = dataTarget.GetRasterBand(1)
targetDataType = dataraster.convertGdalDataTypeToOTB(dataTargetBand.DataType)
# create doy vrt using false image
dataSource = gdal.Open(sourceImage)
GeoTransform = dataSource.GetGeoTransform()
Projection = dataSource.GetProjection()
#im = dataTarget.GetRasterBand(1).ReadAsArray()
im = np.zeros([dataSource.RasterYSize,dataSource.RasterXSize])
dataraster.create_uniquevalue_tiff(tempDir+'/mask1.tif',im,1,GeoTransform,Projection,1,dataTargetBand.DataType)
dataraster.create_uniquevalue_tiff(tempDir+'/mask0.tif',im,1,GeoTransform,Projection,0,dataTargetBand.DataType)
for i in combineDOY[needMask==1]:
for spectral in range(nSpectralBands):
bashCommand = "gdalbuildvrt "+str(tempDir)+'/temp_'+str(spectral+1)+"_"+str(i)+".tif "+str(tempDir)+"/mask1.tif"
bashCommandMask = "gdalbuildvrt "+str(tempDir)+'/temp_'+str(spectral+1)+"_"+str(i)+"_mask.tif "+str(tempDir)+"/mask1.tif"
os.system(bashCommand)
os.system(bashCommandMask)
# create doy vrt using real image
for i,j in enumerate(sourceDOY):
for spectral in range(nSpectralBands):
bandToKeep = (spectral*nSourceBands)+int(i)+1
bashCommand = ("gdalbuildvrt -b "+str(bandToKeep)+' '+str(tempDir)+'/temp_'+str(spectral+1)+'_'+str(j)+".tif "+sourceImage)
bashCommandMask = ("gdalbuildvrt "+str(tempDir)+'/temp_'+str(spectral+1)+'_'+str(j)+"_mask.tif "+str(tempDir)+'/mask0.tif')
os.system(bashCommand)
os.system(bashCommandMask)
pushFeedback(10,feedback=feedback)
"""
for i in convertToDateTime(createEmptyMaskDates):
date = i.strftime('%Y%m%d')
#bashCommand = "gdalbuildvrt "+str(WDIR)+"/mask1.tif"
bashCommand = ("gdalbuildvrt "+str(WDIR)+'temp_'+str(date)+".tif "+str(WDIR)+"mask1.tif ")*4+' -separate'
bashCommandMask = "gdalbuildvrt "+str(WDIR)+'temp_'+str(date)+".nuages.tif "+str(WDIR)+"mask1.tif "
os.system(bashCommand)
os.system(bashCommandMask)
"""
#os.remove(WDIR+'mask1.tif')
#nDOY = (datetime.date(2013,1,1) - datetime.date(2012, 1, 1)).days
srcDate=convertToDateTime(sourceDatesArr)[0]
sourceYEAR = srcDate.year
def DOYtoDates(doys,sourceYEAR,convertToInt=False):
dates = []
for i in doys:
currentDate = datetime.date(sourceYEAR, 1, 1) + datetime.timedelta(int(i)-1)
if convertToInt:
currentDate = currentDate.strftime('%Y%m%d')
dates.append(currentDate)
return dates
newDates = np.unique(DOYtoDates(combineDOY,sourceYEAR,convertToInt=True))
np.savetxt(tempDir+'/sampleTime.csv',np.asarray(newDates,dtype=int),fmt='%d')
"""
tokeep=''
initDOY=0
for k in range(4):
for i in DOY[0]:
tokeep=tokeep+'-b '+str(initDOY+i)+' '
initDOY+=nDOY
bashCommand='gdalbuildvrt '+tokeep+"SITS_2012_temp.tif "+WDIR+"SITS_2010.tif "
os.system(bashCommand)
bashCommand='gdal_translate SITS_2012_temp.tif '+WDIR+'NODTW_SameDatesAsRef/SITS_2012.tif && rm SITS_2012_temp.tif'
os.system(bashCommand)
bashCommand='cp /mnt/Data_2/Formosat/SITS_2/SITS_2010.tif '+WDIR+'NODTW_SameDatesAsRef/SITS_2010.tif'
os.system(bashCommand)
"""
#os.remove(tempDir+'/mask0.tif')
#os.remove(tempDir+'/mask1.tif')
#print DOY
sourceDOYidx = np.unique(np.searchsorted(combineDOY,targetDOY))
#bandsToKeepInVrt = listToStr(sourceDOYidx+1,sep=' -b ')
total = 60/nSpectralBands
for spectral in range(nSpectralBands):
try:
if feedback.isCanceled():
break
except:
pass
pushFeedback(10+total*(spectral+1),feedback=feedback)
pushFeedback('Gap-filling timeseries ('+str(spectral+1)+'/'+str(nSpectralBands)+')',feedback=feedback)
toGapFill = glob.glob(tempDir+'/*_'+str(spectral+1)+'_*[0-9].tif')
toGapFillMask = glob.glob(tempDir+'/*_'+str(spectral+1)+'_*[0-9]_mask.tif')
vrt = 'gdalbuildvrt -separate '+tempDir+'/temp.vrt'+listToStr(sorted(toGapFill))
os.system(vrt)
vrtmask = 'gdalbuildvrt -separate '+tempDir+'/temp_mask.vrt'+listToStr(sorted(toGapFillMask))
os.system(vrtmask)
bashCommand = ("otbcli_ImageTimeSeriesGapFilling -in {0} -mask {1} -out {2} {4} -comp 1 -it linear -id {3}").format(tempDir+'/temp.vrt',tempDir+'/temp_mask.vrt',tempDir+'/temp_'+str(spectral+1)+'.tif',tempDir+'/sampleTime.csv',targetDataType)
pushFeedback('Executing gap filling : '+bashCommand,feedback=feedback)
os.system(bashCommand)
os.remove(tempDir+'/temp.vrt')
os.remove(tempDir+'/temp_mask.vrt')
tempList = []
for i,j in enumerate(sourceDOYidx):
currentVrt = (tempDir+'/band_temp_{}_{}.tif').format(str(spectral+1),i)
tempList.append(currentVrt)
vrt = ('gdalbuildvrt -b {0} {1} '+tempDir+'/temp_{2}.tif').format(j+1,currentVrt,str(spectral+1))
os.system(vrt)
vrt = 'gdalbuildvrt -separate '+tempDir+'/temp_'+str(spectral+1)+'.vrt'+listToStr(tempList)
os.system(vrt)
bandList = [tempDir+'/temp_'+str(x+1)+'.vrt' for x in range(nSpectralBands)]
# if outfolder not exists, create
conca = 'otbcli_ConcatenateImages -il {} -out {} {}'.format(listToStr(bandList),resampledImage,targetDataType)
pushFeedback(80,feedback=feedback)
pushFeedback('Executing image concatenation : '+conca,feedback=feedback)
os.system(conca)
files = glob.glob(tempDir+'/*')
for file in files:
os.remove(file)
os.removedirs(tempDir)
return resampledImage