def createStage3ImageSubsets(self, inputImage, s2BordersImage, s3BordersClumps, subsetImgsDIR, subsetImgsMaskedDIR, subImgBaseName, minSize):
segmentation.clump(s2BordersImage, s3BordersClumps, 'KEA', True, 0)
rastergis.populateStats(s3BordersClumps, True, True)
rastergis.spatialExtent(s3BordersClumps, 'minXX', 'minXY', 'maxXX', 'maxXY', 'minYX', 'minYY', 'maxYX', 'maxYY')
rsgisUtils = rsgislib.RSGISPyUtils()
dataType = rsgisUtils.getRSGISLibDataTypeFromImg(inputImage)
ratDS = gdal.Open(s3BordersClumps, gdal.GA_Update)
minX = rat.readColumn(ratDS, "minXX")
maxX = rat.readColumn(ratDS, "maxXX")
minY = rat.readColumn(ratDS, "minYY")
maxY = rat.readColumn(ratDS, "maxYY")
Histogram = rat.readColumn(ratDS, "Histogram")
for i in range(minX.shape[0]):
if i > 0:
subImage = os.path.join(subsetImgsDIR, subImgBaseName + str(i) + '.kea')
#print( "[" + str(minX[i]) + ", " + str(maxX[i]) + "][" + str(minY[i]) + ", " + str(maxY[i]) + "]" )
imageutils.subsetbbox(inputImage, subImage, 'KEA', dataType, minX[i], maxX[i], minY[i], maxY[i])
if Histogram[i] > minSize:
maskedFile = os.path.join(subsetImgsMaskedDIR, subImgBaseName + str(i) + '_masked.kea')
else:
maskedFile = os.path.join(subsetImgsMaskedDIR, subImgBaseName + str(i) + '_burn.kea')
imageutils.maskImage(subImage, s2BordersImage, maskedFile, 'KEA', dataType, 0, 0)
rastergis.populateStats(maskedFile, True, False)
ratDS = None
def mergeStage3TilesToOutput(self, clumpsImage, subsetSegsDIR, subsetImgsMaskedDIR, subImgBaseName):
burnTiles = glob.glob(os.path.join(subsetImgsMaskedDIR, subImgBaseName+"*_burn.kea"))
if len(burnTiles) > 0:
segmentation.mergeClumpImages(burnTiles, clumpsImage)
segTiles = glob.glob(os.path.join(subsetSegsDIR, subImgBaseName+"*_segs.kea"))
segmentation.mergeClumpImages(segTiles, clumpsImage)
rastergis.populateStats(clumpsImage, True, True)
def mergeStage2TilesToOutput(self, clumpsImage, tilesSegsDIR, tilesSegBordersDIR, tilesBase, s2BordersImage):
segTiles = glob.glob(os.path.join(tilesSegsDIR, tilesBase+"*_segs.kea"))
segmentation.mergeClumpImages(segTiles, clumpsImage)
rastergis.populateStats(clumpsImage, True, True)
tileBorders = glob.glob(os.path.join(tilesSegBordersDIR, tilesBase+"*_segsborder.kea"))
imageutils.createCopyImage(clumpsImage, s2BordersImage, 1, 0, 'KEA', rsgislib.TYPE_8UINT)
imageutils.includeImages(s2BordersImage, tileBorders)
def attribute_segments(listInputs):
dataStack = listInputs[0]# data with values
habFile = listInputs[1] # data with habitat codes
clumpsFile = listInputs[2] # segments
# Populate with stats, so it can be read for the RAT
rastergis.populateStats(dataStack)
rastergis.populateStats(clumpsFile)
rastergis.populateStats(habFile)
ratutils.populateImageStats(dataStack, clumpsFile, calcMean=True)
# Convert training data to RAT!!!
codeStats = [] #list()
codeStats.append(rastergis.BandAttStats(band=1, minField='Class'))
rastergis.populateRATWithStats(habFile, habFile, codeStats)
# Attribute segments with class: yes, so I can compare later !!
rastergis.strClassMajority(clumpsFile, habFile, 'Class', 'Class', False)
rsgislib.imageutils.setBandNames(outputImage, bandList)
inputImage=outputImage # define lee image as input for rest of the script
##############################################################################################
# Shepherd segmentation - this output will be used in second script to run classifier
##############################################################################################
print('Performing the segmentation...')
outputClumps=inputImage.replace('.tif','_clumps2.kea')
outputMeanImg=inputImage.replace('.tif','_clumps2_mean.kea')
segutils.runShepherdSegmentation(inputImage, outputClumps, outputMeanImg, minPxls=100)
bandList=['VV','VH','VVdivVH']
rsgislib.imageutils.setBandNames(outputMeanImg, bandList)
rastergis.populateStats(outputClumps, True, True, True, 1)
rastergis.populateStats(outputMeanImg, True, True, True, 1)
clumps=outputClumps # rename clumps image
ratutils.populateImageStats(inputImage, clumps, calcMin=True,calcMax=True,calcMean=True, calcStDev=True) # add radar stats
##############################################################################################
# add global urban footprint stats
gufSnap=guf.split('.')[0]+'_'+inputImage.split('/')[-1].split('_')[4]+'.tif'
inRefImg=clumps # base raster to snap to
gdalFormat = 'GTiff'
rsgislib.imageutils.resampleImage2Match(inRefImg, guf, gufSnap, gdalFormat, interpMethod='nearestneighbour', datatype=None) # perform resampling/snap
bandList=['guf']
rsgislib.imageutils.setBandNames(gufSnap, bandList)
ratutils.populateImageStats(gufSnap, clumps, calcMax=True) # add urban footprint stats
# Convert array to uint32 format
Array = Array.astype('uint32')
# assign unique integers to non-zero pixels
NewVals = numpy.arange(1, ValidPixels + 1, 1, dtype='uint32')
numpy.place(Array, MaskArray, NewVals)
del NewVals
del MaskArray
# Create output raster
print("Creating raster...")
gdalformat = 'KEA'
datatype = gdal.GDT_UInt32
driver = gdal.GetDriverByName(gdalformat)
metadata = driver.GetMetadata()
output = driver.Create(OutImage, X_Size, Y_Size, 1, datatype)
output.SetProjection(Projection)
output.SetGeoTransform(GeoTransform)
outband = output.GetRasterBand(1)
outband.WriteArray(Array)
# Close datasets
output = None
del Array
# Add a raster attribute table
rastergis.populateStats(OutImage, True, True, True, 1)
print("Done.")
outKEAFile = inSHPFile.replace('.shp', '_raster.' + outExt)
# Check if KEA file exists or needs creating - assume if it exists it has the attribute table already
if os.path.exists(outKEAFile) == False:
# Rasterize polygon using gdal
print('Creating raster')
rasterizeCommand = ['gdal_rasterize', '-of', outFormat,
'-ot', 'UInt32',
'-tr', outXRes, outYRes,
'-a', 'MUKEY',
inSHPFile, outKEAFile]
subprocess.call(rasterizeCommand)
# Convert to RAT and add entry 'mukey' using RSGISLib
rastergis.populateStats(outKEAFile, True, True)
stats2Calc = [rastergis.BandAttStats(band=1, minField='mukey')]
rastergis.populateRATWithStats(outKEAFile, outKEAFile, stats2Calc)
# Iterate through column names
for outColName in args.colname:
# Set up name for standard raster file
outRasterImage = inSHPFile.replace('.shp', '_raster_' + outColName + '.' + outExt)
# Get field from dictionary
outColNum = cHorizonFields[outColName.strip()]
# JOIN ATTRIBUTES FROM TEXT FILE
print('Adding ' + outColName + ' (column ' + str(outColNum) + ') to RAT')
# Open SSURGO text files
componentFileName = os.path.join(inDIRName, 'tabular', 'comp.txt')
def processSingleFile(inputFile, outputDIR, tmpath, calcExtraBands, palsar2=False):
inputFile = os.path.abspath(inputFile)
outputDIR = os.path.abspath(outputDIR)
tmpath = os.path.abspath(tmpath)
print("Processing: " + inputFile)
baseName = os.path.basename(inputFile).split(".")[0]
print("\t" + baseName)
rsgisUtils = rsgislib.RSGISPyUtils()
uidStr = "_"+rsgisUtils.uidGenerator()
createdTmp = False
if not os.path.exists(tmpath):
os.makedirs(tmpath)
createdTmp = True
extract2DIR = os.path.join(tmpath, baseName+uidStr)
if not os.path.exists(extract2DIR):
os.makedirs(extract2DIR)
os.chdir(extract2DIR)
cmd = 'tar -xzf ' + inputFile
print(cmd)
subprocess.call(cmd, shell=True)
try:
if palsar2:
hhFiles = glob.glob(os.path.join(extract2DIR, HH_P2_FILE_PATTERN))
hvFiles = glob.glob(os.path.join(extract2DIR, HV_P2_FILE_PATTERN))
if len(hhFiles) == 0:
hhFiles = glob.glob(os.path.join(extract2DIR, HH_P2_FP_FILE_PATTERN))
if len(hvFiles) == 0:
hvFiles = glob.glob(os.path.join(extract2DIR, HV_P2_FP_FILE_PATTERN))
in_hh_file = hhFiles[0]
in_hv_file = hvFiles[0]
else:
in_hh_file = glob.glob(os.path.join(extract2DIR, HH_P1_FILE_PATTERN))[0]
in_hv_file = glob.glob(os.path.join(extract2DIR, HV_P1_FILE_PATTERN))[0]
except IndexError:
raise Exception('Could not find data - check filenames')
bands_list = [in_hh_file, in_hv_file]
band_names = ['HH','HV']
# Create extra image bands
for calcBand in calcExtraBands:
if calcBand == 'COVARHH':
extraBandFile = os.path.join(extract2DIR, baseName + '_covhh.kea')
imagefilter.applyCoeffOfVarFilter(in_hh_file, extraBandFile, 5, 'KEA', rsgislib.TYPE_32FLOAT)
bandName = 'CoVHH'
bands_list.append(extraBandFile)
band_names.append(bandName)
if calcBand == 'COVARHV':
extraBandFile = os.path.join(extract2DIR, baseName + '_covhv.kea')
imagefilter.applyCoeffOfVarFilter(in_hv_file, extraBandFile, 5, 'KEA', rsgislib.TYPE_32FLOAT)
bandName = 'CoVHV'
bands_list.append(extraBandFile)
band_names.append(bandName)
if calcBand == 'HHHV':
extraBandFile = os.path.join(extract2DIR, baseName + '_hhhv.kea')
bandDefns = [imagecalc.BandDefn('hh', in_hh_file, 1),
imagecalc.BandDefn('hv', in_hv_file, 1)]
imagecalc.bandMath(extraBandFile, 'hv==0?0:hh/hv', 'KEA', rsgislib.TYPE_32FLOAT, bandDefns)
bandName = 'HH/HV'
bands_list.append(extraBandFile)
band_names.append(bandName)
# Create stack
stackFile = os.path.join(outputDIR, baseName + '_stack.kea')
imageutils.stackImageBands(bands_list, band_names, stackFile, None, 0, 'KEA', rsgislib.TYPE_32FLOAT)
imageutils.popImageStats(stackFile, usenodataval=True, nodataval=0, calcpyramids=True)
try:
in_mask_file = glob.glob(os.path.join(extract2DIR, MASK_FILE_PATTERN))[0]
out_mask_file = os.path.join(outputDIR, baseName + '_mask.kea')
cmd = 'gdal_translate -of KEA ' + in_mask_file + ' ' + out_mask_file
subprocess.call(cmd, shell=True)
rastergis.populateStats(out_mask_file, True, True)
except IndexError:
print("WARNING: Could not find the mask file... Ignoring.")
try:
in_date_file = glob.glob(os.path.join(extract2DIR, DATE_FILE_PATTERN))[0]
out_date_file = os.path.join(outputDIR, baseName + '_date.kea')
cmd = 'gdal_translate -of KEA ' + in_date_file + ' ' + out_date_file
subprocess.call(cmd, shell=True)
rastergis.populateStats(out_date_file, True, True)
except IndexError:
print("WARNING: Could not find the date file... Ignoring.")
try:
in_linci_file = glob.glob(os.path.join(extract2DIR, LINCI_FILE_PATTERN))[0]
out_linci_file = os.path.join(outputDIR, baseName + '_linci.kea')
cmd = 'gdal_translate -of KEA ' + in_linci_file + ' ' + out_linci_file
subprocess.call(cmd, shell=True)
rastergis.populateStats(out_linci_file, True, True)
except IndexError:
print("WARNING: Could not find the linci file... Ignoring.")
shutil.rmtree(extract2DIR)
if createdTmp:
shutil.rmtree(tmpath)
def colour_SM_image(inimage, colourimage, max_value=0.5, band=1):
"""
Colour image
"""
try:
import rsgislib
from rsgislib import imagecalc
from rsgislib.imagecalc import BandDefn
from rsgislib import rastergis
except ImportError:
raise ImportError("Could not import RSGISLib, required"
" to colour image")
if max_value == 0.5:
# Add class field:
bandDefns = []
bandDefns.append(BandDefn('SM', inimage, band))
expression = []
expression.append('0.00')
expression.append('(SM > 0.00) && (SM <= 0.05)? 1 : 0')
expression.append('(SM > 0.05) && (SM <= 0.10)? 2 : 0')
expression.append('(SM > 0.10) && (SM <= 0.15)? 3 : 0')
expression.append('(SM > 0.15) && (SM <= 0.20)? 4 : 0')
expression.append('(SM > 0.20) && (SM <= 0.25)? 5 : 0')
expression.append('(SM > 0.25) && (SM <= 0.30)? 6 : 0')
expression.append('(SM > 0.30) && (SM <= 0.35)? 7 : 0')
expression.append('(SM > 0.35) && (SM <= 0.40)? 8 : 0')
expression.append('(SM > 0.40) && (SM <= 0.45)? 9 : 0')
expression.append('(SM > 0.45) && (SM <= 0.50)? 10 : 0')
gdalformat = get_gdal_format(colourimage)
datatype = rsgislib.TYPE_8INT
temp_dir = tempfile.mkdtemp(prefix='soilscape_upscaling')
colourbase = os.path.basename(colourimage)
colourbase, colourext = os.path.splitext(colourbase)
colour = []
for i in range(11):
colour.append(
os.path.join(temp_dir,
"{}_{}{}".format(colourbase, i + 1, colourext)))
imagecalc.bandMath(colour[i], expression[i], gdalformat, datatype,
bandDefns)
bandDefns = []
for i in range(11):
SMclass = 'SM' + str(i)
bandDefns.append(BandDefn(SMclass, colour[i], 1))
if i == 0:
expression = SMclass
else:
expression = expression + '+' + SMclass
imagecalc.bandMath(colourimage, expression, gdalformat, datatype,
bandDefns)
shutil.rmtree(temp_dir)
# Populate stats (converts to RAT):
rastergis.populateStats(colourimage, False, False, True, ratband=1)
# Add class field:
bandStats = []
bandStats.append(rastergis.BandAttStats(band=1, maxField="SMclass"))
rastergis.populateRATWithStats(colourimage,
colourimage,
bandStats,
ratband=1)
field = 'SMclass'
classcolours = {}
colourCat = collections.namedtuple('ColourCat',
['red', 'green', 'blue', 'alpha'])
for i in range(11):
classcolours[i] = colourCat(red=0, green=0, blue=0, alpha=255)
classcolours[0] = colourCat(red=0, green=0, blue=0, alpha=255)
classcolours[1] = colourCat(red=165, green=0, blue=38, alpha=255)
classcolours[2] = colourCat(red=215, green=48, blue=39, alpha=255)
classcolours[3] = colourCat(red=244, green=109, blue=67, alpha=255)
classcolours[4] = colourCat(red=253, green=174, blue=97, alpha=255)
classcolours[5] = colourCat(red=254, green=224, blue=144, alpha=255)
classcolours[6] = colourCat(red=224, green=243, blue=248, alpha=255)
classcolours[7] = colourCat(red=171, green=217, blue=233, alpha=255)
classcolours[8] = colourCat(red=116, green=173, blue=209, alpha=255)
classcolours[9] = colourCat(red=69, green=117, blue=180, alpha=255)
classcolours[10] = colourCat(red=49, green=54, blue=149, alpha=255)
rastergis.colourClasses(colourimage, field, classcolours)
else:
raise ValueError('Failed to find colours for specified max_value')
#outputMeanImg = inputImage.split('.')[0]+'_clust5_clumps2_mean.kea'
#segutils.runShepherdSegmentation(inputImage, outputClumps, outputMeanImg, minPxls=100, numClusters=5)
gdalformat='KEA'
rsgislib.segmentation.generateRegularGrid(inputImage, clumps, gdalformat, numXPxls, numYPxls)
# set output band names
#bandList=['VV','VH','VVdivVH']
#rsgislib.imageutils.setBandNames(inputImage, bandList)
#rsgislib.imageutils.setBandNames(outputMeanImg, bandList)
# rastergis.populateStats(outputMeanImg, True, True, True, 1)
# populate RAT with mean stats from S1
#rsgiscalcimgstats.py -t -n 0 -i /Users/Andy/Documents/Zambia/RemoteSensing/Sentinel_1/GRD/Out/Subset/2017/S1B_IW_GRDH_1SDV_20170318T165713_Sigma0_stack_lee_chboard.kea
rastergis.populateStats(clumps, addclrtab=clrTab, calcpyramids=pyramids, ignorezero=ignoreZeros)
ratutils.populateImageStats(inputImage, clumps, calcMean=True)
##############################################################################################
##############################################################################################
# snapping global water product to SAR image
##############################################################################################
print('Snapping watermask...')
#waterSnap=globalWater.split('.')[0]+'_'+inputImage.split('_')[-2]+'_new.tif'
waterSnap=globalWater.split('.')[0]+'_'+inputImage.split('_')[-4]+'_clump.tif'
inRefImg=inputImage # base raster to snap to
stan_rast_fn,
segments_fn,
tmpath=temp_fn,
gdalformat='KEA',
noStretch=True, # Issue when this is false
noStats=True, # Issue when this is false
numClusters=60,
minPxls=25,
distThres=10,
sampling=100,
kmMaxIter=200)
# ------------------------------------------------- #
# Image statistics
print('Calculating image statistics...')
# Spectral stats
rastergis.populateStats(segments_fn)
ratutils.populateImageStats(
stan_rast_fn,
segments_fn,
calcMean=True,
calcMin=True,
calcMax=True,
calcStDev=True) # populates clumps file with stats
# reading in RAT to add to
segment_rat = gdal.Open(
segments_fn, gdal.GA_Update) # Read in segments raster as gdal
segment_ds = segment_rat.GetRasterBand(
1).GetDefaultRAT() # Read in RAT
band = segment_rat.GetRasterBand(
1).ReadAsArray() # read in segment raster band, with seg ids
band_data = np.dstack(band)
def createMinDataTiles(inputImage,
outshp,
outclumpsFile,
width,
height,
validDataThreshold,
maskIntersect=None,
offset=False,
force=True,
tmpdir='tilestemp',
inImgNoDataVal=0.0):
"""
A function to create a tiling for an input image where each tile has a minimum amount of valid data.
Where:
:param inputImage: is a string for the image to be tiled
:param outshp: is a string for the output shapefile the tiling will be written to (if None a shapefile won't be outputted).
:param outclumpsFile: is a string for the output image file containing the tiling
:param width: is an int for the width of the tiles
:param height: is an int for the height of the tiles
:param validDataThreshold: is a float (0-1) with the proportion of valid data needed within a tile.
:param force: is a boolean (default True) to delete the output shapefile if it already exists.
:param tmpdir: is a string with a temporary directory for temp outputs to be stored (they will be deleted).
if tmpdir doesn't exist it will be created and then deleted during the processing.
:param inImgNoDataVal: is a float for providing the input image no data value (Default: 0.0)
"""
tmpPresent = True
if not os.path.exists(tmpdir):
print("WARNING: tmpdir directory does not exist so creating it...")
os.makedirs(tmpdir)
tmpPresent = False
inImgBaseName = os.path.splitext(os.path.basename(inputImage))[0]
tileClumpsImage = os.path.join(tmpdir, inImgBaseName + '_tilesimg.kea')
segmentation.generateRegularGrid(inputImage, tileClumpsImage, 'KEA', width,
height, offset)
rastergis.populateStats(tileClumpsImage, True, True)
if not maskIntersect == None:
bs = []
bs.append(rastergis.BandAttStats(band=1, maxField='Mask'))
rastergis.populateRATWithStats(maskIntersect, tileClumpsImage, bs)
ratDS = gdal.Open(tileClumpsImage, gdal.GA_Update)
MaskField = rat.readColumn(ratDS, "Mask")
Selected = numpy.zeros_like(MaskField, dtype=int)
Selected[MaskField == inImgNoDataVal] = 1
rat.writeColumn(ratDS, "Selected", Selected)
ratDS = None
tileClumpsImageDropClumps = os.path.join(
tmpdir, inImgBaseName + '_tilesimgdropped.kea')
segmentation.dropSelectedClumps(tileClumpsImage,
tileClumpsImageDropClumps, 'KEA',
'Selected')
os.remove(tileClumpsImage)
tileClumpsImage = tileClumpsImageDropClumps
rastergis.populateRATWithPropValidPxls(inputImage, tileClumpsImage,
"ValidPxls", inImgNoDataVal)
ratDS = gdal.Open(tileClumpsImage, gdal.GA_Update)
ValidPxls = rat.readColumn(ratDS, "ValidPxls")
Selected = numpy.zeros_like(ValidPxls, dtype=int)
NoDataClumps = numpy.zeros_like(ValidPxls, dtype=int)
Selected[ValidPxls < validDataThreshold] = 1
NoDataClumps[ValidPxls == 0] = 1
Selected[ValidPxls == 0] = 0
rat.writeColumn(ratDS, "Selected", Selected)
rat.writeColumn(ratDS, "NoDataClumps", NoDataClumps)
ratDS = None
segmentation.mergeSegments2Neighbours(tileClumpsImage, inputImage,
outclumpsFile, 'KEA', "Selected",
"NoDataClumps")
if not outshp is None:
tilesDS = gdal.Open(outclumpsFile, gdal.GA_ReadOnly)
tilesDSBand = tilesDS.GetRasterBand(1)
dst_layername = os.path.splitext(os.path.basename(outshp))[0]
#print(dst_layername)
drv = ogr.GetDriverByName("ESRI Shapefile")
if force and os.path.exists(outshp):
drv.DeleteDataSource(outshp)
dst_ds = drv.CreateDataSource(outshp)
dst_layer = dst_ds.CreateLayer(dst_layername, srs=None)
gdal.Polygonize(tilesDSBand,
tilesDSBand,
dst_layer,
-1, [],
callback=None)
tilesDS = None
dst_ds = None
if not tmpPresent:
shutil.rmtree(tmpdir, ignore_errors=True)
else:
os.remove(tileClumpsImage)
