def createTilesFromMasks(inputImage, tilesBase, tilesMetaDIR, tilesImgDIR,
datatype, gdalformat):
"""
A function to apply the image tile masks defined in createTileMaskImages to the input image to extract the individual tiles.
Where:
:param inputImage: is the input image being tiled.
:param tileMasksBase: is the base path for the tile masks. glob will be used to find them with \*.kea added to the end.
:param outTilesBase: is the base file name for the tiles.
"""
maskFiles = glob.glob(os.path.join(tilesMetaDIR, tilesBase + "*.kea"))
idx = 1
for maskFile in maskFiles:
tileImage = os.path.join(tilesImgDIR, os.path.basename(maskFile))
imageutils.maskImage(inputImage, maskFile, tileImage, gdalformat,
datatype, 0, 0)
imageutils.popImageStats(tileImage, True, 0., True)
default='Float32',
help="Data type")
args = parser.parse_args()
# Get output extension from file
outFormat = getGDALFormat(args.outstack)
imageList = []
bandNamesList = []
fileList = os.listdir(args.indir)
replaceFileStrList = args.search.split('*')
for fName in fileList:
if fnmatch.fnmatch(fName, args.search): # Match search string
imageList.append(os.path.join(args.indir, fName))
for replaceFileStr in replaceFileStrList:
bandName = fName.replace(replaceFileStr, '')
bandNamesList.append(bandName)
gdalformat = 'KEA'
dataType = rsgislib.TYPE_32FLOAT
# Stack bands
imageutils.stackImageBands(imageList, bandNamesList, args.outstack, None, 0,
'KEA', getRSGISLibDataType(args.datatype))
# Calculate stats
imageutils.popImageStats(args.outstack, True, 0., True)
if (d >= 2) & (d <= 6): # select dry season images based on image month
print('')
print('Finished...')
print('')
else:
print('')
print('Refinement needed.')
print('')
#mask the original stacked S1 image using open water prediction
gdalformat='KEA'
datatype=rsgislib.TYPE_32FLOAT
imgMask=outimage
outImg = inputImage.replace('.tif','_wb_mask.kea')
rsgislib.imageutils.maskImage(inputImage, imgMask, outImg, gdalformat, datatype, 0, [2,3]) # mask out dry or wetVeg pixels
imageutils.popImageStats(outImg, True, 0.0, True)
#segment image and add stats from S1 image
inImg=outImg # in image based on masked radar image
clumps=inImg.replace('.kea','_clumps2.kea')
clumpsMean=outImg.replace('.kea','_clumps2_mean.kea')
segutils.runShepherdSegmentation(inImg, clumps, clumpsMean, minPxls=100, numClusters=5)
bandList=['VV','VH','VVdivVH']
rsgislib.imageutils.setBandNames(inImg, bandList)
####################################################################
# extract otsu threshold from VV band
ds1 = gdal.Open(clumpsMean)
maskVV = np.array(ds1.GetRasterBand(1).ReadAsArray())
maskVV=maskVV[maskVV<0]
threshold=filters.threshold_otsu(maskVV)
def ClassifyImage(InputImage, OutputImage, GDALformat, SpectralClasses,
SampleSize):
''' A function to classify an image in blocks.'''
# Define the classifier
clf = MiniBatchKMeans(n_clusters=SpectralClasses,
init='k-means++',
max_iter=20,
batch_size=1000,
verbose=0,
compute_labels=True,
random_state=None,
tol=0.0,
max_no_improvement=100,
init_size=10000,
n_init=10,
reassignment_ratio=0.05)
print('Performing K-means unsupervised classification with ' +
str(SpectralClasses) + ' spectral classes...')
# Read the input image:
InputRaster = gdal.Open(InputImage, gdal.GA_ReadOnly)
RasterBands = InputRaster.RasterCount
SRS = InputRaster.GetProjection()
GeoT = InputRaster.GetGeoTransform()
SizeX = InputRaster.RasterXSize
SizeY = InputRaster.RasterYSize
BlockSize = InputRaster.GetRasterBand(1).GetBlockSize()
NoDataValue = InputRaster.GetRasterBand(1).GetNoDataValue()
# Get the extent of each image block
RasterBlocks = GetImageBlocks(SizeX, SizeY, BlockSize[0], BlockSize[1])
n_blocks = len(RasterBlocks)
TrainingData = []
# iterate over each raster block and obtain a sample of pixels for classifier training
print('Extracting training data from each image block...')
for idx, Block in enumerate(RasterBlocks):
BlockData = []
# iterate over each raster band
for Band in range(RasterBands):
Band += 1
BandData = InputRaster.GetRasterBand(Band).ReadAsArray(
int(Block[0]), int(Block[1]), int(Block[2]), int(Block[3]))
BandData = numpy.ma.compressed(
numpy.ma.masked_equal(BandData, NoDataValue))
if len(BandData) != 0:
BlockData.append(BandData)
del BandData
if len(BlockData) != 0:
Pixels = int(len(BlockData[0]))
Sample = int(Pixels * SampleSize)
BlockData = numpy.array(BlockData).T
BlockData = BlockData[numpy.random.choice(
BlockData.shape[0], size=Sample,
replace=False), :] # Sample without replacement.
TrainingData.append(BlockData)
del BlockData
ProgressBar(n_blocks - 1, idx)
# fit the training data
TrainingData = numpy.concatenate(TrainingData)
print('Training the classifier using ' + str(len(TrainingData)) +
' pixels...')
clf.fit(TrainingData, y=None)
del TrainingData
# Create output raster
Driver = gdal.GetDriverByName(GDALformat)
OutputRaster = Driver.Create(OutputImage, SizeX, SizeY, 1, 1)
OutputRaster.SetProjection(SRS)
OutputRaster.SetGeoTransform(GeoT)
OutBand = OutputRaster.GetRasterBand(1)
OutBand.SetNoDataValue(0)
# Read the input image in blocks and perform classification
print('Classifying ' + str(len(RasterBlocks)) + ' blocks...')
for idx, Block in enumerate(RasterBlocks):
TestData = []
for Band in range(RasterBands):
Band += 1
BandData = InputRaster.GetRasterBand(Band).ReadAsArray(
int(Block[0]), int(Block[1]), int(Block[2]), int(Block[3]))
if Band == 1: # Create a binary mask for
BinaryMask = numpy.ones_like(BandData)
BinaryMask = numpy.where(BandData == NoDataValue, 0,
BinaryMask)
TestData.append(BandData.flatten())
del BandData
# Perform classification
TestData = numpy.array(TestData).T
PredClass = clf.predict(
TestData) + 1 # Add 1 to avoid having a class value = 0.
del TestData
PredClass = numpy.reshape(PredClass, (int(Block[3]), int(Block[2])))
PredClass = PredClass * BinaryMask # Reclassify nodata regions to zero using Binary Mask.
del BinaryMask
# write classification to the output raster
OutBand.WriteArray(PredClass, int(Block[0]), int(Block[1]))
del PredClass
ProgressBar(n_blocks - 1, idx)
del RasterBlocks
# Close the input and output rasters
InputRaster, OutputRaster = None, None
# Build image overviews for faster viewing in external software (e.g. QGIS or Tuiview)
print('Generating image overviews...')
imageutils.popImageStats(OutputImage, True, 0, True)
default=False,
help=
"Don't calculate statistics and pyramids for mosaic (default is to calculate)"
)
args = parser.parse_args()
file_list = []
for zip_file in args.inputfiles:
file_list.extend(zip_to_gdal_path(zip_file))
if len(file_list) == 0:
print('No ".asc" found within zip file(s) provided as input',
file=sys.stderr)
print('\nCreating mosaic...')
imageutils.createImageMosaic(
file_list, args.outmosaic, OUT_NODATA, SOURCE_NODATA, 1, 0,
rsgislib.RSGISPyUtils().getGDALFormatFromExt(args.outmosaic),
rsgislib.TYPE_32FLOAT)
# Assign Projection
print('\nAssigning projection')
imageutils.assignProj(args.outmosaic, wktString=OSGB_WKT_STRING)
if not args.nostats:
# Create pyramids
print('\nCalculating stats and pyramids...')
imageutils.popImageStats(args.outmosaic, True, 0., True)
print('Finished')
def processSingleFile(inputFile, outputDIR, tmpath, calcExtraBands, palsar2=False):
inputFile = os.path.abspath(inputFile)
outputDIR = os.path.abspath(outputDIR)
tmpath = os.path.abspath(tmpath)
print("Processing: " + inputFile)
baseName = os.path.basename(inputFile).split(".")[0]
print("\t" + baseName)
rsgisUtils = rsgislib.RSGISPyUtils()
uidStr = "_"+rsgisUtils.uidGenerator()
createdTmp = False
if not os.path.exists(tmpath):
os.makedirs(tmpath)
createdTmp = True
extract2DIR = os.path.join(tmpath, baseName+uidStr)
if not os.path.exists(extract2DIR):
os.makedirs(extract2DIR)
os.chdir(extract2DIR)
cmd = 'tar -xzf ' + inputFile
print(cmd)
subprocess.call(cmd, shell=True)
try:
if palsar2:
hhFiles = glob.glob(os.path.join(extract2DIR, HH_P2_FILE_PATTERN))
hvFiles = glob.glob(os.path.join(extract2DIR, HV_P2_FILE_PATTERN))
if len(hhFiles) == 0:
hhFiles = glob.glob(os.path.join(extract2DIR, HH_P2_FP_FILE_PATTERN))
if len(hvFiles) == 0:
hvFiles = glob.glob(os.path.join(extract2DIR, HV_P2_FP_FILE_PATTERN))
in_hh_file = hhFiles[0]
in_hv_file = hvFiles[0]
else:
in_hh_file = glob.glob(os.path.join(extract2DIR, HH_P1_FILE_PATTERN))[0]
in_hv_file = glob.glob(os.path.join(extract2DIR, HV_P1_FILE_PATTERN))[0]
except IndexError:
raise Exception('Could not find data - check filenames')
bands_list = [in_hh_file, in_hv_file]
band_names = ['HH','HV']
# Create extra image bands
for calcBand in calcExtraBands:
if calcBand == 'COVARHH':
extraBandFile = os.path.join(extract2DIR, baseName + '_covhh.kea')
imagefilter.applyCoeffOfVarFilter(in_hh_file, extraBandFile, 5, 'KEA', rsgislib.TYPE_32FLOAT)
bandName = 'CoVHH'
bands_list.append(extraBandFile)
band_names.append(bandName)
if calcBand == 'COVARHV':
extraBandFile = os.path.join(extract2DIR, baseName + '_covhv.kea')
imagefilter.applyCoeffOfVarFilter(in_hv_file, extraBandFile, 5, 'KEA', rsgislib.TYPE_32FLOAT)
bandName = 'CoVHV'
bands_list.append(extraBandFile)
band_names.append(bandName)
if calcBand == 'HHHV':
extraBandFile = os.path.join(extract2DIR, baseName + '_hhhv.kea')
bandDefns = [imagecalc.BandDefn('hh', in_hh_file, 1),
imagecalc.BandDefn('hv', in_hv_file, 1)]
imagecalc.bandMath(extraBandFile, 'hv==0?0:hh/hv', 'KEA', rsgislib.TYPE_32FLOAT, bandDefns)
bandName = 'HH/HV'
bands_list.append(extraBandFile)
band_names.append(bandName)
# Create stack
stackFile = os.path.join(outputDIR, baseName + '_stack.kea')
imageutils.stackImageBands(bands_list, band_names, stackFile, None, 0, 'KEA', rsgislib.TYPE_32FLOAT)
imageutils.popImageStats(stackFile, usenodataval=True, nodataval=0, calcpyramids=True)
try:
in_mask_file = glob.glob(os.path.join(extract2DIR, MASK_FILE_PATTERN))[0]
out_mask_file = os.path.join(outputDIR, baseName + '_mask.kea')
cmd = 'gdal_translate -of KEA ' + in_mask_file + ' ' + out_mask_file
subprocess.call(cmd, shell=True)
rastergis.populateStats(out_mask_file, True, True)
except IndexError:
print("WARNING: Could not find the mask file... Ignoring.")
try:
in_date_file = glob.glob(os.path.join(extract2DIR, DATE_FILE_PATTERN))[0]
out_date_file = os.path.join(outputDIR, baseName + '_date.kea')
cmd = 'gdal_translate -of KEA ' + in_date_file + ' ' + out_date_file
subprocess.call(cmd, shell=True)
rastergis.populateStats(out_date_file, True, True)
except IndexError:
print("WARNING: Could not find the date file... Ignoring.")
try:
in_linci_file = glob.glob(os.path.join(extract2DIR, LINCI_FILE_PATTERN))[0]
out_linci_file = os.path.join(outputDIR, baseName + '_linci.kea')
cmd = 'gdal_translate -of KEA ' + in_linci_file + ' ' + out_linci_file
subprocess.call(cmd, shell=True)
rastergis.populateStats(out_linci_file, True, True)
except IndexError:
print("WARNING: Could not find the linci file... Ignoring.")
shutil.rmtree(extract2DIR)
if createdTmp:
shutil.rmtree(tmpath)
c=contextvect,
i=inMOLAFile.replace("TOSHIBA EXT",
"TOSHIBA\ EXT"),
out=outDTM)
print(gdwarpcmd)
subprocess.call(gdwarpcmd, shell=True)
if not (MGSmode):
skipLyrSt = False
try:
imageutils.subset(inLyrSt, contextvect, outLyrSt,
'KEA', rsgislib.TYPE_32FLOAT)
imageutils.selectImageBands(outLyrSt, outLyrSt2, 'KEA',
rsgislib.TYPE_32FLOAT,
[1, 3, 2])
imageutils.popImageStats(outLyrSt, True, 0, True)
imageutils.popImageStats(outLyrSt2, True, 0, True)
except:
input("subset not working")
gdwarpcmd = "gdalwarp -of KEA -cutline {c} -crop_to_cutline {i} {out}".format(
c=contextvect,
i=inLyrSt.replace("TOSHIBA EXT", "TOSHIBA\ EXT"),
out=outLyrSt)
print(gdwarpcmd)
subprocess.call(gdwarpcmd, shell=True)
skipLyrSt = True
else:
skipLyrSt = True
# I had some problems with this so used gdal_rasterize
#vectorutils.rasterise2Image(contextvect,outND4, outCTXrast, 'GTiff', 'FID')
for fileName in fList:
if fnmatch.fnmatch(fileName, args.search): # Match search string
fileList.append(os.path.join(dName, fileName))
else:
fileList = args.inputimages
fileCount=len(fileList)
if fileCount == 0:
print('ERROR: No files found')
sys.exit()
else:
print('Found %i files'%fileCount)
# Save list of files
if args.outlist is not None:
rsgisUtils.writeList2File(fileList, args.outlist)
print('Creating mosaic...')
t = rsgislib.RSGISTime()
t.start(True)
imageutils.createImageMosaic(fileList, args.outmosaic, args.backgroundval, args.skipval, args.skipband, overlapBehaviour, outFormat, rsgisUtils.getRSGISLibDataType(args.datatype))
t.end()
if not args.nostats:
print('\nCalculating stats and pyramids...')
t.start(True)
imageutils.popImageStats(args.outmosaic, True, args.backgroundval, True)
t.end()
print('rsgismosaic.py - Finished')
import rsgislib, glob
from rsgislib import imageutils
listFiles=glob.glob('*20180101T082329_20180101T084222_T34LGH.tif')
#inputImage='Fractions_and_Class_MARCH_2017_18.tif'
for inputImage in listFiles:
imageutils.popImageStats(inputImage, False,0,True)
inputImg = '/Users/Andy/Documents/Zambia/RemoteSensing/Sentinel_1/GRD/Out/Subset/S1B_IW_GRDH_1SDV_20170704T165718_Sigma0_stack_lee3.kea'
outputClumps = '/Users/Andy/Documents/Zambia/RemoteSensing/Sentinel_1/GRD/Out/Subset/S1B_IW_GRDH_1SDV_20170704T165718_Sigma0_stack_lee3_clumps2.kea'
outputMeanImg = inputImg.split('.')[0] + '_clumps_mean.' + inputImg.split(
'.')[1]
#
#
# run segmentation
#
#
print('Performing the segmentation...')
segutils.runShepherdSegmentation(inputImg,
outputClumps,
outputMeanImg,
minPxls=100)
imageutils.popImageStats(outputClumps, True, 0., True)
# populate RAT with mean stats from S1
clumps = outputClumps # rename clumps image
ratutils.populateImageStats(inputImg, clumps, calcMean=True, calcStDev=True)
# populate clumps with training data
print('Populating clumps with stats...')
classesDict = dict()
classesDict['Water'] = [
1,
'/Users/Andy/Documents/Zambia/RemoteSensing/WB_classification/Supporting_data/global_surface_water/watermask_barotseland_m18dB_20170704.shp'
]
tmpPath = './temp'
classesIntCol = 'ClassInt'
classesNameCol = 'ClassStr'
ratutils.populateClumpsWithClassTraining(clumps, classesDict, tmpPath,
print('DTM requested - classifying ground returns')
print(' Running Progressive Morphology Filter')
pmfCMD = ['spdpmfgrd',
'-r','50',
'--overlap','10',
'--maxfilter','14',
'-i',spdfile,'-o',spdfile_grd]
subprocess.call(pmfCMD)
print('Creating DTM')
dtmCMD = ['spdinterp','--dtm','--topo',
'--in',args.interpolation,
'-f',args.of,'-b','1','-i',spdfile_grd,'-o',outdtm]
subprocess.call(dtmCMD)
if haveRSGISLib and args.of == 'KEA':
imageutils.popImageStats(outdtm,True,0.,True)
if args.hillshade:
print('Creating DTM Hillshade')
hillshadeCMD = ['gdaldem','hillshade','-of',args.of,
outdtm, outdtm_hillshade]
subprocess.call(hillshadeCMD)
if haveRSGISLib and args.of == 'KEA':
imageutils.popImageStats(outdtm_hillshade,True,0.,True)
if args.dsm:
if not args.dtm:
spdfile_grd = spdfile
print('Creating DSM')
dsmCMD = ['spdinterp','--dsm','--topo',