def calcImgBasicStats4RefRegion(ref_img,
stats_imgs,
output_img,
gdalformat='KEA'):
"""
A function which calculates the mean and standard deviation through a series of
input images. The region for processing is defined by the reference image and
images padded with no-data where no data is present.
The output image has twice the number of bands as the input image providing
a mean and standard deviation for each input band.
If the input images has 2 bands then the output bands will have the following
order:
1. band 1 mean
2. band 1 std dev
3. band 2 mean
4. band 2 std dev
:param ref_img: reference image which defines the output image
:param stats_imgs: a list of input images over which the stats will be calculated.
:param output_img: the output image path and file name
:param gdalformat: the output image file format. Default KEA.
"""
import rsgislib.imageutils
from rios import applier
rsgis_utils = rsgislib.RSGISPyUtils()
first = True
n_bands = 0
no_data_val = 0
for img in stats_imgs:
print(img)
if first:
n_bands = rsgis_utils.getImageBandCount(img)
no_data_val = rsgis_utils.getImageNoDataValue(img)
first = False
else:
if n_bands != rsgis_utils.getImageBandCount(img):
raise Exception(
"The number of bands must be the same in all input images."
)
if no_data_val != rsgis_utils.getImageNoDataValue(img):
raise Exception(
"The no data value should be the same in all input images."
)
# RIOS internal function to calculate mean and standard deviation of the input images
def _calcBasicStats(info, inputs, outputs, otherargs):
n_imgs = len(inputs.imgs)
blk_shp = inputs.imgs[0].shape
if blk_shp[0] != otherargs.n_bands:
raise Exception(
"Block shape and the number of input image bands do not align."
)
outputs.output_img = numpy.zeros(
(blk_shp[0] * 2, blk_shp[1], blk_shp[2]), dtype=float)
band_arr = []
for band in range(blk_shp[0]):
band_arr.append(
numpy.zeros((n_imgs, blk_shp[1], blk_shp[2]), dtype=float))
img_idx = 0
for img_blk in inputs.imgs:
for band in range(blk_shp[0]):
band_arr[band][img_idx] = img_blk[band]
img_idx = img_idx + 1
for band in range(blk_shp[0]):
band_arr[band][band_arr[band] == otherargs.no_data_val] = numpy.nan
outputs.output_img[band * 2] = numpy.nanmean(band_arr[band],
axis=0)
outputs.output_img[band * 2 + 1] = numpy.nanstd(band_arr[band],
axis=0)
outputs.output_img[band * 2][numpy.isnan(
outputs.output_img[band * 2])] = otherargs.no_data_val
outputs.output_img[band * 2 + 1][numpy.isnan(
outputs.output_img[band * 2 + 1])] = 0.0
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.imgs = stats_imgs
otherargs = applier.OtherInputs()
otherargs.n_bands = n_bands
otherargs.no_data_val = no_data_val
outfiles = applier.FilenameAssociations()
outfiles.output_img = output_img
aControls = applier.ApplierControls()
aControls.referenceImage = ref_img
aControls.footprint = applier.BOUNDS_FROM_REFERENCE
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Calculating Stats Image.")
applier.apply(_calcBasicStats,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed")
rsgislib.imageutils.popImageStats(output_img,
usenodataval=True,
nodataval=no_data_val,
calcpyramids=True)
def apply_xgboost_binary_classifier(model_file,
imgMask,
imgMaskVal,
imgFileInfo,
outProbImg,
gdalformat,
outClassImg=None,
class_thres=5000,
nthread=1):
"""
This function applies a trained binary (i.e., two classes) xgboost model. The function train_xgboost_binary_classifer
can be used to train such as model. The output image will contain the probability of membership to the class of
interest. You will need to threshold this image to get a final hard classification. Alternative, a hard class output
image and threshold can be applied to this image.
:param model_file: a trained xgboost binary model which can be loaded with lgb.Booster(model_file=model_file).
:param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the
valid data regions (rsgislib.imageutils.genValidMask)
:param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied.
Can be used to create a heirachical classification.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to
be used for the classification so it adheres to the training data.
:param outProbImg: output image file with the classification probabilities - this image is scaled by
multiplying by 10000.
:param gdalformat: is the output image format - all GDAL supported formats are supported.
:param outClassImg: Optional output image which will contain the hard classification, defined with a threshold on the
probability image.
:param class_thres: The threshold used to define the hard classification. Default is 5000 (i.e., probability of 0.5).
:param nthread: The number of threads to use for the classifier.
"""
def _applyXGBClassifier(info, inputs, outputs, otherargs):
outClassVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint16)
if numpy.any(inputs.imageMask == otherargs.mskVal):
outClassVals = outClassVals.flatten()
imgMaskVals = inputs.imageMask.flatten()
classVars = numpy.zeros(
(outClassVals.shape[0], otherargs.numClassVars),
dtype=numpy.float)
# Array index which can be used to populate the output array following masking etc.
ID = numpy.arange(imgMaskVals.shape[0])
classVarsIdx = 0
for imgFile in otherargs.imgFileInfo:
imgArr = inputs.__dict__[imgFile.name]
for band in imgFile.bands:
classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten()
classVarsIdx = classVarsIdx + 1
classVars = classVars[imgMaskVals == otherargs.mskVal]
ID = ID[imgMaskVals == otherargs.mskVal]
predClass = numpy.around(
otherargs.classifier.predict(xgb.DMatrix(classVars)) * 10000)
outClassVals[ID] = predClass
outClassVals = numpy.expand_dims(outClassVals.reshape(
(inputs.imageMask.shape[1], inputs.imageMask.shape[2])),
axis=0)
outputs.outimage = outClassVals
classifier = xgb.Booster({'nthread': nthread})
classifier.load_model(model_file)
infiles = applier.FilenameAssociations()
infiles.imageMask = imgMask
numClassVars = 0
for imgFile in imgFileInfo:
infiles.__dict__[imgFile.name] = imgFile.fileName
numClassVars = numClassVars + len(imgFile.bands)
outfiles = applier.FilenameAssociations()
outfiles.outimage = outProbImg
otherargs = applier.OtherInputs()
otherargs.classifier = classifier
otherargs.mskVal = imgMaskVal
otherargs.numClassVars = numClassVars
otherargs.imgFileInfo = imgFileInfo
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Applying the Classifier")
applier.apply(_applyXGBClassifier,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed")
rsgislib.imageutils.popImageStats(outProbImg,
usenodataval=True,
nodataval=0,
calcpyramids=True)
if outClassImg is not None:
rsgislib.imagecalc.imageMath(outProbImg, outClassImg,
'b1>{}?1:0'.format(class_thres),
gdalformat, rsgislib.TYPE_8UINT)
rsgislib.rastergis.populateStats(outClassImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
def translate(info,
infile,
outfile,
colTypes,
pulseCols=None,
expectRange=None,
scaling=None,
classificationTranslation=None,
nullVals=None,
constCols=None):
"""
Main function which does the work.
* Info is a fileinfo object for the input file.
* infile and outfile are paths to the input and output files respectively.
* expectRange is a list of tuples with (type, varname, min, max).
* scaling is a list of tuples with (type, varname, gain, offset).
* colTypes is a list of name and data type tuples for every column
* pulseCols is a list of strings defining the pulse columns
* classificationTranslation is a list of tuples specifying how to translate
between the codes within the files and the
lidarprocessor.CLASSIFICATION_* ones. First element of tuple is file
number, second the lidarprocessor code.
* nullVals is a list of tuples with (type, varname, value)
* constCols is a list of tupes with (type, varname, dtype, value)
"""
scalingsDict = translatecommon.overRideDefaultScalings(scaling)
# set up the variables
dataFiles = lidarprocessor.DataFiles()
dataFiles.input1 = lidarprocessor.LidarFile(infile, lidarprocessor.READ)
# convert from strings to numpy dtypes
numpyColTypes = []
for name, typeString in colTypes:
numpydtype = translatecommon.STRING_TO_DTYPE[typeString.upper()]
numpyColTypes.append((name, numpydtype))
dataFiles.input1.setLiDARDriverOption('COL_TYPES', numpyColTypes)
if pulseCols is not None:
dataFiles.input1.setLiDARDriverOption('PULSE_COLS', pulseCols)
if classificationTranslation is not None:
dataFiles.input1.setLiDARDriverOption('CLASSIFICATION_CODES',
classificationTranslation)
controls = lidarprocessor.Controls()
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
controls.setSpatialProcessing(False)
otherArgs = lidarprocessor.OtherArgs()
otherArgs.scaling = scalingsDict
otherArgs.expectRange = expectRange
otherArgs.nullVals = nullVals
otherArgs.constCols = constCols
dataFiles.output1 = lidarprocessor.LidarFile(outfile,
lidarprocessor.CREATE)
dataFiles.output1.setLiDARDriver('SPDV4')
dataFiles.output1.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING',
False)
lidarprocessor.doProcessing(transFunc,
dataFiles,
controls=controls,
otherArgs=otherArgs)
def rescaleImgPxlVals(inputImg,
outputImg,
gdalformat,
datatype,
bandRescale,
trim2Limits=True):
"""
Function which rescales an input image base on a list of rescaling parameters.
:param inputImg: the input image
:param outputImg: the output image file name and path (will be same dimensions as the input)
:param gdalformat: the GDAL image file format of the output image file.
:param bandRescale: list of ImageBandRescale objects
:param trim2Limits: whether to trim the output to the output min/max values.
"""
from rios import applier
bandRescaleDict = dict()
for rescaleObj in bandRescale:
bandRescaleDict[rescaleObj.band - 1] = rescaleObj
rsgis_utils = rsgislib.RSGISPyUtils()
numpyDT = rsgis_utils.getNumpyDataType(datatype)
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.image = inputImg
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.rescaleDict = bandRescaleDict
otherargs.trim = trim2Limits
otherargs.numpyDT = numpyDT
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyRescale(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpyDT)
for idx in range(inputs.image.shape[0]):
outputs.outimage[idx] = numpy.where(
inputs.image[idx] == otherargs.rescaleDict[idx].inNoData,
otherargs.rescaleDict[idx].outNoData,
(((inputs.image[idx] - otherargs.rescaleDict[idx].inMin) /
(inputs.image[idx] - otherargs.rescaleDict[idx].inMax -
inputs.image[idx] - otherargs.rescaleDict[idx].inMin)) *
(inputs.image[idx] - otherargs.rescaleDict[idx].outMax -
inputs.image[idx] - otherargs.rescaleDict[idx].outMin)) +
inputs.image[idx] - otherargs.rescaleDict[idx].outMin)
if otherargs.trim:
outputs.outimage[idx] = numpy.where(
(outputs.outimage[idx] !=
otherargs.rescaleDict[idx].outNoData) &
(outputs.outimage[idx] <
otherargs.rescaleDict[idx].outMin),
otherargs.rescaleDict[idx].outMin, outputs.outimage[idx])
outputs.outimage[idx] = numpy.where(
(outputs.outimage[idx] !=
otherargs.rescaleDict[idx].outNoData) &
(outputs.outimage[idx] >
otherargs.rescaleDict[idx].outMax),
otherargs.rescaleDict[idx].outMax, outputs.outimage[idx])
applier.apply(_applyRescale,
infiles,
outfiles,
otherargs,
controls=aControls)
def apply_keras_pixel_classifier(classTrainInfo,
keras_cls_mdl,
imgMask,
imgMaskVal,
imgFileInfo,
outClassImg,
gdalformat,
pred_batch_size=32,
classClrNames=True):
"""
This function applies a trained single pixel keras model to an image. The function train_keras_pixel_classifer
can be used to train such as model. The output image will contain the hard membership of the predicted class.
:param classTrainInfo: dict (where the key is the class name) of rsgislib.classification.ClassInfoObj
objects which will be used to train the classifier (i.e., train_keras_pixel_classifer()),
provide pixel value id and RGB class values.
:param keras_cls_mdl: a trained keras model object, with a input dimensions equivlent to the number of image
bands specified in the imgFileInfo input and output layer which provides an output array
of the length of the number of classes.
:param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the
valid data regions (rsgislib.imageutils.genValidMask)
:param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied.
Can be used to create a heirachical classification.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to
be used for the classification so it adheres to the training data.
:param outClassImg: Output image which will contain the hard classification.
:param gdalformat: is the output image format - all GDAL supported formats are supported.
:param pred_batch_size: the batch size used for the classification.
:param classClrNames: default is True and therefore a colour table will the colours specified in ClassInfoObj
and a ClassName (from classTrainInfo) column will be added to the output file.
"""
def _applyKerasPxlClassifier(info, inputs, outputs, otherargs):
outClassIdVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint16)
if numpy.any(inputs.imageMask == otherargs.mskVal):
n_pxls = inputs.imageMask.shape[1] * inputs.imageMask.shape[2]
outClassIdVals = outClassIdVals.flatten()
imgMaskVals = inputs.imageMask.flatten()
classVars = numpy.zeros((n_pxls, otherargs.numClassVars),
dtype=numpy.float)
# Array index which can be used to populate the output array following masking etc.
ID = numpy.arange(imgMaskVals.shape[0])
classVarsIdx = 0
for imgFile in otherargs.imgFileInfo:
imgArr = inputs.__dict__[imgFile.name]
for band in imgFile.bands:
classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten()
classVarsIdx = classVarsIdx + 1
classVars = classVars[imgMaskVals == otherargs.mskVal]
ID = ID[imgMaskVals == otherargs.mskVal]
preds_idxs = numpy.argmax(otherargs.classifier.predict(
classVars, batch_size=otherargs.pred_batch_size),
axis=1)
preds_cls_ids = numpy.zeros_like(preds_idxs, dtype=numpy.uint16)
for cld_id, idx in zip(otherargs.cls_id_lut,
numpy.arange(0, len(otherargs.cls_id_lut))):
preds_cls_ids[preds_idxs == idx] = cld_id
outClassIdVals[ID] = preds_cls_ids
outClassIdVals = numpy.expand_dims(outClassIdVals.reshape(
(inputs.imageMask.shape[1], inputs.imageMask.shape[2])),
axis=0)
outputs.outclsimage = outClassIdVals
infiles = applier.FilenameAssociations()
infiles.imageMask = imgMask
numClassVars = 0
for imgFile in imgFileInfo:
infiles.__dict__[imgFile.name] = imgFile.fileName
numClassVars = numClassVars + len(imgFile.bands)
n_classes = len(classTrainInfo)
cls_id_lut = numpy.zeros(n_classes)
for clsname in classTrainInfo:
if classTrainInfo[clsname].id >= n_classes:
raise (
"ClassInfoObj '{}' id ({}) is not consecutive starting from 0."
.format(clsname, classTrainInfo[clsname].id))
cls_id_lut[classTrainInfo[clsname].id] = classTrainInfo[clsname].out_id
outfiles = applier.FilenameAssociations()
outfiles.outclsimage = outClassImg
otherargs = applier.OtherInputs()
otherargs.classifier = keras_cls_mdl
otherargs.pred_batch_size = pred_batch_size
otherargs.mskVal = imgMaskVal
otherargs.numClassVars = numClassVars
otherargs.imgFileInfo = imgFileInfo
otherargs.n_classes = n_classes
otherargs.cls_id_lut = cls_id_lut
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Applying the Classifier")
applier.apply(_applyKerasPxlClassifier,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed Classification")
if classClrNames:
rsgislib.rastergis.populateStats(outClassImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
ratDataset = gdal.Open(outClassImg, gdal.GA_Update)
red = rat.readColumn(ratDataset, 'Red')
green = rat.readColumn(ratDataset, 'Green')
blue = rat.readColumn(ratDataset, 'Blue')
ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255'))
ClassName[...] = ""
for classKey in classTrainInfo:
print("Apply Colour to class \'" + classKey + "\'")
red[classTrainInfo[classKey].out_id] = classTrainInfo[classKey].red
green[classTrainInfo[classKey].
out_id] = classTrainInfo[classKey].green
blue[classTrainInfo[classKey].
out_id] = classTrainInfo[classKey].blue
ClassName[classTrainInfo[classKey].out_id] = classKey
rat.writeColumn(ratDataset, "Red", red)
rat.writeColumn(ratDataset, "Green", green)
rat.writeColumn(ratDataset, "Blue", blue)
rat.writeColumn(ratDataset, "ClassName", ClassName)
ratDataset = None
def find_class_outliers(pyod_obj,
img,
img_mask,
out_lbls_img,
out_scores_img=None,
img_mask_val=1,
img_bands=None,
gdalformat="KEA"):
"""
This function uses the pyod (https://github.com/yzhao062/pyod) library to find outliers within a class.
It is assumed that the input images are from a different date than the mask (classification) and therefore
the outliners will related to class changes.
:param pyod_obj: an instance of a pyod.models (e.g., pyod.models.knn.KNN) pass parameters to the constructor
:param img: input image used for analysis
:param img_mask: input image mask use to define the region of interest.
:param out_lbls_img: output image with pixel over of 1 for within mask but not outlier and 2 for in mask and outlier.
:param out_scores_img: output image (optional, None and won't be provided; Default None) providing the probability of
each pixel being an outlier
:param img_mask_val: the pixel value within the mask image for the class of interest. (Default 1)
:param img_bands: the image bands to be used for the analysis. If None then all used (Default: None)
:param gdalformat: file format for the output image(s). Default KEA.
"""
if not haveRIOS:
raise Exception(
"The rios module is required for this function could not be imported\n\t"
+ riosErr)
rsgis_utils = rsgislib.RSGISPyUtils()
if img_bands is not None:
if not ((type(img_bands) is list) or (type(img_bands) is tuple)):
raise rsgislib.RSGISPyException(
"If provided then img_bands should be a list (or None)")
else:
n_bands = rsgis_utils.getImageBandCount(img)
img_bands = numpy.arange(1, n_bands + 1)
num_vars = len(img_bands)
img_val_no_data = rsgis_utils.getImageNoDataValue(img)
msk_arr_vals = rsgislib.imageutils.extractImgPxlValsInMsk(
img, img_bands, img_mask, img_mask_val, img_val_no_data)
print("There were {} pixels within the mask.".format(
msk_arr_vals.shape[0]))
print("Fitting oulier detector")
pyod_obj.fit(msk_arr_vals)
print("Fitted oulier detector")
# RIOS function to apply classifer
def _applyPyOB(info, inputs, outputs, otherargs):
# Internal function for rios applier. Used within find_class_outliers.
out_lbls_vals = numpy.zeros_like(inputs.image_mask, dtype=numpy.uint8)
if otherargs.out_scores:
out_scores_vals = numpy.zeros_like(inputs.image_mask,
dtype=numpy.float)
if numpy.any(inputs.image_mask == otherargs.msk_val):
out_lbls_vals = out_lbls_vals.flatten()
img_msk_vals = inputs.image_mask.flatten()
if otherargs.out_scores:
out_scores_vals = out_scores_vals.flatten()
ID = numpy.arange(img_msk_vals.shape[0])
img_shape = inputs.img.shape
img_bands = inputs.img.reshape(
(img_shape[0], (img_shape[1] * img_shape[2])))
band_lst = []
for band in otherargs.img_bands:
if (band > 0) and (band <= img_shape[0]):
band_lst.append(img_bands[band - 1])
else:
raise Exception(
"Band ({}) specified is not within the image".format(
band))
img_bands_sel = numpy.stack(band_lst, axis=0)
img_bands_trans = numpy.transpose(img_bands_sel)
if otherargs.no_data_val is not None:
ID = ID[(img_bands_trans != otherargs.no_data_val).all(axis=1)]
img_msk_vals = img_msk_vals[(
img_bands_trans != otherargs.no_data_val).all(axis=1)]
img_bands_trans = img_bands_trans[(
img_bands_trans != otherargs.no_data_val).all(axis=1)]
ID = ID[img_msk_vals == otherargs.msk_val]
img_bands_trans = img_bands_trans[img_msk_vals ==
otherargs.msk_val]
if img_bands_trans.shape[0] > 0:
pred_lbls = otherargs.pyod_obj.predict(img_bands_trans)
pred_lbls = pred_lbls + 1
out_lbls_vals[ID] = pred_lbls
out_lbls_vals = numpy.expand_dims(out_lbls_vals.reshape(
(inputs.image_mask.shape[1], inputs.image_mask.shape[2])),
axis=0)
if otherargs.out_scores:
if img_bands_trans.shape[0] > 0:
pred_probs = otherargs.pyod_obj.predict_proba(
img_bands_trans, method='unify')
out_scores_vals[ID] = pred_probs[:, 1]
out_scores_vals = numpy.expand_dims(out_scores_vals.reshape(
(inputs.image_mask.shape[1], inputs.image_mask.shape[2])),
axis=0)
outputs.out_lbls_img = out_lbls_vals
if otherargs.out_scores:
outputs.out_scores_img = out_scores_vals
infiles = applier.FilenameAssociations()
infiles.image_mask = img_mask
infiles.img = img
otherargs = applier.OtherInputs()
otherargs.pyod_obj = pyod_obj
otherargs.msk_val = img_mask_val
otherargs.img_bands = img_bands
otherargs.out_scores = False
otherargs.no_data_val = img_val_no_data
outfiles = applier.FilenameAssociations()
outfiles.out_lbls_img = out_lbls_img
if out_scores_img is not None:
outfiles.out_scores_img = out_scores_img
otherargs.out_scores = True
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Applying the Outlier Detector")
applier.apply(_applyPyOB, infiles, outfiles, otherargs, controls=aControls)
print("Completed")
rsgislib.rastergis.populateStats(clumps=out_lbls_img,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
if out_scores_img is not None:
rsgislib.imageutils.popImageStats(out_scores_img,
usenodataval=True,
nodataval=0,
calcpyramids=True)
def translate(info,
infile,
outfile,
expectRange=None,
spatial=None,
extent=None,
scaling=None,
epsg=None,
binSize=None,
buildPulses=False,
pulseIndex=None,
nullVals=None,
constCols=None,
useLASScaling=False):
"""
Main function which does the work.
* Info is a fileinfo object for the input file.
* infile and outfile are paths to the input and output files respectively.
* expectRange is a list of tuples with (type, varname, min, max).
* spatial is True or False - dictates whether we are processing spatially or not.
If True then spatial index will be created on the output file on the fly.
* extent is a tuple of values specifying the extent to work with.
xmin ymin xmax ymax
* scaling is a list of tuples with (type, varname, dtype, gain, offset).
* if epsg is not None should be a EPSG number to use as the coord system
* binSize is the used by the LAS spatial index
* buildPulses dictates whether to attempt to build the pulse structure
* pulseIndex should be 'FIRST_RETURN' or 'LAST_RETURN' and determines how the
pulses are indexed.
* nullVals is a list of tuples with (type, varname, value)
* constCols is a list of tupes with (type, varname, dtype, value)
* if useLASScaling is True, then the scaling used in the LAS file
is used for columns. Overrides anything given in 'scaling'
"""
scalingsDict = translatecommon.overRideDefaultScalings(scaling)
if epsg is None and (info.wkt is None or len(info.wkt) == 0):
msg = 'No projection set in las file. Must set EPSG on command line'
raise generic.LiDARInvalidSetting(msg)
if spatial and not info.hasSpatialIndex:
msg = 'Spatial processing requested but file does not have spatial index'
raise generic.LiDARInvalidSetting(msg)
if spatial and binSize is None:
msg = "For spatial processing, the bin size must be set"
raise generic.LiDARInvalidSetting(msg)
if extent is not None and not spatial:
msg = 'Extent can only be set when processing spatially'
raise generic.LiDARInvalidSetting(msg)
# set up the variables
dataFiles = lidarprocessor.DataFiles()
dataFiles.input1 = lidarprocessor.LidarFile(infile, lidarprocessor.READ)
if pulseIndex == 'FIRST_RETURN':
dataFiles.input1.setLiDARDriverOption('PULSE_INDEX', las.FIRST_RETURN)
elif pulseIndex == 'LAST_RETURN':
dataFiles.input1.setLiDARDriverOption('PULSE_INDEX', las.LAST_RETURN)
else:
msg = "Pulse index argument not recognised."
raise generic.LiDARInvalidSetting(msg)
dataFiles.input1.setLiDARDriverOption('BUILD_PULSES', buildPulses)
if spatial:
dataFiles.input1.setLiDARDriverOption('BIN_SIZE', float(binSize))
controls = lidarprocessor.Controls()
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
controls.setSpatialProcessing(spatial)
otherArgs = lidarprocessor.OtherArgs()
otherArgs.scaling = scalingsDict
otherArgs.epsg = epsg
otherArgs.expectRange = expectRange
otherArgs.lasInfo = info
otherArgs.nullVals = nullVals
otherArgs.constCols = constCols
otherArgs.useLASScaling = useLASScaling
if extent is not None:
extent = [float(x) for x in extent]
pixgrid = pixelgrid.PixelGridDefn(xMin=extent[0],
yMin=extent[1],
xMax=extent[2],
yMax=extent[3],
xRes=binSize,
yRes=binSize)
controls.setReferencePixgrid(pixgrid)
controls.setFootprint(lidarprocessor.BOUNDS_FROM_REFERENCE)
dataFiles.output1 = lidarprocessor.LidarFile(outfile,
lidarprocessor.CREATE)
dataFiles.output1.setLiDARDriver('SPDV4')
dataFiles.output1.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING',
False)
lidarprocessor.doProcessing(transFunc,
dataFiles,
controls=controls,
otherArgs=otherArgs)
def translate(info,
infile,
outfile,
expectRange=None,
scalings=None,
internalrotation=False,
magneticdeclination=0.0,
externalrotationfn=None,
nullVals=None,
constCols=None,
epsg=None,
wkt=None):
"""
Main function which does the work.
* Info is a fileinfo object for the input file.
* infile and outfile are paths to the input and output files respectively.
* expectRange is a list of tuples with (type, varname, min, max).
* scaling is a list of tuples with (type, varname, gain, offset).
* if internalrotation is True then the internal rotation will be applied
to data. Overrides externalrotationfn
* if externalrotationfn is not None then then the external rotation matrix
will be read from this file and applied to the data
* magneticdeclination. If not 0, then this will be applied to the data
* nullVals is a list of tuples with (type, varname, value)
* constCols is a list of tupes with (type, varname, dtype, value)
"""
scalingsDict = translatecommon.overRideDefaultScalings(scalings)
# set up the variables
dataFiles = lidarprocessor.DataFiles()
dataFiles.input1 = lidarprocessor.LidarFile(infile, lidarprocessor.READ)
# first set the rotation matrix if asked for
if internalrotation and externalrotationfn:
msg = "Can't use both internal and external rotation"
raise generic.LiDARInvalidSetting(msg)
rotationMatrix = None
if internalrotation:
if "ROTATION_MATRIX" in info.header:
dataFiles.input1.setLiDARDriverOption(
"ROTATION_MATRIX", info.header["ROTATION_MATRIX"])
rotationMatrix = info.header["ROTATION_MATRIX"]
else:
msg = "Internal Rotation requested but no information found in input file"
raise generic.LiDARInvalidSetting(msg)
elif externalrotationfn is not None:
externalrotation = numpy.loadtxt(externalrotationfn,
ndmin=2,
delimiter=" ",
dtype=numpy.float32)
dataFiles.input1.setLiDARDriverOption("ROTATION_MATRIX",
externalrotation)
rotationMatrix = externalrotation
# set the magnetic declination if not 0 (the default)
if magneticdeclination != 0:
dataFiles.input1.setLiDARDriverOption("MAGNETIC_DECLINATION",
magneticdeclination)
controls = lidarprocessor.Controls()
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
controls.setSpatialProcessing(False)
otherArgs = lidarprocessor.OtherArgs()
# and the header so we don't collect it again
otherArgs.rieglInfo = info.header
# also need the default/overriden scaling
otherArgs.scaling = scalingsDict
# Add the rotation matrix to otherArgs
# for updating the header
otherArgs.rotationMatrix = rotationMatrix
# expected range of the data
otherArgs.expectRange = expectRange
# null values
otherArgs.nullVals = nullVals
# constant columns
otherArgs.constCols = constCols
otherArgs.epsg = epsg
otherArgs.wkt = wkt
dataFiles.output1 = lidarprocessor.LidarFile(outfile,
lidarprocessor.CREATE)
dataFiles.output1.setLiDARDriver('SPDV4')
dataFiles.output1.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING',
False)
lidarprocessor.doProcessing(transFunc,
dataFiles,
controls=controls,
otherArgs=otherArgs)
def rasterize(infiles,
outfile,
attributes,
function=DEFAULT_FUNCTION,
atype=DEFAULT_ATTRIBUTE,
background=0,
binSize=None,
extraModule=None,
quiet=False,
footprint=None,
windowSize=None,
driverName=None,
driverOptions=None):
"""
Apply the given function to the list of input files and create
an output raster file. attributes is a list of attributes to run
the function on. The function name must contain a module
name and the specified function must take a masked array, plus
the 'axis' parameter. atype should be a string containing
either POINT|PULSE.
background is the background raster value to use. binSize is the bin size
to use which defaults to that of the spatial indices used.
extraModule should be a string with an extra module to import - use
this for modules other than numpy that are needed by your function.
quiet means no progress etc
footprint specifies the footprint type
"""
dataFiles = lidarprocessor.DataFiles()
dataFiles.inList = [
lidarprocessor.LidarFile(fname, lidarprocessor.READ)
for fname in infiles
]
dataFiles.imageOut = lidarprocessor.ImageFile(outfile,
lidarprocessor.CREATE)
dataFiles.imageOut.setRasterIgnore(background)
if driverName is not None:
dataFiles.imageOut.setRasterDriver(driverName)
if driverOptions is not None:
dataFiles.imageOut.setRasterDriverOptions(driverOptions)
# import any other modules required
globalsDict = globals()
if extraModule is not None:
globalsDict[extraModule] = importlib.import_module(extraModule)
controls = lidarprocessor.Controls()
controls.setSpatialProcessing(True)
if not quiet:
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
if binSize is not None:
controls.setReferenceResolution(binSize)
if footprint is not None:
controls.setFootprint(footprint)
if windowSize is not None:
controls.setWindowSize(windowSize)
otherArgs = lidarprocessor.OtherArgs()
# reference to the function to call
otherArgs.func = eval(function, globalsDict)
otherArgs.attributes = attributes
otherArgs.background = background
atype = atype.upper()
if atype == 'POINT':
otherArgs.atype = POINT
elif atype == 'PULSE':
otherArgs.atype = PULSE
else:
msg = 'Unsupported type %s' % atype
raise RasterizationError(msg)
lidarprocessor.doProcessing(writeImageFunc,
dataFiles,
controls=controls,
otherArgs=otherArgs)
def calcSolarAzimuthZenith(inputImg, inImgDateTime, outputImg, gdalformat):
"""
Function which calculate a solar azimuth (band 1) and zenith (band 2) image.
:param inputImg: input image file (can be any image with a spatial header)
:param inImgDateTime: a datatime object for the data/time of the acquasition
:param outputImg: output image file and path
:param gdalformat: output file format (e.g., KEA)
"""
if not havePysolar:
raise Exception(
"The PySolar module required for this function could not be imported."
)
if not haveRIOS:
raise Exception(
"The RIOS module required for this function could not be imported."
)
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.image1 = inputImg
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.dateTime = inImgDateTime
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
wgs84latlonProj = osr.SpatialReference()
wgs84latlonProj.ImportFromEPSG(4326)
otherargs.wgs84latlonProj = wgs84latlonProj
def _calcSolarAzimuthZenith(info, inputs, outputs, otherargs):
"""
Internal functions used within calcSolarAzimuthZenith() - don't call independently.
"""
xBlock, yBlock = info.getBlockCoordArrays()
inProj = osr.SpatialReference()
inProj.ImportFromWkt(info.getProjection())
transform = osr.CoordinateTransformation(inProj,
otherargs.wgs84latlonProj)
xBlockF = xBlock.flatten()
yBlockF = yBlock.flatten()
pts = numpy.zeros((xBlockF.shape[0], 2), dtype=numpy.float)
pts[..., 0] = xBlockF
pts[..., 1] = yBlockF
outPts = numpy.array(transform.TransformPoints(pts))
outAz = numpy.zeros_like(xBlockF, dtype=numpy.float)
outZen = numpy.zeros_like(xBlockF, dtype=numpy.float)
for i in range(outPts.shape[0]):
outAz[i] = 90 - Pysolar.solar.GetAltitude(
outPts[i, 1], outPts[i, 0], otherargs.dateTime)
outZen[i] = ((-1) * Pysolar.solar.GetAzimuth(
outPts[i, 1], outPts[i, 0], otherargs.dateTime)) - 180
outAz = numpy.reshape(outAz, xBlock.shape)
outZen = numpy.reshape(outZen, xBlock.shape)
#print("Block End")
outputs.outimage = numpy.stack((outAz, outZen))
# Apply the multiply function.
applier.apply(_calcSolarAzimuthZenith,
infiles,
outfiles,
otherargs,
controls=aControls)
def calc_unmixing_rmse_residualerr(self,
input_img,
input_unmix_img,
endmembers_file,
output_img,
gdalformat,
scale_factor=1000):
"""
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
endmembers_info = self.read_endmembers_mtxt(endmembers_file)
infiles = applier.FilenameAssociations()
infiles.image_orig = input_img
infiles.image_unmix = input_unmix_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.endmembers_q = endmembers_info[0]
otherargs.endmembers_p = endmembers_info[1]
otherargs.endmembers_arr = endmembers_info[2]
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _calc_unmix_err_rmse(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
block_refl_shp = inputs.image_orig.shape
img_orig_refl_flat = inputs.image_orig.reshape(
[block_refl_shp[0], (block_refl_shp[1] * block_refl_shp[2])]).T
block_unmix_shp = inputs.image_unmix.shape
img_unmix_coef_flat = inputs.image_unmix.reshape([
block_unmix_shp[0], (block_unmix_shp[1] * block_unmix_shp[2])
]).T
img_flat_shp = img_orig_refl_flat.shape
img_orig_refl_nodata = numpy.where(img_orig_refl_flat == 0, True,
False)
img_orig_refl_flat_nodata = numpy.all(img_orig_refl_nodata, axis=1)
ID = numpy.arange(img_flat_shp[0])
ID = ID[img_orig_refl_flat_nodata == False]
img_orig_refl_flat_data = img_orig_refl_flat[
img_orig_refl_flat_nodata == False, ...]
img_orig_refl_flat_data_flt = numpy.zeros_like(
img_orig_refl_flat_data, dtype=numpy.float32)
img_orig_refl_flat_data_flt[...] = img_orig_refl_flat_data
img_unmix_coef_flat_data = img_unmix_coef_flat[
img_orig_refl_flat_nodata == False, ...]
img_unmix_coef_flat_data_flt = numpy.zeros_like(
img_unmix_coef_flat_data, dtype=numpy.float32)
img_unmix_coef_flat_data_flt[
...] = img_unmix_coef_flat_data / float(scale_factor)
img_nodata_flat_shp = img_unmix_coef_flat_data_flt.shape
pred_refl_img = numpy.zeros_like(img_orig_refl_flat_data,
dtype=float)
for i in range(img_nodata_flat_shp[0]):
for q in range(otherargs.endmembers_q):
pred_refl_img[i] = pred_refl_img[i] + (
img_unmix_coef_flat_data_flt[i, q] *
otherargs.endmembers_arr[q])
# Calc Diff
diff = img_orig_refl_flat_data_flt - pred_refl_img
# Calc RMSE
diff_sq = diff * diff
mean_sum_diff_sq = numpy.sum(diff_sq,
axis=1) / otherargs.endmembers_p
rmse_arr = numpy.sqrt(mean_sum_diff_sq)
# Calc Residual Error
abs_diff = numpy.absolute(diff)
residual_arr = numpy.sum(abs_diff, axis=1) / otherargs.endmembers_p
outarr = numpy.zeros((img_flat_shp[0], 2))
outarr[ID] = numpy.stack([rmse_arr, residual_arr], axis=-1)
outarr = outarr.T
outputs.outimage = outarr.reshape(
(2, block_refl_shp[1], block_refl_shp[2]))
applier.apply(_calc_unmix_err_rmse,
infiles,
outfiles,
otherargs,
controls=aControls)
def predict_refl_img_from_simple_unmixing(self,
input_unmix_img,
endmembers_file,
output_img,
gdalformat,
scale_factor=1000):
"""
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
endmembers_info = self.read_endmembers_mtxt(endmembers_file)
infiles = applier.FilenameAssociations()
infiles.image_unmix = input_unmix_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.endmembers_q = endmembers_info[0]
otherargs.endmembers_p = endmembers_info[1]
otherargs.endmembers_arr = endmembers_info[2]
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _predict_refl_img(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
block_unmix_shp = inputs.image_unmix.shape
img_unmix_coef_flat = inputs.image_unmix.reshape([
block_unmix_shp[0], (block_unmix_shp[1] * block_unmix_shp[2])
]).T
img_unmix_coef_flat_data_flt = numpy.zeros_like(
img_unmix_coef_flat, dtype=numpy.float32)
img_unmix_coef_flat_data_flt[
...] = img_unmix_coef_flat / float(scale_factor)
img_nodata_flat_shp = img_unmix_coef_flat_data_flt.shape
pred_refl_img = numpy.zeros(
(img_unmix_coef_flat.shape[0], otherargs.endmembers_p),
dtype=numpy.int16)
for i in range(img_unmix_coef_flat.shape[0]):
for q in range(otherargs.endmembers_q):
pred_refl_img[i] = pred_refl_img[i] + (
img_unmix_coef_flat_data_flt[i, q] *
otherargs.endmembers_arr[q])
pred_refl_img = pred_refl_img.T
outputs.outimage = pred_refl_img.reshape(
(otherargs.endmembers_p, block_unmix_shp[1],
block_unmix_shp[2]))
applier.apply(_predict_refl_img,
infiles,
outfiles,
otherargs,
controls=aControls)
def perform_simple_unmixing(self,
input_image,
output_image,
gdalformat,
endmembers_file,
weight=None,
scale_factor=1000):
"""
A function which uses the RIOS to iterate through the input image
and perform a simple/standard spectral unmixing on the input image
using the calc_abundance function.
:param input_image: The file path to a GDAL readable input image.
:param output_image: The file path to the GDAL writable output image
(Note pixel values will be between 0-1000)
:param gdalformat: The output image format to be used.
:param endmembers_file: The endmembers for the unmixing in the RSGISLib mtxt format.
:param weight: Optional (if None ignored) to provide a weight to implement the approach of Scarth et al (2010)
adding a weight to the least squares optimisation to get the abundances to sum to 1.
:param scale_factor: Scale factor for integerising the resulting image. If value is 1 then output image
will be a floating point image.
References:
Scarth, P., Roder, A., & Schmidt, M. (2010). Tracking grazing pressure and climate
interaction—The role of Landsat fractional cover in time series analysis. Proceedings of
the 15th Australasian Remote Sensing and Photogrammetry ConferenceAustralia: Alice Springs.
http://dx.doi.org/10.6084/m9.figshare.94250.
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
if weight is not None:
endmembers_info = self.read_endmembers_mtxt_weight(
endmembers_file, weight)
else:
endmembers_info = self.read_endmembers_mtxt(endmembers_file)
print(endmembers_info)
infiles = applier.FilenameAssociations()
infiles.image = input_image
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_image
otherargs = applier.OtherInputs()
otherargs.endmembers_q = endmembers_info[0]
otherargs.endmembers_p = endmembers_info[1]
otherargs.endmembers_arr = endmembers_info[2]
otherargs.weight = weight
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _simple_unmix(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
block_shp = inputs.image.shape
img_flat = inputs.image.reshape(
[block_shp[0], (block_shp[1] * block_shp[2])]).T
if otherargs.weight is not None:
img_flat_dtype = img_flat.dtype
tmp_img_flat = img_flat
img_flat = numpy.zeros(
(img_flat.shape[0], img_flat.shape[1] + 1),
dtype=img_flat_dtype)
img_flat[...] = weight
img_flat[:, :-1] = tmp_img_flat
img_flat_shp = img_flat.shape
img_nodata = numpy.where(img_flat == 0, True, False)
img_flat_nodata = numpy.all(img_nodata, axis=1)
ID = numpy.arange(img_flat_shp[0])
ID = ID[img_flat_nodata == False]
img_flat_data = img_flat[img_flat_nodata == False, ...]
abundances_arr = self.calc_abundance(img_flat_data,
otherargs.endmembers_arr)
if otherargs.scale_factor > 1:
outarr = numpy.zeros([img_flat_shp[0], otherargs.endmembers_q],
dtype=numpy.int16)
else:
outarr = numpy.zeros([img_flat_shp[0], otherargs.endmembers_q],
dtype=numpy.float32)
outarr[ID] = (abundances_arr * otherargs.scale_factor)
outarr = outarr.T
outputs.outimage = outarr.reshape(
(otherargs.endmembers_q, block_shp[1], block_shp[2]))
applier.apply(_simple_unmix,
infiles,
outfiles,
otherargs,
controls=aControls)
def translate(info,
infile,
outfile,
expectRange=None,
spatial=False,
extent=None,
scaling=None,
nullVals=None,
constCols=None):
"""
Main function which does the work.
* Info is a fileinfo object for the input file.
* infile and outfile are paths to the input and output files respectively.
* expectRange is a list of tuples with (type, varname, min, max).
* spatial is True or False - dictates whether we are processing spatially or not.
If True then spatial index will be created on the output file on the fly.
* extent is a tuple of values specifying the extent to work with.
xmin ymin xmax ymax
* scaling is a list of tuples with (type, varname, gain, offset).
* nullVals is a list of tuples with (type, varname, value)
* constCols is a list of tupes with (type, varname, dtype, value)
"""
scalingsDict = translatecommon.overRideDefaultScalings(scaling)
# first we need to determine if the file is spatial or not
if spatial and not info.hasSpatialIndex:
msg = "Spatial processing requested but file does not have spatial index"
raise generic.LiDARInvalidSetting(msg)
if extent is not None and not spatial:
msg = 'Extent can only be set when processing spatially'
raise generic.LiDARInvalidSetting(msg)
dataFiles = lidarprocessor.DataFiles()
dataFiles.input1 = lidarprocessor.LidarFile(infile, lidarprocessor.READ)
dataFiles.output1 = lidarprocessor.LidarFile(outfile,
lidarprocessor.CREATE)
dataFiles.output1.setLiDARDriver('SPDV4')
dataFiles.output1.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING',
False)
controls = lidarprocessor.Controls()
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
controls.setSpatialProcessing(spatial)
if extent is not None:
extent = [float(x) for x in extent]
binSize = info.header['BIN_SIZE']
pixgrid = pixelgrid.PixelGridDefn(xMin=extent[0],
yMin=extent[1],
xMax=extent[2],
yMax=extent[3],
xRes=binSize,
yRes=binSize)
controls.setReferencePixgrid(pixgrid)
controls.setFootprint(lidarprocessor.BOUNDS_FROM_REFERENCE)
otherArgs = lidarprocessor.OtherArgs()
otherArgs.scaling = scalingsDict
otherArgs.expectRange = expectRange
otherArgs.nullVals = nullVals
otherArgs.constCols = constCols
lidarprocessor.doProcessing(transFunc,
dataFiles,
controls=controls,
otherArgs=otherArgs)
def splitFileIntoTiles(infiles, binSize=1.0, blockSize=None,
tempDir='.', extent=None, indexType=INDEX_CARTESIAN,
pulseIndexMethod=PULSE_INDEX_FIRST_RETURN,
footprint=lidarprocessor.UNION, outputFormat='SPDV4',
buildPulses=False):
"""
Takes a filename (or list of filenames) and creates a tempfile for every
block (using blockSize).
If blockSize isn't set then it is picked using BLOCKSIZE_N_BLOCKS.
binSize is the size of the bins to create the spatial index.
indexType is one of the INDEX_* constants.
pulseIndexMethod is one of the PULSE_INDEX_* constants.
footprint is one of lidarprocessor.UNION or lidarprocessor.INTERSECTION
and is how to combine extents if there is more than one file.
outputFormat is either 'SPDV4' or 'LAS'. 'LAS' outputs only supported
when input is 'LAS'.
buildPulses relevant for 'LAS' and determines whether to build the
pulse structure or not.
returns the header of the first input file, the extent used and a list
of (fname, extent) tuples that contain the information for
each tempfile.
"""
if isinstance(infiles, basestring):
infiles = [infiles]
# use the first file for header. Not
# clear how to combine headers from multiple inputs
# or if one should.
# leave setting this in case we grab it when working out the extent.
firstHeader = None
if extent is None:
# work out from headers
pixGrid = None
for infile in infiles:
info = generic.getLidarFileInfo(infile)
header = info.header
if firstHeader is None:
firstHeader = header
try:
if indexType == INDEX_CARTESIAN:
xMax = header['X_MAX']
xMin = header['X_MIN']
yMax = header['Y_MAX']
yMin = header['Y_MIN']
elif indexType == INDEX_SPHERICAL:
xMax = header['AZIMUTH_MAX']
xMin = header['AZIMUTH_MIN']
yMax = header['ZENITH_MAX']
yMin = header['ZENITH_MIN']
elif indexType == INDEX_SCAN:
xMax = header['SCANLINE_IDX_MAX']
xMin = header['SCANLINE_IDX_MIN']
yMax = header['SCANLINE_MAX']
yMin = header['SCANLINE_MIN']
else:
msg = 'unsupported indexing method'
raise generic.LiDARSpatialIndexNotAvailable(msg)
except KeyError:
msg = 'info for creating bounding box not available'
raise generic.LiDARFunctionUnsupported(msg)
newPixGrid = pixelgrid.PixelGridDefn(xMin=xMin, xMax=xMax,
yMin=yMin, yMax=yMax, xRes=binSize, yRes=binSize)
if pixGrid is None:
pixGrid = newPixGrid
elif footprint == lidarprocessor.UNION:
pixGrid = pixGrid.union(newPixGrid)
elif footprint == lidarprocessor.INTERSECTION:
pixGrid = pixGrid.intersection(newPixGrid)
else:
msg = 'Unsupported footprint option'
raise generic.LiDARFunctionUnsupported(msg)
# TODO: we treat points as being in the block when they are >=
# the min coords and < the max coords. What happens on the bottom
# and right margins?? We could possibly miss points that are there.
# round the coords to the nearest multiple
xMin = numpy.floor(pixGrid.xMin / binSize) * binSize
yMin = numpy.floor(pixGrid.yMin / binSize) * binSize
xMax = numpy.ceil(pixGrid.xMax / binSize) * binSize
yMax = numpy.ceil(pixGrid.yMax / binSize) * binSize
extent = Extent(xMin, xMax, yMin, yMax, binSize)
else:
# ensure that our binSize comes from their exent
binSize = extent.binSize
# get the first header since we aren't doing the above
info = generic.getLidarFileInfo(infiles[0])
firstHeader = info.header
if blockSize is None:
minAxis = min(extent.xMax - extent.xMin, extent.yMax - extent.yMin)
blockSize = min(minAxis / BLOCKSIZE_N_BLOCKS, 200.0)
# make it a multiple of binSize
blockSize = int(numpy.ceil(blockSize / binSize)) * binSize
else:
# ensure that their given block size can be evenly divided by
# the binSize
# the modulo operator doesn't work too well with floats
# so we take a different approach
a = blockSize / binSize
if a != int(a):
msg = 'blockSize must be evenly divisible be the binSize'
raise generic.LiDARInvalidData(msg)
extentList = []
subExtent = Extent(extent.xMin, extent.xMin + blockSize,
extent.yMax - blockSize, extent.yMax, binSize)
controls = lidarprocessor.Controls()
controls.setSpatialProcessing(False)
tmpSuffix = '.' + outputFormat.lower()
bMoreToDo = True
while bMoreToDo:
fd, fname = tempfile.mkstemp(suffix=tmpSuffix, dir=tempDir)
os.close(fd)
userClass = lidarprocessor.LidarFile(fname, generic.CREATE)
if outputFormat == 'SPDV4':
userClass.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING', False)
driver = spdv4.SPDV4File(fname, generic.CREATE, controls, userClass)
elif outputFormat == 'LAS':
driver = las.LasFile(fname, generic.CREATE, controls, userClass)
else:
msg = 'Unsupported output format %s' % outputFormat
raise generic.LiDARFunctionUnsupported(msg)
data = (copy.copy(subExtent), driver)
extentList.append(data)
# move it along
subExtent.xMin += blockSize
subExtent.xMax += blockSize
if subExtent.xMin >= extent.xMax:
# next line down
subExtent.xMin = extent.xMin
subExtent.xMax = extent.xMin + blockSize
subExtent.yMax -= blockSize
subExtent.yMin -= blockSize
# done?
bMoreToDo = subExtent.yMax > extent.yMin
# ok now set up to read the input files using lidarprocessor
dataFiles = lidarprocessor.DataFiles()
dataFiles.inputs = []
for infile in infiles:
input = lidarprocessor.LidarFile(infile, lidarprocessor.READ)
# must be a better way of doing this, but this is what
# translate does. We don't know what formats we are getting ahead of time
info = generic.getLidarFileInfo(infile)
inFormat = info.getDriverName()
if inFormat == 'LAS':
input.setLiDARDriverOption('BUILD_PULSES', buildPulses)
dataFiles.inputs.append(input)
controls = lidarprocessor.Controls()
progress = cuiprogress.GDALProgressBar()
progress.setLabelText('Splitting...')
controls.setProgress(progress)
controls.setSpatialProcessing(False)
controls.setMessageHandler(lidarprocessor.silentMessageFn)
otherArgs = lidarprocessor.OtherArgs()
otherArgs.outList = extentList
otherArgs.indexType = indexType
otherArgs.pulseIndexMethod = pulseIndexMethod
lidarprocessor.doProcessing(classifyFunc, dataFiles, controls=controls,
otherArgs=otherArgs)
# close all the output files and save their names to return
newExtentList = []
for subExtent, driver in extentList:
fname = driver.fname
driver.close()
data = (fname, subExtent)
newExtentList.append(data)
return firstHeader, extent, newExtentList
def convert_to_dB(input_img, output_img, gdal_format, out_int_imgs=False):
"""
Convert power image to decibels (dB) by applying 10 x log10(pwr)
:param input_img: Input power image
:param output_img: Output dB image
:param gdal_format: GDAL image format for output image
:param out_int_imgs: if False then output image is Float32 if True then Int16 with gain of 100
"""
try:
import tqdm
progress_bar = TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
def _apply_calc_dB(info, inputs, outputs, otherargs):
# Internal Function...
msk_arr = numpy.where(
((inputs.image > 0.0) & numpy.isfinite(inputs.image)), 1, 0)
msk_arr = numpy.amin(msk_arr, axis=0)
dB_flt_img_arr = numpy.where(msk_arr == 1,
10 * numpy.log10(inputs.image), 999)
dB_flt_img_arr[numpy.isnan(dB_flt_img_arr)] = 999
dB_flt_img_arr[numpy.isinf(dB_flt_img_arr)] = 999
msk_arr = numpy.where((dB_flt_img_arr > -60) & (dB_flt_img_arr < 20),
1, 0)
msk_arr = numpy.amin(msk_arr, axis=0)
numpy.where(msk_arr == 1, dB_flt_img_arr, 999)
if otherargs.out_int_imgs:
out_img_arr = numpy.zeros_like(inputs.image, dtype=numpy.int16)
out_img_arr[...] = numpy.around((dB_flt_img_arr * 100), 0)
out_img_arr[dB_flt_img_arr == 999] = 32767
else:
out_img_arr = dB_flt_img_arr
outputs.outimage = out_img_arr
infiles = applier.FilenameAssociations()
infiles.image = input_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.out_int_imgs = out_int_imgs
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdal_format
aControls.omitPyramids = True
aControls.calcStats = False
if gdal_format == 'GTIFF':
aControls.creationoptions = GTIFF_CREATION_OPTS
applier.apply(_apply_calc_dB,
infiles,
outfiles,
otherargs,
controls=aControls)
def indexAndMerge(extentList, extent, wkt, outfile, header):
"""
Internal method to merge all the temporary files into the output
spatially indexing as we go.
"""
controls = lidarprocessor.Controls()
controls.setSpatialProcessing(False)
# open in read mode
driverExtentList = []
for fname, subExtent in extentList:
userClass = lidarprocessor.LidarFile(fname, generic.READ)
driver = spdv4.SPDV4File(fname, generic.READ, controls, userClass)
data = (subExtent, driver)
driverExtentList.append(data)
# create output file
userClass = lidarprocessor.LidarFile(outfile, generic.CREATE)
userClass.setLiDARDriverOption('SCALING_BUT_NO_DATA_WARNING', False)
controls = lidarprocessor.Controls()
controls.setSpatialProcessing(True)
outDriver = spdv4.SPDV4File(outfile, generic.CREATE, controls, userClass)
pixGrid = pixelgrid.PixelGridDefn(xMin=extent.xMin, xMax=extent.xMax,
yMin=extent.yMin, yMax=extent.yMax, projection=wkt,
xRes=extent.binSize, yRes=extent.binSize)
outDriver.setPixelGrid(pixGrid)
# update header
nrows,ncols = pixGrid.getDimensions()
header['NUMBER_BINS_X'] = ncols
header['NUMBER_BINS_Y'] = nrows
# clobber these values since we don't want to
# start with the number in the original file
# they will be reset to 0 in the new file
del header['NUMBER_OF_POINTS']
del header['NUMBER_OF_PULSES']
# these too
del header['GENERATING_SOFTWARE']
del header['CREATION_DATETIME']
progress = cuiprogress.GDALProgressBar()
progress.setLabelText('Merging...')
progress.setTotalSteps(len(extentList))
progress.setProgress(0)
nFilesProcessed = 0
nFilesWritten = 0
for subExtent, driver in driverExtentList:
# read in all the data
# NOTE: can't write data in blocks as the driver needs to be able to
# sort all the data in one go.
bDataWritten = False
npulses = driver.getTotalNumberPulses()
if npulses > 0:
pulseRange = generic.PulseRange(0, npulses)
driver.setPulseRange(pulseRange)
pulses = driver.readPulsesForRange()
points = driver.readPointsByPulse()
waveformInfo = driver.readWaveformInfo()
recv = driver.readReceived()
trans = driver.readTransmitted()
outDriver.setExtent(subExtent)
if nFilesWritten == 0:
copyScaling(driver, outDriver)
outDriver.setHeader(header)
# on create, a spatial index is created
outDriver.writeData(pulses, points, trans, recv,
waveformInfo)
nFilesWritten += 1
# close the driver while we are here
driver.close()
if bDataWritten:
nFilesWritten += 1
nFilesProcessed += 1
progress.setProgress(nFilesProcessed)
outDriver.close()
def get_ST_masks(json_fp, bands=None, roi_img=None, gdal_format='KEA', num_processes=1, threshold=3):
"""Main function to run to generate the output masks. Given an input JSON file,
generates a mask for each date, for each band where 0=Inlier, 1=High outlier,
-1=Low outlier. Opening/closing of files, generation of blocks and use of
multiprocessing is all handled by RIOS.
A minimum of 12 observations is required to create the masks.
json_fp: Path to JSON file which provides a dictionary where for each
date, an input file name and an output file name are provided.
gdal_format: Short driver name for GDAL, e.g. KEA, GTiff.
num_processes: Number of concurrent processes to use.
bands: List of GDAL band numbers to use, e.g. [1, 3, 5]. Defaults to all.
threshold: Threshold for screening. Defaults to 3, meaning that observations
outside 3*RMSE of the fitted model will be counted as outliers.
Lower values will result in more outliers being detected.
"""
ip_paths = []
op_paths = []
dates = []
try:
# Open and read JSON file containing date:filepath pairs
with open(json_fp) as json_file:
image_list = json.load(json_file)
for date in image_list.items():
dates.append([datetime.strptime(date[0], '%Y-%m-%d').toordinal()])
ip_paths.append(date[1]['input'])
op_paths.append(date[1]['output'])
except FileNotFoundError:
print('Could not find the provided JSON file.')
sys.exit()
except json.decoder.JSONDecodeError as e:
print('There is an error in the provided JSON file: {}'.format(e))
sys.exit()
# Create object to hold input files
infiles = applier.FilenameAssociations()
infiles.images = ip_paths
# Create object to hold output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = op_paths
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set window size to 1 because we are working per-pixel
app.setWindowXsize(1)
app.setWindowYsize(1)
# Set progress
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
app.progress = progress_bar
# Set output file type
app.setOutputDriverName(gdal_format)
if roi_img is not None:
app.setReferenceImage(roi_img)
app.setFootprintType(applier.BOUNDS_FROM_REFERENCE)
app.setResampleMethod('near')
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Open first image in list to use as a template
template_image = fileinfo.ImageInfo(infiles.images[0])
# Get no data value
nodata = template_image.nodataval[0]
if not bands: # No bands specified - default to all
num_bands = template_image.rasterCount
bands = [i for i in range(1, num_bands + 1)]
else: # If a list of bands is provided
# Number of bands determines things like the size of the output array
num_bands = len(bands)
# Need to tell the applier to only use the specified bands
app.selectInputImageLayers(bands)
full_names = [template_image.layerNameFromNumber(i) for i in bands]
# Set up output layer name
app.setLayerNames(full_names)
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.dates = dates
other_args.threshold = threshold
other_args.nodata = nodata
other_args.num_bands = num_bands
template_image = None
try:
applier.apply(_gen_band_masks, infiles, outfiles, otherArgs=other_args, controls=app)
except RuntimeError as e:
print('There was an error processing the images: {}'.format(e))
print('Do all images in the JSON file exist?')
def perform_analysis(self, scn_db_obj, sen_obj, plgin_objs):
logger.info("Processing Scene: {}".format(scn_db_obj.PID))
if scn_db_obj.Invalid:
return False, None, False
rsgis_utils = rsgislib.RSGISPyUtils()
eodd_utils = EODataDownUtils()
success = True
outputs = False
out_dict = None
if 'GenChngSummaryFeats' in plgin_objs:
if plgin_objs['GenChngSummaryFeats'].Completed and plgin_objs[
'GenChngSummaryFeats'].Outputs and plgin_objs[
'GenChngSummaryFeats'].Success:
scn_chng_info = plgin_objs['GenChngSummaryFeats'].ExtendedInfo
scn_unq_name = sen_obj.get_scn_unq_name_record(scn_db_obj)
out_vec_file = os.path.join(
self.params['outvecdir'],
"{}_chng_vec.gpkg".format(scn_unq_name))
if os.path.exists(out_vec_file):
delete_vector_file(out_vec_file)
if sen_obj.get_sensor_name() == 'LandsatGOOG':
scn_obs_date = scn_db_obj.Sensing_Time
elif sen_obj.get_sensor_name() == 'Sentinel2GOOG':
scn_obs_date = scn_db_obj.Sensing_Time
elif sen_obj.get_sensor_name() == 'Sentinel1ASF':
scn_obs_date = scn_db_obj.Acquisition_Date
else:
raise Exception("Did not recognise the sensor name...")
start_date = datetime.datetime(year=2019, month=4, day=30)
if scn_obs_date > start_date:
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
drv = gdal.GetDriverByName("GPKG")
if drv is None:
raise Exception("Driver GPKG is not avaiable.")
ds = drv.Create(out_vec_file, 0, 0, 0, gdal.GDT_Unknown)
if ds is None:
raise Exception(
"Could not create output file: {}.".format(
out_vec_file))
out_dict = dict()
for tile in scn_chng_info:
logger.debug("Processing tile {}...".format(tile))
clumps_img = scn_chng_info[tile]
in_rats = ratapplier.RatAssociations()
out_rats = ratapplier.RatAssociations()
in_rats.inrat = ratapplier.RatHandle(clumps_img)
lyr = ds.CreateLayer(tile, None, ogr.wkbPoint)
if lyr is None:
raise Exception(
"Could not create output layer: {}.".format(
tile))
field_uid_defn = ogr.FieldDefn("uid", ogr.OFTInteger)
if lyr.CreateField(field_uid_defn) != 0:
raise Exception("Could not create field: 'uid'.")
field_prop_chng_defn = ogr.FieldDefn(
"prop_chng", ogr.OFTReal)
if lyr.CreateField(field_prop_chng_defn) != 0:
raise Exception(
"Could not create field: 'prop_chng'.")
field_score_defn = ogr.FieldDefn(
"score", ogr.OFTInteger)
if lyr.CreateField(field_score_defn) != 0:
raise Exception("Could not create field: 'score'.")
# First Observation Date
field_firstobsday_defn = ogr.FieldDefn(
"firstobsday", ogr.OFTInteger)
if lyr.CreateField(field_firstobsday_defn) != 0:
raise Exception(
"Could not create field: 'firstobsday'.")
field_firstobsmonth_defn = ogr.FieldDefn(
"firstobsmonth", ogr.OFTInteger)
if lyr.CreateField(field_firstobsmonth_defn) != 0:
raise Exception(
"Could not create field: 'firstobsmonth'.")
field_firstobsyear_defn = ogr.FieldDefn(
"firstobsyear", ogr.OFTInteger)
if lyr.CreateField(field_firstobsyear_defn) != 0:
raise Exception(
"Could not create field: 'firstobsyear'.")
# Last Observation Date
field_lastobsday_defn = ogr.FieldDefn(
"lastobsday", ogr.OFTInteger)
if lyr.CreateField(field_lastobsday_defn) != 0:
raise Exception(
"Could not create field: 'lastobsday'.")
field_lastobsmonth_defn = ogr.FieldDefn(
"lastobsmonth", ogr.OFTInteger)
if lyr.CreateField(field_lastobsmonth_defn) != 0:
raise Exception(
"Could not create field: 'lastobsmonth'.")
field_lastobsyear_defn = ogr.FieldDefn(
"lastobsyear", ogr.OFTInteger)
if lyr.CreateField(field_lastobsyear_defn) != 0:
raise Exception(
"Could not create field: 'lastobsyear'.")
# Observation Date Where Score Reached 5
field_scr5obsday_defn = ogr.FieldDefn(
"scr5obsday", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsday_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsday'.")
field_scr5obsmonth_defn = ogr.FieldDefn(
"scr5obsmonth", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsmonth_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsmonth'.")
field_scr5obsyear_defn = ogr.FieldDefn(
"scr5obsyear", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsyear_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsyear'.")
lyr_defn = lyr.GetLayerDefn()
otherargs = ratapplier.OtherArguments()
otherargs.lyr = lyr
otherargs.lyr_defn = lyr_defn
ratcontrols = ratapplier.RatApplierControls()
ratcontrols.setProgress(progress_bar)
ratapplier.apply(_ratapplier_check_string_col_valid,
in_rats,
out_rats,
otherargs,
controls=ratcontrols)
# Update (create) the JSON LUT file.
lut_file_name = "gmw_{}_lut.json".format(tile)
lut_file_path = os.path.join(self.params["outlutdir"],
lut_file_name)
eodd_utils.get_file_lock(lut_file_path,
sleep_period=1,
wait_iters=120,
use_except=True)
if os.path.exists(lut_file_path):
lut_dict = rsgis_utils.readJSON2Dict(lut_file_path)
else:
lut_dict = dict()
obs_date_iso_str = scn_obs_date.isoformat()
lut_dict[obs_date_iso_str] = dict()
lut_dict[obs_date_iso_str]["file"] = out_vec_file
lut_dict[obs_date_iso_str]["layer"] = tile
rsgis_utils.writeDict2JSON(lut_dict, lut_file_path)
eodd_utils.release_file_lock(lut_file_path)
out_dict[tile] = out_vec_file
ds = None
outputs = True
success = True
else:
logger.debug(
"Scene is within window used to mask change outside of range."
)
else:
logger.debug(
"No change features available as outputs from previous steps..."
)
else:
logger.debug(
"GenChngSummaryFeats was not available so previous step had not run..."
)
return success, out_dict, outputs
def apply_sklearn_classifer(classTrainInfo,
skClassifier,
imgMask,
imgMaskVal,
imgFileInfo,
outputImg,
gdalformat,
classClrNames=True):
"""
This function uses a trained classifier and applies it to the provided input image.
:param classTrainInfo: dict (where the key is the class name) of rsgislib.classification.ClassSimpleInfoObj
objects which will be used to train the classifier (i.e., train_sklearn_classifier()),
provide pixel value id and RGB class values.
:param skClassifier: a trained instance of a scikit-learn classifier
(e.g., use train_sklearn_classifier or train_sklearn_classifer_gridsearch)
:param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all
the valid data regions (rsgislib.imageutils.genValidMask)
:param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied.
Can be used to create a heirachical classification.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to
be used for the classification so it adheres to the training data.
:param outputImg: output image file with the classification. Note. by default a colour table and class names column
is added to the image. If an error is produced use HFA or KEA formats.
:param gdalformat: is the output image format - all GDAL supported formats are supported.
:param classClrNames: default is True and therefore a colour table will the colours specified in classTrainInfo
and a ClassName column (from imgFileInfo) will be added to the output file.
"""
infiles = applier.FilenameAssociations()
infiles.imageMask = imgMask
numClassVars = 0
for imgFile in imgFileInfo:
infiles.__dict__[imgFile.name] = imgFile.fileName
numClassVars = numClassVars + len(imgFile.bands)
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.classifier = skClassifier
otherargs.mskVal = imgMaskVal
otherargs.numClassVars = numClassVars
otherargs.imgFileInfo = imgFileInfo
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
# RIOS function to apply classifer
def _applySKClassifier(info, inputs, outputs, otherargs):
"""
Internal function for rios applier. Used within applyClassifer.
"""
outClassVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint32)
if numpy.any(inputs.imageMask == otherargs.mskVal):
outClassVals = outClassVals.flatten()
imgMaskVals = inputs.imageMask.flatten()
classVars = numpy.zeros(
(outClassVals.shape[0], otherargs.numClassVars),
dtype=numpy.float)
# Array index which can be used to populate the output array following masking etc.
ID = numpy.arange(imgMaskVals.shape[0])
classVarsIdx = 0
for imgFile in otherargs.imgFileInfo:
imgArr = inputs.__dict__[imgFile.name]
for band in imgFile.bands:
classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten()
classVarsIdx = classVarsIdx + 1
classVars = classVars[imgMaskVals == otherargs.mskVal]
ID = ID[imgMaskVals == otherargs.mskVal]
predClass = otherargs.classifier.predict(classVars)
outClassVals[ID] = predClass
outClassVals = numpy.expand_dims(outClassVals.reshape(
(inputs.imageMask.shape[1], inputs.imageMask.shape[2])),
axis=0)
outputs.outimage = outClassVals
print("Applying the Classifier")
applier.apply(_applySKClassifier,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed")
rsgislib.rastergis.populateStats(clumps=outputImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
if classClrNames:
ratDataset = gdal.Open(outputImg, gdal.GA_Update)
red = rat.readColumn(ratDataset, 'Red')
green = rat.readColumn(ratDataset, 'Green')
blue = rat.readColumn(ratDataset, 'Blue')
ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255'))
for classKey in classTrainInfo:
print("Apply Colour to class \'" + classKey + "\'")
red[classTrainInfo[classKey].id] = classTrainInfo[classKey].red
green[classTrainInfo[classKey].id] = classTrainInfo[classKey].green
blue[classTrainInfo[classKey].id] = classTrainInfo[classKey].blue
ClassName[classTrainInfo[classKey].id] = classKey
rat.writeColumn(ratDataset, "Red", red)
rat.writeColumn(ratDataset, "Green", green)
rat.writeColumn(ratDataset, "Blue", blue)
rat.writeColumn(ratDataset, "ClassName", ClassName)
ratDataset = None
def doClump(infile, outfile, tempDir):
"""
Do the clumping
"""
inputs = applier.FilenameAssociations()
inputs.infile = infile
# create temporary file with the clumps done on a per tile basis
outputs = applier.FilenameAssociations()
fileh, tmpClump = tempfile.mkstemp('.kea', dir=tempDir)
os.close(fileh)
outputs.outfile = tmpClump
# start at clumpid 1 - will be zeros where no data
otherinputs = applier.OtherInputs()
otherinputs.clumpId = 1
controls = applier.ApplierControls()
controls.progress = cuiprogress.GDALProgressBar()
# don't need stats for this since it is just temporary
controls.calcStats = False
applier.apply(riosClump, inputs, outputs, otherinputs, controls=controls)
# run it on the input image again, but also read in the tile clumps
inputs.tileclump = tmpClump
# overlap of 1 so we can work out which neighbouring
# clumps need to be merged
controls.overlap = 1
# make thematic
# but still don't do stats - only when we finally know we succeeded
controls.thematic = True
outputs = applier.FilenameAssociations()
finished = False
while not finished:
# it creates the output file as it goes
# just create temp at this stage until we know it has succeeded
fileh, tmpMerged = tempfile.mkstemp('.kea', dir=tempDir)
os.close(fileh)
outputs.clump = tmpMerged
otherinputs.nFailedRecodes = 0
# ok now we have to merge the clumps
# create a recode table
recode = numpy.arange(otherinputs.clumpId, dtype=numpy.uint32)
otherinputs.recode = recode
# create a boolean array with clumps not to recode
# obviously if you recode 39 -> 23 you don't want 23 being recoded to
# something else
dontrecode = numpy.zeros_like(recode, dtype=numpy.bool)
otherinputs.dontrecode = dontrecode
applier.apply(riosMerge,
inputs,
outputs,
otherinputs,
controls=controls)
# clobber the last temp input
os.remove(inputs.tileclump)
inputs.tileclump = tmpMerged
dontrecodesum = dontrecode.sum()
finished = dontrecodesum == 0
if not finished:
print('%d clumps failed to merge. %d recoded' %
(otherinputs.nFailedRecodes, dontrecodesum))
print('having another go')
# now we save the final output as the output name and calc stats
cmd = 'gdalcalcstats %s -ignore 0' % tmpMerged
os.system(cmd)
# use move rather than rename in case we are on different filesystems
shutil.move(tmpMerged, outfile)
# just be careful here since the permissions will be set strangely
# for outfile since it was created by tempfile. Set to match current umask
current_umask = os.umask(0)
os.umask(current_umask) # can't get without setting!
# need to convert from umask to mode and remove exe bits
mode = 0o0777 & ~current_umask
mode = (((mode ^ stat.S_IXUSR) ^ stat.S_IXGRP) ^ stat.S_IXOTH)
os.chmod(outfile, mode)
history.insertMetadataFilename(outfile, [infile], {})