def calcImgsPxlMode(inputImgs, outputImg, gdalformat, no_data_val=0):
"""
Function which calculates the mode of a group of images.
Warning, this function can be very slow. Use rsgislib.imagecalc.imagePixelColumnSummary
:param inputImgs: the list of images
:param outputImg: the output image file name and path (will be same dimensions as the input)
:param gdalformat: the GDAL image file format of the output image file.
"""
import scipy.stats
from rios import applier
rsgis_utils = rsgislib.RSGISPyUtils()
datatype = rsgis_utils.getRSGISLibDataTypeFromImg(inputImgs[0])
numpyDT = rsgis_utils.getNumpyDataType(datatype)
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.images = inputImgs
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.no_data_val = no_data_val
otherargs.numpyDT = numpyDT
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyCalcMode(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
image_data = numpy.concatenate(inputs.images,
axis=0).astype(numpy.float32)
image_data[image_data == otherargs.no_data_val] = numpy.nan
mode_arr, count_arr = scipy.stats.mode(image_data,
axis=0,
nan_policy='omit')
outputs.outimage = mode_arr.astype(otherargs.numpyDT)
applier.apply(_applyCalcMode,
infiles,
outfiles,
otherargs,
controls=aControls)
def update_uid_image(uid_img, chng_img, nxt_scr5_img, clrsky_img,
obs_day_since_base, tmp_uid_tile):
from rios import applier
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.uid_img = uid_img
infiles.chng_img = chng_img
infiles.nxt_scr5_img = nxt_scr5_img
infiles.clrsky_img = clrsky_img
outfiles = applier.FilenameAssociations()
outfiles.uid_img_out = tmp_uid_tile
otherargs = applier.OtherInputs()
otherargs.obs_day_since_base = obs_day_since_base
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.omitPyramids = True
aControls.calcStats = False
def _update_uid_img(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
uid_img_arr = numpy.array(inputs.uid_img,
dtype=numpy.uint32,
copy=True)
start_year = inputs.uid_img[0]
chng_feats = inputs.chng_img[0]
new_chng_pxls = numpy.zeros_like(start_year)
new_chng_pxls[(start_year == 0) & (chng_feats == 1)] = 1
chng_scr5_year = inputs.uid_img[4]
new_scr5_pxls = numpy.zeros_like(chng_scr5_year)
new_scr5_pxls[(chng_scr5_year == 0)
& (inputs.nxt_scr5_img[0] == 1)] = 1
uid_img_arr[0, new_chng_pxls == 1] = 1970
uid_img_arr[1, new_chng_pxls == 1] = otherargs.obs_day_since_base
uid_img_arr[2, inputs.clrsky_img[0] == 1] = 1970
uid_img_arr[3,
inputs.clrsky_img[0] == 1] = otherargs.obs_day_since_base
uid_img_arr[4, new_scr5_pxls == 1] = 1970
uid_img_arr[5, new_scr5_pxls == 1] = otherargs.obs_day_since_base
outputs.uid_img_out = uid_img_arr
applier.apply(_update_uid_img,
infiles,
outfiles,
otherargs,
controls=aControls)
def calcWSG84PixelArea(img, out_img, scale=10000, gdalformat='KEA'):
"""
A function which calculates the area (in metres) of the pixel projected in WGS84.
:param img: input image, for which the per-pixel area will be calculated.
:param out_img: output image file.
:param scale: scale the output area to unit of interest. Scale=10000(Ha), Scale=1(sq m), Scale=1000000(sq km), Scale=4046.856(Acre), Scale=2590000(sq miles), Scale=0.0929022668(sq feet)
"""
import rsgislib.tools
from rios import applier
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
rsgis_utils = rsgislib.RSGISPyUtils()
x_res, y_res = rsgis_utils.getImageRes(img)
infiles = applier.FilenameAssociations()
infiles.img = img
outfiles = applier.FilenameAssociations()
outfiles.outimage = out_img
otherargs = applier.OtherInputs()
otherargs.x_res = x_res
otherargs.y_res = y_res
otherargs.scale = float(scale)
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = False
aControls.calcStats = False
def _calcPixelArea(info, inputs, outputs, otherargs):
xBlock, yBlock = info.getBlockCoordArrays()
x_res_arr = numpy.zeros_like(yBlock, dtype=float)
x_res_arr[...] = otherargs.x_res
y_res_arr = numpy.zeros_like(yBlock, dtype=float)
y_res_arr[...] = otherargs.y_res
x_res_arr_m, y_res_arr_m = rsgislib.tools.degrees_to_metres(
yBlock, x_res_arr, y_res_arr)
outputs.outimage = numpy.expand_dims(
(x_res_arr_m * y_res_arr_m) / otherargs.scale, axis=0)
applier.apply(_calcPixelArea,
infiles,
outfiles,
otherargs,
controls=aControls)
def findminclumpval(clumpsImg, valsImg, valBand, out_col):
import rsgislib.rastergis
import rsgislib.rastergis.ratutils
from rios import applier
import numpy
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
n_clumps = rsgislib.rastergis.getRATLength(clumpsImg)
min_vals = numpy.zeros(n_clumps, dtype=int)
infiles = applier.FilenameAssociations()
infiles.clumpsImg = clumpsImg
infiles.valsImg = valsImg
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
otherargs.min_vals = min_vals
otherargs.valBand = valBand - 1
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.omitPyramids = True
aControls.calcStats = False
def _findMinVals(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
unqClumpVals = numpy.unique(inputs.clumpsImg)
clumpValsFlat = inputs.clumpsImg.flatten()
for clump in unqClumpVals:
pxl_vals = inputs.valsImg[otherargs.valBand].flatten()[
clumpValsFlat == clump]
pxl_vals = pxl_vals[pxl_vals > 0]
if pxl_vals.shape[0] > 0:
clump_min_val = pxl_vals.min()
if otherargs.min_vals[clump] == 0:
otherargs.min_vals[clump] = clump_min_val
else:
if clump_min_val < otherargs.min_vals[clump]:
otherargs.min_vals[clump] = clump_min_val
applier.apply(_findMinVals,
infiles,
outfiles,
otherargs,
controls=aControls)
rsgislib.rastergis.ratutils.setColumnData(clumpsImg, out_col, min_vals)
def rescale_unmixing_results(input_img,
output_img,
gdalformat,
scale_factor=1000):
"""
A function which rescales an output from a spectral unmixing
(e.g., rsgislib.imagecalc.specunmixing.calc_unconstrained_unmixing) so that
negative values are removed and each pixel sums to 1.
:param inputImg: Input image with the spectral unmixing result (pixels need to range from 0-1)
:param outputImg: Output image with the result of the rescaling (pixel values will be in range 0-1)
:param gdalformat: the file format of the output file.
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.image = input_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyUnmixRescale(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
inputs.image[inputs.image < 0] = 0
if otherargs.scale_factor == 1:
outputs.outimage = numpy.zeros_like(inputs.image,
dtype=numpy.float32)
else:
outputs.outimage = numpy.zeros_like(inputs.image,
dtype=numpy.uint16)
for idx in range(inputs.image.shape[0]):
outputs.outimage[idx] = (inputs.image[idx] / numpy.sum(
inputs.image, axis=0)) * otherargs.scale_factor
applier.apply(_applyUnmixRescale,
infiles,
outfiles,
otherargs,
controls=aControls)
def apply_sklearn_classifer(classTrainInfo,
skClassifier,
imgMask,
imgMaskVal,
imgFileInfo,
outputImg,
gdalformat,
classClrNames=True):
"""
This function uses a trained classifier and applies it to the provided input image.
:param classTrainInfo: dict (where the key is the class name) of rsgislib.classification.ClassSimpleInfoObj
objects which will be used to train the classifier (i.e., train_sklearn_classifier()),
provide pixel value id and RGB class values.
:param skClassifier: a trained instance of a scikit-learn classifier
(e.g., use train_sklearn_classifier or train_sklearn_classifer_gridsearch)
:param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all
the valid data regions (rsgislib.imageutils.genValidMask)
:param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied.
Can be used to create a heirachical classification.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to
be used for the classification so it adheres to the training data.
:param outputImg: output image file with the classification. Note. by default a colour table and class names column
is added to the image. If an error is produced use HFA or KEA formats.
:param gdalformat: is the output image format - all GDAL supported formats are supported.
:param classClrNames: default is True and therefore a colour table will the colours specified in classTrainInfo
and a ClassName column (from imgFileInfo) will be added to the output file.
"""
infiles = applier.FilenameAssociations()
infiles.imageMask = imgMask
numClassVars = 0
for imgFile in imgFileInfo:
infiles.__dict__[imgFile.name] = imgFile.fileName
numClassVars = numClassVars + len(imgFile.bands)
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.classifier = skClassifier
otherargs.mskVal = imgMaskVal
otherargs.numClassVars = numClassVars
otherargs.imgFileInfo = imgFileInfo
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
# RIOS function to apply classifer
def _applySKClassifier(info, inputs, outputs, otherargs):
"""
Internal function for rios applier. Used within applyClassifer.
"""
outClassVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint32)
if numpy.any(inputs.imageMask == otherargs.mskVal):
outClassVals = outClassVals.flatten()
imgMaskVals = inputs.imageMask.flatten()
classVars = numpy.zeros(
(outClassVals.shape[0], otherargs.numClassVars),
dtype=numpy.float)
# Array index which can be used to populate the output array following masking etc.
ID = numpy.arange(imgMaskVals.shape[0])
classVarsIdx = 0
for imgFile in otherargs.imgFileInfo:
imgArr = inputs.__dict__[imgFile.name]
for band in imgFile.bands:
classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten()
classVarsIdx = classVarsIdx + 1
classVars = classVars[imgMaskVals == otherargs.mskVal]
ID = ID[imgMaskVals == otherargs.mskVal]
predClass = otherargs.classifier.predict(classVars)
outClassVals[ID] = predClass
outClassVals = numpy.expand_dims(outClassVals.reshape(
(inputs.imageMask.shape[1], inputs.imageMask.shape[2])),
axis=0)
outputs.outimage = outClassVals
print("Applying the Classifier")
applier.apply(_applySKClassifier,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed")
rsgislib.rastergis.populateStats(clumps=outputImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
if classClrNames:
ratDataset = gdal.Open(outputImg, gdal.GA_Update)
red = rat.readColumn(ratDataset, 'Red')
green = rat.readColumn(ratDataset, 'Green')
blue = rat.readColumn(ratDataset, 'Blue')
ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255'))
for classKey in classTrainInfo:
print("Apply Colour to class \'" + classKey + "\'")
red[classTrainInfo[classKey].id] = classTrainInfo[classKey].red
green[classTrainInfo[classKey].id] = classTrainInfo[classKey].green
blue[classTrainInfo[classKey].id] = classTrainInfo[classKey].blue
ClassName[classTrainInfo[classKey].id] = classKey
rat.writeColumn(ratDataset, "Red", red)
rat.writeColumn(ratDataset, "Green", green)
rat.writeColumn(ratDataset, "Blue", blue)
rat.writeColumn(ratDataset, "ClassName", ClassName)
ratDataset = None
def apply_keras_pixel_classifier(classTrainInfo,
keras_cls_mdl,
imgMask,
imgMaskVal,
imgFileInfo,
outClassImg,
gdalformat,
pred_batch_size=32,
classClrNames=True):
"""
This function applies a trained single pixel keras model to an image. The function train_keras_pixel_classifer
can be used to train such as model. The output image will contain the hard membership of the predicted class.
:param classTrainInfo: dict (where the key is the class name) of rsgislib.classification.ClassInfoObj
objects which will be used to train the classifier (i.e., train_keras_pixel_classifer()),
provide pixel value id and RGB class values.
:param keras_cls_mdl: a trained keras model object, with a input dimensions equivlent to the number of image
bands specified in the imgFileInfo input and output layer which provides an output array
of the length of the number of classes.
:param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the
valid data regions (rsgislib.imageutils.genValidMask)
:param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied.
Can be used to create a heirachical classification.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to
be used for the classification so it adheres to the training data.
:param outClassImg: Output image which will contain the hard classification.
:param gdalformat: is the output image format - all GDAL supported formats are supported.
:param pred_batch_size: the batch size used for the classification.
:param classClrNames: default is True and therefore a colour table will the colours specified in ClassInfoObj
and a ClassName (from classTrainInfo) column will be added to the output file.
"""
def _applyKerasPxlClassifier(info, inputs, outputs, otherargs):
outClassIdVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint16)
if numpy.any(inputs.imageMask == otherargs.mskVal):
n_pxls = inputs.imageMask.shape[1] * inputs.imageMask.shape[2]
outClassIdVals = outClassIdVals.flatten()
imgMaskVals = inputs.imageMask.flatten()
classVars = numpy.zeros((n_pxls, otherargs.numClassVars),
dtype=numpy.float)
# Array index which can be used to populate the output array following masking etc.
ID = numpy.arange(imgMaskVals.shape[0])
classVarsIdx = 0
for imgFile in otherargs.imgFileInfo:
imgArr = inputs.__dict__[imgFile.name]
for band in imgFile.bands:
classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten()
classVarsIdx = classVarsIdx + 1
classVars = classVars[imgMaskVals == otherargs.mskVal]
ID = ID[imgMaskVals == otherargs.mskVal]
preds_idxs = numpy.argmax(otherargs.classifier.predict(
classVars, batch_size=otherargs.pred_batch_size),
axis=1)
preds_cls_ids = numpy.zeros_like(preds_idxs, dtype=numpy.uint16)
for cld_id, idx in zip(otherargs.cls_id_lut,
numpy.arange(0, len(otherargs.cls_id_lut))):
preds_cls_ids[preds_idxs == idx] = cld_id
outClassIdVals[ID] = preds_cls_ids
outClassIdVals = numpy.expand_dims(outClassIdVals.reshape(
(inputs.imageMask.shape[1], inputs.imageMask.shape[2])),
axis=0)
outputs.outclsimage = outClassIdVals
infiles = applier.FilenameAssociations()
infiles.imageMask = imgMask
numClassVars = 0
for imgFile in imgFileInfo:
infiles.__dict__[imgFile.name] = imgFile.fileName
numClassVars = numClassVars + len(imgFile.bands)
n_classes = len(classTrainInfo)
cls_id_lut = numpy.zeros(n_classes)
for clsname in classTrainInfo:
if classTrainInfo[clsname].id >= n_classes:
raise (
"ClassInfoObj '{}' id ({}) is not consecutive starting from 0."
.format(clsname, classTrainInfo[clsname].id))
cls_id_lut[classTrainInfo[clsname].id] = classTrainInfo[clsname].out_id
outfiles = applier.FilenameAssociations()
outfiles.outclsimage = outClassImg
otherargs = applier.OtherInputs()
otherargs.classifier = keras_cls_mdl
otherargs.pred_batch_size = pred_batch_size
otherargs.mskVal = imgMaskVal
otherargs.numClassVars = numClassVars
otherargs.imgFileInfo = imgFileInfo
otherargs.n_classes = n_classes
otherargs.cls_id_lut = cls_id_lut
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Applying the Classifier")
applier.apply(_applyKerasPxlClassifier,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed Classification")
if classClrNames:
rsgislib.rastergis.populateStats(outClassImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
ratDataset = gdal.Open(outClassImg, gdal.GA_Update)
red = rat.readColumn(ratDataset, 'Red')
green = rat.readColumn(ratDataset, 'Green')
blue = rat.readColumn(ratDataset, 'Blue')
ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255'))
ClassName[...] = ""
for classKey in classTrainInfo:
print("Apply Colour to class \'" + classKey + "\'")
red[classTrainInfo[classKey].out_id] = classTrainInfo[classKey].red
green[classTrainInfo[classKey].
out_id] = classTrainInfo[classKey].green
blue[classTrainInfo[classKey].
out_id] = classTrainInfo[classKey].blue
ClassName[classTrainInfo[classKey].out_id] = classKey
rat.writeColumn(ratDataset, "Red", red)
rat.writeColumn(ratDataset, "Green", green)
rat.writeColumn(ratDataset, "Blue", blue)
rat.writeColumn(ratDataset, "ClassName", ClassName)
ratDataset = None
def get_ST_masks(json_fp, bands=None, roi_img=None, gdal_format='KEA', num_processes=1, threshold=3):
"""Main function to run to generate the output masks. Given an input JSON file,
generates a mask for each date, for each band where 0=Inlier, 1=High outlier,
-1=Low outlier. Opening/closing of files, generation of blocks and use of
multiprocessing is all handled by RIOS.
A minimum of 12 observations is required to create the masks.
json_fp: Path to JSON file which provides a dictionary where for each
date, an input file name and an output file name are provided.
gdal_format: Short driver name for GDAL, e.g. KEA, GTiff.
num_processes: Number of concurrent processes to use.
bands: List of GDAL band numbers to use, e.g. [1, 3, 5]. Defaults to all.
threshold: Threshold for screening. Defaults to 3, meaning that observations
outside 3*RMSE of the fitted model will be counted as outliers.
Lower values will result in more outliers being detected.
"""
ip_paths = []
op_paths = []
dates = []
try:
# Open and read JSON file containing date:filepath pairs
with open(json_fp) as json_file:
image_list = json.load(json_file)
for date in image_list.items():
dates.append([datetime.strptime(date[0], '%Y-%m-%d').toordinal()])
ip_paths.append(date[1]['input'])
op_paths.append(date[1]['output'])
except FileNotFoundError:
print('Could not find the provided JSON file.')
sys.exit()
except json.decoder.JSONDecodeError as e:
print('There is an error in the provided JSON file: {}'.format(e))
sys.exit()
# Create object to hold input files
infiles = applier.FilenameAssociations()
infiles.images = ip_paths
# Create object to hold output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = op_paths
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set window size to 1 because we are working per-pixel
app.setWindowXsize(1)
app.setWindowYsize(1)
# Set progress
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
app.progress = progress_bar
# Set output file type
app.setOutputDriverName(gdal_format)
if roi_img is not None:
app.setReferenceImage(roi_img)
app.setFootprintType(applier.BOUNDS_FROM_REFERENCE)
app.setResampleMethod('near')
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Open first image in list to use as a template
template_image = fileinfo.ImageInfo(infiles.images[0])
# Get no data value
nodata = template_image.nodataval[0]
if not bands: # No bands specified - default to all
num_bands = template_image.rasterCount
bands = [i for i in range(1, num_bands + 1)]
else: # If a list of bands is provided
# Number of bands determines things like the size of the output array
num_bands = len(bands)
# Need to tell the applier to only use the specified bands
app.selectInputImageLayers(bands)
full_names = [template_image.layerNameFromNumber(i) for i in bands]
# Set up output layer name
app.setLayerNames(full_names)
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.dates = dates
other_args.threshold = threshold
other_args.nodata = nodata
other_args.num_bands = num_bands
template_image = None
try:
applier.apply(_gen_band_masks, infiles, outfiles, otherArgs=other_args, controls=app)
except RuntimeError as e:
print('There was an error processing the images: {}'.format(e))
print('Do all images in the JSON file exist?')
def apply_xgboost_binary_classifier(model_file,
imgMask,
imgMaskVal,
imgFileInfo,
outProbImg,
gdalformat,
outClassImg=None,
class_thres=5000,
nthread=1):
"""
This function applies a trained binary (i.e., two classes) xgboost model. The function train_xgboost_binary_classifer
can be used to train such as model. The output image will contain the probability of membership to the class of
interest. You will need to threshold this image to get a final hard classification. Alternative, a hard class output
image and threshold can be applied to this image.
:param model_file: a trained xgboost binary model which can be loaded with lgb.Booster(model_file=model_file).
:param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the
valid data regions (rsgislib.imageutils.genValidMask)
:param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied.
Can be used to create a heirachical classification.
:param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within
rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to
be used for the classification so it adheres to the training data.
:param outProbImg: output image file with the classification probabilities - this image is scaled by
multiplying by 10000.
:param gdalformat: is the output image format - all GDAL supported formats are supported.
:param outClassImg: Optional output image which will contain the hard classification, defined with a threshold on the
probability image.
:param class_thres: The threshold used to define the hard classification. Default is 5000 (i.e., probability of 0.5).
:param nthread: The number of threads to use for the classifier.
"""
def _applyXGBClassifier(info, inputs, outputs, otherargs):
outClassVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint16)
if numpy.any(inputs.imageMask == otherargs.mskVal):
outClassVals = outClassVals.flatten()
imgMaskVals = inputs.imageMask.flatten()
classVars = numpy.zeros(
(outClassVals.shape[0], otherargs.numClassVars),
dtype=numpy.float)
# Array index which can be used to populate the output array following masking etc.
ID = numpy.arange(imgMaskVals.shape[0])
classVarsIdx = 0
for imgFile in otherargs.imgFileInfo:
imgArr = inputs.__dict__[imgFile.name]
for band in imgFile.bands:
classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten()
classVarsIdx = classVarsIdx + 1
classVars = classVars[imgMaskVals == otherargs.mskVal]
ID = ID[imgMaskVals == otherargs.mskVal]
predClass = numpy.around(
otherargs.classifier.predict(xgb.DMatrix(classVars)) * 10000)
outClassVals[ID] = predClass
outClassVals = numpy.expand_dims(outClassVals.reshape(
(inputs.imageMask.shape[1], inputs.imageMask.shape[2])),
axis=0)
outputs.outimage = outClassVals
classifier = xgb.Booster({'nthread': nthread})
classifier.load_model(model_file)
infiles = applier.FilenameAssociations()
infiles.imageMask = imgMask
numClassVars = 0
for imgFile in imgFileInfo:
infiles.__dict__[imgFile.name] = imgFile.fileName
numClassVars = numClassVars + len(imgFile.bands)
outfiles = applier.FilenameAssociations()
outfiles.outimage = outProbImg
otherargs = applier.OtherInputs()
otherargs.classifier = classifier
otherargs.mskVal = imgMaskVal
otherargs.numClassVars = numClassVars
otherargs.imgFileInfo = imgFileInfo
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Applying the Classifier")
applier.apply(_applyXGBClassifier,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed")
rsgislib.imageutils.popImageStats(outProbImg,
usenodataval=True,
nodataval=0,
calcpyramids=True)
if outClassImg is not None:
rsgislib.imagecalc.imageMath(outProbImg, outClassImg,
'b1>{}?1:0'.format(class_thres),
gdalformat, rsgislib.TYPE_8UINT)
rsgislib.rastergis.populateStats(outClassImg,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
def calcImgBasicStats4RefRegion(ref_img,
stats_imgs,
output_img,
gdalformat='KEA'):
"""
A function which calculates the mean and standard deviation through a series of
input images. The region for processing is defined by the reference image and
images padded with no-data where no data is present.
The output image has twice the number of bands as the input image providing
a mean and standard deviation for each input band.
If the input images has 2 bands then the output bands will have the following
order:
1. band 1 mean
2. band 1 std dev
3. band 2 mean
4. band 2 std dev
:param ref_img: reference image which defines the output image
:param stats_imgs: a list of input images over which the stats will be calculated.
:param output_img: the output image path and file name
:param gdalformat: the output image file format. Default KEA.
"""
import rsgislib.imageutils
from rios import applier
rsgis_utils = rsgislib.RSGISPyUtils()
first = True
n_bands = 0
no_data_val = 0
for img in stats_imgs:
print(img)
if first:
n_bands = rsgis_utils.getImageBandCount(img)
no_data_val = rsgis_utils.getImageNoDataValue(img)
first = False
else:
if n_bands != rsgis_utils.getImageBandCount(img):
raise Exception(
"The number of bands must be the same in all input images."
)
if no_data_val != rsgis_utils.getImageNoDataValue(img):
raise Exception(
"The no data value should be the same in all input images."
)
# RIOS internal function to calculate mean and standard deviation of the input images
def _calcBasicStats(info, inputs, outputs, otherargs):
n_imgs = len(inputs.imgs)
blk_shp = inputs.imgs[0].shape
if blk_shp[0] != otherargs.n_bands:
raise Exception(
"Block shape and the number of input image bands do not align."
)
outputs.output_img = numpy.zeros(
(blk_shp[0] * 2, blk_shp[1], blk_shp[2]), dtype=float)
band_arr = []
for band in range(blk_shp[0]):
band_arr.append(
numpy.zeros((n_imgs, blk_shp[1], blk_shp[2]), dtype=float))
img_idx = 0
for img_blk in inputs.imgs:
for band in range(blk_shp[0]):
band_arr[band][img_idx] = img_blk[band]
img_idx = img_idx + 1
for band in range(blk_shp[0]):
band_arr[band][band_arr[band] == otherargs.no_data_val] = numpy.nan
outputs.output_img[band * 2] = numpy.nanmean(band_arr[band],
axis=0)
outputs.output_img[band * 2 + 1] = numpy.nanstd(band_arr[band],
axis=0)
outputs.output_img[band * 2][numpy.isnan(
outputs.output_img[band * 2])] = otherargs.no_data_val
outputs.output_img[band * 2 + 1][numpy.isnan(
outputs.output_img[band * 2 + 1])] = 0.0
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.imgs = stats_imgs
otherargs = applier.OtherInputs()
otherargs.n_bands = n_bands
otherargs.no_data_val = no_data_val
outfiles = applier.FilenameAssociations()
outfiles.output_img = output_img
aControls = applier.ApplierControls()
aControls.referenceImage = ref_img
aControls.footprint = applier.BOUNDS_FROM_REFERENCE
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Calculating Stats Image.")
applier.apply(_calcBasicStats,
infiles,
outfiles,
otherargs,
controls=aControls)
print("Completed")
rsgislib.imageutils.popImageStats(output_img,
usenodataval=True,
nodataval=no_data_val,
calcpyramids=True)
def perform_simple_unmixing(self,
input_image,
output_image,
gdalformat,
endmembers_file,
weight=None,
scale_factor=1000):
"""
A function which uses the RIOS to iterate through the input image
and perform a simple/standard spectral unmixing on the input image
using the calc_abundance function.
:param input_image: The file path to a GDAL readable input image.
:param output_image: The file path to the GDAL writable output image
(Note pixel values will be between 0-1000)
:param gdalformat: The output image format to be used.
:param endmembers_file: The endmembers for the unmixing in the RSGISLib mtxt format.
:param weight: Optional (if None ignored) to provide a weight to implement the approach of Scarth et al (2010)
adding a weight to the least squares optimisation to get the abundances to sum to 1.
:param scale_factor: Scale factor for integerising the resulting image. If value is 1 then output image
will be a floating point image.
References:
Scarth, P., Roder, A., & Schmidt, M. (2010). Tracking grazing pressure and climate
interaction—The role of Landsat fractional cover in time series analysis. Proceedings of
the 15th Australasian Remote Sensing and Photogrammetry ConferenceAustralia: Alice Springs.
http://dx.doi.org/10.6084/m9.figshare.94250.
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
if weight is not None:
endmembers_info = self.read_endmembers_mtxt_weight(
endmembers_file, weight)
else:
endmembers_info = self.read_endmembers_mtxt(endmembers_file)
print(endmembers_info)
infiles = applier.FilenameAssociations()
infiles.image = input_image
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_image
otherargs = applier.OtherInputs()
otherargs.endmembers_q = endmembers_info[0]
otherargs.endmembers_p = endmembers_info[1]
otherargs.endmembers_arr = endmembers_info[2]
otherargs.weight = weight
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _simple_unmix(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
block_shp = inputs.image.shape
img_flat = inputs.image.reshape(
[block_shp[0], (block_shp[1] * block_shp[2])]).T
if otherargs.weight is not None:
img_flat_dtype = img_flat.dtype
tmp_img_flat = img_flat
img_flat = numpy.zeros(
(img_flat.shape[0], img_flat.shape[1] + 1),
dtype=img_flat_dtype)
img_flat[...] = weight
img_flat[:, :-1] = tmp_img_flat
img_flat_shp = img_flat.shape
img_nodata = numpy.where(img_flat == 0, True, False)
img_flat_nodata = numpy.all(img_nodata, axis=1)
ID = numpy.arange(img_flat_shp[0])
ID = ID[img_flat_nodata == False]
img_flat_data = img_flat[img_flat_nodata == False, ...]
abundances_arr = self.calc_abundance(img_flat_data,
otherargs.endmembers_arr)
if otherargs.scale_factor > 1:
outarr = numpy.zeros([img_flat_shp[0], otherargs.endmembers_q],
dtype=numpy.int16)
else:
outarr = numpy.zeros([img_flat_shp[0], otherargs.endmembers_q],
dtype=numpy.float32)
outarr[ID] = (abundances_arr * otherargs.scale_factor)
outarr = outarr.T
outputs.outimage = outarr.reshape(
(otherargs.endmembers_q, block_shp[1], block_shp[2]))
applier.apply(_simple_unmix,
infiles,
outfiles,
otherargs,
controls=aControls)
def rescaleImgPxlVals(inputImg,
outputImg,
gdalformat,
datatype,
bandRescale,
trim2Limits=True):
"""
Function which rescales an input image base on a list of rescaling parameters.
:param inputImg: the input image
:param outputImg: the output image file name and path (will be same dimensions as the input)
:param gdalformat: the GDAL image file format of the output image file.
:param bandRescale: list of ImageBandRescale objects
:param trim2Limits: whether to trim the output to the output min/max values.
"""
from rios import applier
bandRescaleDict = dict()
for rescaleObj in bandRescale:
bandRescaleDict[rescaleObj.band - 1] = rescaleObj
rsgis_utils = rsgislib.RSGISPyUtils()
numpyDT = rsgis_utils.getNumpyDataType(datatype)
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.image = inputImg
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.rescaleDict = bandRescaleDict
otherargs.trim = trim2Limits
otherargs.numpyDT = numpyDT
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyRescale(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpyDT)
for idx in range(inputs.image.shape[0]):
outputs.outimage[idx] = numpy.where(
inputs.image[idx] == otherargs.rescaleDict[idx].inNoData,
otherargs.rescaleDict[idx].outNoData,
(((inputs.image[idx] - otherargs.rescaleDict[idx].inMin) /
(inputs.image[idx] - otherargs.rescaleDict[idx].inMax -
inputs.image[idx] - otherargs.rescaleDict[idx].inMin)) *
(inputs.image[idx] - otherargs.rescaleDict[idx].outMax -
inputs.image[idx] - otherargs.rescaleDict[idx].outMin)) +
inputs.image[idx] - otherargs.rescaleDict[idx].outMin)
if otherargs.trim:
outputs.outimage[idx] = numpy.where(
(outputs.outimage[idx] !=
otherargs.rescaleDict[idx].outNoData) &
(outputs.outimage[idx] <
otherargs.rescaleDict[idx].outMin),
otherargs.rescaleDict[idx].outMin, outputs.outimage[idx])
outputs.outimage[idx] = numpy.where(
(outputs.outimage[idx] !=
otherargs.rescaleDict[idx].outNoData) &
(outputs.outimage[idx] >
otherargs.rescaleDict[idx].outMax),
otherargs.rescaleDict[idx].outMax, outputs.outimage[idx])
applier.apply(_applyRescale,
infiles,
outfiles,
otherargs,
controls=aControls)
def perform_analysis(self, scn_db_obj, sen_obj, plgin_objs):
logger.info("Processing Scene: {}".format(scn_db_obj.PID))
if scn_db_obj.Invalid:
return False, None, False
rsgis_utils = rsgislib.RSGISPyUtils()
eodd_utils = EODataDownUtils()
success = True
outputs = False
out_dict = None
if 'GenChngSummaryFeats' in plgin_objs:
if plgin_objs['GenChngSummaryFeats'].Completed and plgin_objs[
'GenChngSummaryFeats'].Outputs and plgin_objs[
'GenChngSummaryFeats'].Success:
scn_chng_info = plgin_objs['GenChngSummaryFeats'].ExtendedInfo
scn_unq_name = sen_obj.get_scn_unq_name_record(scn_db_obj)
out_vec_file = os.path.join(
self.params['outvecdir'],
"{}_chng_vec.gpkg".format(scn_unq_name))
if os.path.exists(out_vec_file):
delete_vector_file(out_vec_file)
if sen_obj.get_sensor_name() == 'LandsatGOOG':
scn_obs_date = scn_db_obj.Sensing_Time
elif sen_obj.get_sensor_name() == 'Sentinel2GOOG':
scn_obs_date = scn_db_obj.Sensing_Time
elif sen_obj.get_sensor_name() == 'Sentinel1ASF':
scn_obs_date = scn_db_obj.Acquisition_Date
else:
raise Exception("Did not recognise the sensor name...")
start_date = datetime.datetime(year=2019, month=4, day=30)
if scn_obs_date > start_date:
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
drv = gdal.GetDriverByName("GPKG")
if drv is None:
raise Exception("Driver GPKG is not avaiable.")
ds = drv.Create(out_vec_file, 0, 0, 0, gdal.GDT_Unknown)
if ds is None:
raise Exception(
"Could not create output file: {}.".format(
out_vec_file))
out_dict = dict()
for tile in scn_chng_info:
logger.debug("Processing tile {}...".format(tile))
clumps_img = scn_chng_info[tile]
in_rats = ratapplier.RatAssociations()
out_rats = ratapplier.RatAssociations()
in_rats.inrat = ratapplier.RatHandle(clumps_img)
lyr = ds.CreateLayer(tile, None, ogr.wkbPoint)
if lyr is None:
raise Exception(
"Could not create output layer: {}.".format(
tile))
field_uid_defn = ogr.FieldDefn("uid", ogr.OFTInteger)
if lyr.CreateField(field_uid_defn) != 0:
raise Exception("Could not create field: 'uid'.")
field_prop_chng_defn = ogr.FieldDefn(
"prop_chng", ogr.OFTReal)
if lyr.CreateField(field_prop_chng_defn) != 0:
raise Exception(
"Could not create field: 'prop_chng'.")
field_score_defn = ogr.FieldDefn(
"score", ogr.OFTInteger)
if lyr.CreateField(field_score_defn) != 0:
raise Exception("Could not create field: 'score'.")
# First Observation Date
field_firstobsday_defn = ogr.FieldDefn(
"firstobsday", ogr.OFTInteger)
if lyr.CreateField(field_firstobsday_defn) != 0:
raise Exception(
"Could not create field: 'firstobsday'.")
field_firstobsmonth_defn = ogr.FieldDefn(
"firstobsmonth", ogr.OFTInteger)
if lyr.CreateField(field_firstobsmonth_defn) != 0:
raise Exception(
"Could not create field: 'firstobsmonth'.")
field_firstobsyear_defn = ogr.FieldDefn(
"firstobsyear", ogr.OFTInteger)
if lyr.CreateField(field_firstobsyear_defn) != 0:
raise Exception(
"Could not create field: 'firstobsyear'.")
# Last Observation Date
field_lastobsday_defn = ogr.FieldDefn(
"lastobsday", ogr.OFTInteger)
if lyr.CreateField(field_lastobsday_defn) != 0:
raise Exception(
"Could not create field: 'lastobsday'.")
field_lastobsmonth_defn = ogr.FieldDefn(
"lastobsmonth", ogr.OFTInteger)
if lyr.CreateField(field_lastobsmonth_defn) != 0:
raise Exception(
"Could not create field: 'lastobsmonth'.")
field_lastobsyear_defn = ogr.FieldDefn(
"lastobsyear", ogr.OFTInteger)
if lyr.CreateField(field_lastobsyear_defn) != 0:
raise Exception(
"Could not create field: 'lastobsyear'.")
# Observation Date Where Score Reached 5
field_scr5obsday_defn = ogr.FieldDefn(
"scr5obsday", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsday_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsday'.")
field_scr5obsmonth_defn = ogr.FieldDefn(
"scr5obsmonth", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsmonth_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsmonth'.")
field_scr5obsyear_defn = ogr.FieldDefn(
"scr5obsyear", ogr.OFTInteger)
if lyr.CreateField(field_scr5obsyear_defn) != 0:
raise Exception(
"Could not create field: 'scr5obsyear'.")
lyr_defn = lyr.GetLayerDefn()
otherargs = ratapplier.OtherArguments()
otherargs.lyr = lyr
otherargs.lyr_defn = lyr_defn
ratcontrols = ratapplier.RatApplierControls()
ratcontrols.setProgress(progress_bar)
ratapplier.apply(_ratapplier_check_string_col_valid,
in_rats,
out_rats,
otherargs,
controls=ratcontrols)
# Update (create) the JSON LUT file.
lut_file_name = "gmw_{}_lut.json".format(tile)
lut_file_path = os.path.join(self.params["outlutdir"],
lut_file_name)
eodd_utils.get_file_lock(lut_file_path,
sleep_period=1,
wait_iters=120,
use_except=True)
if os.path.exists(lut_file_path):
lut_dict = rsgis_utils.readJSON2Dict(lut_file_path)
else:
lut_dict = dict()
obs_date_iso_str = scn_obs_date.isoformat()
lut_dict[obs_date_iso_str] = dict()
lut_dict[obs_date_iso_str]["file"] = out_vec_file
lut_dict[obs_date_iso_str]["layer"] = tile
rsgis_utils.writeDict2JSON(lut_dict, lut_file_path)
eodd_utils.release_file_lock(lut_file_path)
out_dict[tile] = out_vec_file
ds = None
outputs = True
success = True
else:
logger.debug(
"Scene is within window used to mask change outside of range."
)
else:
logger.debug(
"No change features available as outputs from previous steps..."
)
else:
logger.debug(
"GenChngSummaryFeats was not available so previous step had not run..."
)
return success, out_dict, outputs
def calc_unmixing_rmse_residualerr(self,
input_img,
input_unmix_img,
endmembers_file,
output_img,
gdalformat,
scale_factor=1000):
"""
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
endmembers_info = self.read_endmembers_mtxt(endmembers_file)
infiles = applier.FilenameAssociations()
infiles.image_orig = input_img
infiles.image_unmix = input_unmix_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.endmembers_q = endmembers_info[0]
otherargs.endmembers_p = endmembers_info[1]
otherargs.endmembers_arr = endmembers_info[2]
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _calc_unmix_err_rmse(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
block_refl_shp = inputs.image_orig.shape
img_orig_refl_flat = inputs.image_orig.reshape(
[block_refl_shp[0], (block_refl_shp[1] * block_refl_shp[2])]).T
block_unmix_shp = inputs.image_unmix.shape
img_unmix_coef_flat = inputs.image_unmix.reshape([
block_unmix_shp[0], (block_unmix_shp[1] * block_unmix_shp[2])
]).T
img_flat_shp = img_orig_refl_flat.shape
img_orig_refl_nodata = numpy.where(img_orig_refl_flat == 0, True,
False)
img_orig_refl_flat_nodata = numpy.all(img_orig_refl_nodata, axis=1)
ID = numpy.arange(img_flat_shp[0])
ID = ID[img_orig_refl_flat_nodata == False]
img_orig_refl_flat_data = img_orig_refl_flat[
img_orig_refl_flat_nodata == False, ...]
img_orig_refl_flat_data_flt = numpy.zeros_like(
img_orig_refl_flat_data, dtype=numpy.float32)
img_orig_refl_flat_data_flt[...] = img_orig_refl_flat_data
img_unmix_coef_flat_data = img_unmix_coef_flat[
img_orig_refl_flat_nodata == False, ...]
img_unmix_coef_flat_data_flt = numpy.zeros_like(
img_unmix_coef_flat_data, dtype=numpy.float32)
img_unmix_coef_flat_data_flt[
...] = img_unmix_coef_flat_data / float(scale_factor)
img_nodata_flat_shp = img_unmix_coef_flat_data_flt.shape
pred_refl_img = numpy.zeros_like(img_orig_refl_flat_data,
dtype=float)
for i in range(img_nodata_flat_shp[0]):
for q in range(otherargs.endmembers_q):
pred_refl_img[i] = pred_refl_img[i] + (
img_unmix_coef_flat_data_flt[i, q] *
otherargs.endmembers_arr[q])
# Calc Diff
diff = img_orig_refl_flat_data_flt - pred_refl_img
# Calc RMSE
diff_sq = diff * diff
mean_sum_diff_sq = numpy.sum(diff_sq,
axis=1) / otherargs.endmembers_p
rmse_arr = numpy.sqrt(mean_sum_diff_sq)
# Calc Residual Error
abs_diff = numpy.absolute(diff)
residual_arr = numpy.sum(abs_diff, axis=1) / otherargs.endmembers_p
outarr = numpy.zeros((img_flat_shp[0], 2))
outarr[ID] = numpy.stack([rmse_arr, residual_arr], axis=-1)
outarr = outarr.T
outputs.outimage = outarr.reshape(
(2, block_refl_shp[1], block_refl_shp[2]))
applier.apply(_calc_unmix_err_rmse,
infiles,
outfiles,
otherargs,
controls=aControls)
def predict_refl_img_from_simple_unmixing(self,
input_unmix_img,
endmembers_file,
output_img,
gdalformat,
scale_factor=1000):
"""
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
endmembers_info = self.read_endmembers_mtxt(endmembers_file)
infiles = applier.FilenameAssociations()
infiles.image_unmix = input_unmix_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.endmembers_q = endmembers_info[0]
otherargs.endmembers_p = endmembers_info[1]
otherargs.endmembers_arr = endmembers_info[2]
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _predict_refl_img(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
block_unmix_shp = inputs.image_unmix.shape
img_unmix_coef_flat = inputs.image_unmix.reshape([
block_unmix_shp[0], (block_unmix_shp[1] * block_unmix_shp[2])
]).T
img_unmix_coef_flat_data_flt = numpy.zeros_like(
img_unmix_coef_flat, dtype=numpy.float32)
img_unmix_coef_flat_data_flt[
...] = img_unmix_coef_flat / float(scale_factor)
img_nodata_flat_shp = img_unmix_coef_flat_data_flt.shape
pred_refl_img = numpy.zeros(
(img_unmix_coef_flat.shape[0], otherargs.endmembers_p),
dtype=numpy.int16)
for i in range(img_unmix_coef_flat.shape[0]):
for q in range(otherargs.endmembers_q):
pred_refl_img[i] = pred_refl_img[i] + (
img_unmix_coef_flat_data_flt[i, q] *
otherargs.endmembers_arr[q])
pred_refl_img = pred_refl_img.T
outputs.outimage = pred_refl_img.reshape(
(otherargs.endmembers_p, block_unmix_shp[1],
block_unmix_shp[2]))
applier.apply(_predict_refl_img,
infiles,
outfiles,
otherargs,
controls=aControls)
def get_ST_model_coeffs(json_fp, output_fp, gdalformat='KEA', bands=None, num_processes=1, model_type='Lasso',
alpha=20, cv=False):
"""
Main function to run to generate the output image. Given an input JSON file
and an output file path, generates a multi-band output image where each pixel
contains the model details for that pixel. Opening/closing of files, generation
of blocks and use of multiprocessing is all handled by RIOS.
:param json_fp: Path to JSON file of date/filepath pairs.
:param output_fp: Path for output file.
:param gdalformat: Short driver name for GDAL, e.g. KEA, GTiff.
:param bands: List of GDAL band numbers to use in the analysis, e.g. [2, 5, 7].
:param num_processes: Number of concurrent processes to use.
:param model_type: Either 'Lasso' or 'OLS'. The type of model fitting to use. OLS will
be faster, but more likely to overfit. Both types will adjust the number of model
coefficients depending on the number of observations.
:param alpha: If using Lasso fitting, the alpha value controls the degree of
penalization of the coefficients. The lower the value, the closer
the model will fit the data. For surface reflectance, a value of
around 20 (the default) is usually OK.
:param cv: If using Lasso fitting, you can use cross validation to choose
the value of alpha by setting cv=True. However, this is not recommended and will
substantially increase run time.
"""
paths = []
dates = []
try:
# Open and read JSON file containing date:filepath pairs
with open(json_fp) as json_file:
image_list = json.load(json_file)
for date, img_path in image_list.items():
dates.append(datetime.strptime(date, '%Y-%m-%d').toordinal())
paths.append(img_path)
except FileNotFoundError:
print('Could not find the provided JSON file.')
sys.exit()
except json.decoder.JSONDecodeError as e:
print('There is an error in the provided JSON file: {}'.format(e))
sys.exit()
# Create object to hold input files
infiles = applier.FilenameAssociations()
infiles.images = paths
# Create object to hold output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_fp
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set window size to 1 because we are working per-pixel
app.setWindowXsize(1)
app.setWindowYsize(1)
# Set output file type
app.setOutputDriverName(gdalformat)
# Set progress
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
app.progress = progress_bar
# Set that pyramids and statistics are not calculated.
app.omitPyramids = True
app.calcStats = False
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Open first image in list to use as a template
template_image = fileinfo.ImageInfo(infiles.images[0])
# Get no data value
nodata_val = template_image.nodataval[0]
if not bands: # No bands specified - default to all
num_bands = template_image.rasterCount
bands = [i for i in range(1, num_bands + 1)]
else: # If a list of bands is provided
# Number of bands determines things like the size of the output array
num_bands = len(bands)
# Need to tell the applier to only use the specified bands
app.selectInputImageLayers(bands)
# Create list of actual names
full_names = [template_image.layerNameFromNumber(i) for i in bands]
template_image = None
# Set up output layer names based on band numbers
layer_names = gen_layer_names(full_names)
app.setLayerNames(layer_names)
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.dates = dates
other_args.num_bands = num_bands
other_args.nodata_val = nodata_val
other_args.model_type = model_type
other_args.alpha = alpha
other_args.cv = cv
try:
applier.apply(gen_per_band_models, infiles, outfiles, otherArgs=other_args, controls=app)
except RuntimeError as e:
print('There was an error processing the images: {}'.format(e))
print('Do all images in the JSON file exist?')
def predict_for_date(date, input_path, output_path, gdalformat='KEA', num_processes=1):
"""
Main function to generate the predicted image. Given an input image containing
per-band model coefficients, outputs a multi-band predicted image over the same area.
Opening/closing of files, generation of blocks and use of multiprocessing is
all handled by RIOS.
:param date: The date to predict in YYYY-MM-DD format.
:param input_path: Path to the input image generated by get_model_coeffs.py.
:param output_path: Path for the output image.
:param gdalformat: Short driver name for GDAL, e.g. KEA, GTiff.
:param num_processes: Number of concurrent processes to use.
"""
# Create object to hold input files
infile = applier.FilenameAssociations()
infile.coeff_img = input_path
# Create object to hold output file
outfile = applier.FilenameAssociations()
outfile.output_img = output_path
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set output file type
app.setOutputDriverName(gdalformat)
# Set progress
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
app.progress = progress_bar
# Set that pyramids and statistics are not calculated.
app.omitPyramids = True
app.calcStats = False
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Convert provided date to ordinal
ordinal_date = datetime.strptime(date, '%Y-%m-%d').toordinal()
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.date_to_predict = ordinal_date
# Get band names
try:
input_img = fileinfo.ImageInfo(infile.coeff_img)
except:
sys.exit('Could not find input image.')
layer_names = numpy.unique([name.split('_')[0] for name in input_img.lnames])
app.setLayerNames(layer_names)
applier.apply(gen_prediction, infile, outfile, otherArgs=other_args, controls=app)
def find_class_outliers(pyod_obj,
img,
img_mask,
out_lbls_img,
out_scores_img=None,
img_mask_val=1,
img_bands=None,
gdalformat="KEA"):
"""
This function uses the pyod (https://github.com/yzhao062/pyod) library to find outliers within a class.
It is assumed that the input images are from a different date than the mask (classification) and therefore
the outliners will related to class changes.
:param pyod_obj: an instance of a pyod.models (e.g., pyod.models.knn.KNN) pass parameters to the constructor
:param img: input image used for analysis
:param img_mask: input image mask use to define the region of interest.
:param out_lbls_img: output image with pixel over of 1 for within mask but not outlier and 2 for in mask and outlier.
:param out_scores_img: output image (optional, None and won't be provided; Default None) providing the probability of
each pixel being an outlier
:param img_mask_val: the pixel value within the mask image for the class of interest. (Default 1)
:param img_bands: the image bands to be used for the analysis. If None then all used (Default: None)
:param gdalformat: file format for the output image(s). Default KEA.
"""
if not haveRIOS:
raise Exception(
"The rios module is required for this function could not be imported\n\t"
+ riosErr)
rsgis_utils = rsgislib.RSGISPyUtils()
if img_bands is not None:
if not ((type(img_bands) is list) or (type(img_bands) is tuple)):
raise rsgislib.RSGISPyException(
"If provided then img_bands should be a list (or None)")
else:
n_bands = rsgis_utils.getImageBandCount(img)
img_bands = numpy.arange(1, n_bands + 1)
num_vars = len(img_bands)
img_val_no_data = rsgis_utils.getImageNoDataValue(img)
msk_arr_vals = rsgislib.imageutils.extractImgPxlValsInMsk(
img, img_bands, img_mask, img_mask_val, img_val_no_data)
print("There were {} pixels within the mask.".format(
msk_arr_vals.shape[0]))
print("Fitting oulier detector")
pyod_obj.fit(msk_arr_vals)
print("Fitted oulier detector")
# RIOS function to apply classifer
def _applyPyOB(info, inputs, outputs, otherargs):
# Internal function for rios applier. Used within find_class_outliers.
out_lbls_vals = numpy.zeros_like(inputs.image_mask, dtype=numpy.uint8)
if otherargs.out_scores:
out_scores_vals = numpy.zeros_like(inputs.image_mask,
dtype=numpy.float)
if numpy.any(inputs.image_mask == otherargs.msk_val):
out_lbls_vals = out_lbls_vals.flatten()
img_msk_vals = inputs.image_mask.flatten()
if otherargs.out_scores:
out_scores_vals = out_scores_vals.flatten()
ID = numpy.arange(img_msk_vals.shape[0])
img_shape = inputs.img.shape
img_bands = inputs.img.reshape(
(img_shape[0], (img_shape[1] * img_shape[2])))
band_lst = []
for band in otherargs.img_bands:
if (band > 0) and (band <= img_shape[0]):
band_lst.append(img_bands[band - 1])
else:
raise Exception(
"Band ({}) specified is not within the image".format(
band))
img_bands_sel = numpy.stack(band_lst, axis=0)
img_bands_trans = numpy.transpose(img_bands_sel)
if otherargs.no_data_val is not None:
ID = ID[(img_bands_trans != otherargs.no_data_val).all(axis=1)]
img_msk_vals = img_msk_vals[(
img_bands_trans != otherargs.no_data_val).all(axis=1)]
img_bands_trans = img_bands_trans[(
img_bands_trans != otherargs.no_data_val).all(axis=1)]
ID = ID[img_msk_vals == otherargs.msk_val]
img_bands_trans = img_bands_trans[img_msk_vals ==
otherargs.msk_val]
if img_bands_trans.shape[0] > 0:
pred_lbls = otherargs.pyod_obj.predict(img_bands_trans)
pred_lbls = pred_lbls + 1
out_lbls_vals[ID] = pred_lbls
out_lbls_vals = numpy.expand_dims(out_lbls_vals.reshape(
(inputs.image_mask.shape[1], inputs.image_mask.shape[2])),
axis=0)
if otherargs.out_scores:
if img_bands_trans.shape[0] > 0:
pred_probs = otherargs.pyod_obj.predict_proba(
img_bands_trans, method='unify')
out_scores_vals[ID] = pred_probs[:, 1]
out_scores_vals = numpy.expand_dims(out_scores_vals.reshape(
(inputs.image_mask.shape[1], inputs.image_mask.shape[2])),
axis=0)
outputs.out_lbls_img = out_lbls_vals
if otherargs.out_scores:
outputs.out_scores_img = out_scores_vals
infiles = applier.FilenameAssociations()
infiles.image_mask = img_mask
infiles.img = img
otherargs = applier.OtherInputs()
otherargs.pyod_obj = pyod_obj
otherargs.msk_val = img_mask_val
otherargs.img_bands = img_bands
otherargs.out_scores = False
otherargs.no_data_val = img_val_no_data
outfiles = applier.FilenameAssociations()
outfiles.out_lbls_img = out_lbls_img
if out_scores_img is not None:
outfiles.out_scores_img = out_scores_img
otherargs.out_scores = True
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
print("Applying the Outlier Detector")
applier.apply(_applyPyOB, infiles, outfiles, otherargs, controls=aControls)
print("Completed")
rsgislib.rastergis.populateStats(clumps=out_lbls_img,
addclrtab=True,
calcpyramids=True,
ignorezero=True)
if out_scores_img is not None:
rsgislib.imageutils.popImageStats(out_scores_img,
usenodataval=True,
nodataval=0,
calcpyramids=True)