def calcImgsPxlMode(inputImgs, outputImg, gdalformat, no_data_val=0): """ Function which calculates the mode of a group of images. Warning, this function can be very slow. Use rsgislib.imagecalc.imagePixelColumnSummary :param inputImgs: the list of images :param outputImg: the output image file name and path (will be same dimensions as the input) :param gdalformat: the GDAL image file format of the output image file. """ import scipy.stats from rios import applier rsgis_utils = rsgislib.RSGISPyUtils() datatype = rsgis_utils.getRSGISLibDataTypeFromImg(inputImgs[0]) numpyDT = rsgis_utils.getNumpyDataType(datatype) try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.images = inputImgs outfiles = applier.FilenameAssociations() outfiles.outimage = outputImg otherargs = applier.OtherInputs() otherargs.no_data_val = no_data_val otherargs.numpyDT = numpyDT aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _applyCalcMode(info, inputs, outputs, otherargs): """ This is an internal rios function """ image_data = numpy.concatenate(inputs.images, axis=0).astype(numpy.float32) image_data[image_data == otherargs.no_data_val] = numpy.nan mode_arr, count_arr = scipy.stats.mode(image_data, axis=0, nan_policy='omit') outputs.outimage = mode_arr.astype(otherargs.numpyDT) applier.apply(_applyCalcMode, infiles, outfiles, otherargs, controls=aControls)
def update_uid_image(uid_img, chng_img, nxt_scr5_img, clrsky_img, obs_day_since_base, tmp_uid_tile): from rios import applier try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.uid_img = uid_img infiles.chng_img = chng_img infiles.nxt_scr5_img = nxt_scr5_img infiles.clrsky_img = clrsky_img outfiles = applier.FilenameAssociations() outfiles.uid_img_out = tmp_uid_tile otherargs = applier.OtherInputs() otherargs.obs_day_since_base = obs_day_since_base aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.omitPyramids = True aControls.calcStats = False def _update_uid_img(info, inputs, outputs, otherargs): """ This is an internal rios function """ uid_img_arr = numpy.array(inputs.uid_img, dtype=numpy.uint32, copy=True) start_year = inputs.uid_img[0] chng_feats = inputs.chng_img[0] new_chng_pxls = numpy.zeros_like(start_year) new_chng_pxls[(start_year == 0) & (chng_feats == 1)] = 1 chng_scr5_year = inputs.uid_img[4] new_scr5_pxls = numpy.zeros_like(chng_scr5_year) new_scr5_pxls[(chng_scr5_year == 0) & (inputs.nxt_scr5_img[0] == 1)] = 1 uid_img_arr[0, new_chng_pxls == 1] = 1970 uid_img_arr[1, new_chng_pxls == 1] = otherargs.obs_day_since_base uid_img_arr[2, inputs.clrsky_img[0] == 1] = 1970 uid_img_arr[3, inputs.clrsky_img[0] == 1] = otherargs.obs_day_since_base uid_img_arr[4, new_scr5_pxls == 1] = 1970 uid_img_arr[5, new_scr5_pxls == 1] = otherargs.obs_day_since_base outputs.uid_img_out = uid_img_arr applier.apply(_update_uid_img, infiles, outfiles, otherargs, controls=aControls)
def calcWSG84PixelArea(img, out_img, scale=10000, gdalformat='KEA'): """ A function which calculates the area (in metres) of the pixel projected in WGS84. :param img: input image, for which the per-pixel area will be calculated. :param out_img: output image file. :param scale: scale the output area to unit of interest. Scale=10000(Ha), Scale=1(sq m), Scale=1000000(sq km), Scale=4046.856(Acre), Scale=2590000(sq miles), Scale=0.0929022668(sq feet) """ import rsgislib.tools from rios import applier try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() rsgis_utils = rsgislib.RSGISPyUtils() x_res, y_res = rsgis_utils.getImageRes(img) infiles = applier.FilenameAssociations() infiles.img = img outfiles = applier.FilenameAssociations() outfiles.outimage = out_img otherargs = applier.OtherInputs() otherargs.x_res = x_res otherargs.y_res = y_res otherargs.scale = float(scale) aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = False aControls.calcStats = False def _calcPixelArea(info, inputs, outputs, otherargs): xBlock, yBlock = info.getBlockCoordArrays() x_res_arr = numpy.zeros_like(yBlock, dtype=float) x_res_arr[...] = otherargs.x_res y_res_arr = numpy.zeros_like(yBlock, dtype=float) y_res_arr[...] = otherargs.y_res x_res_arr_m, y_res_arr_m = rsgislib.tools.degrees_to_metres( yBlock, x_res_arr, y_res_arr) outputs.outimage = numpy.expand_dims( (x_res_arr_m * y_res_arr_m) / otherargs.scale, axis=0) applier.apply(_calcPixelArea, infiles, outfiles, otherargs, controls=aControls)
def findminclumpval(clumpsImg, valsImg, valBand, out_col): import rsgislib.rastergis import rsgislib.rastergis.ratutils from rios import applier import numpy try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() n_clumps = rsgislib.rastergis.getRATLength(clumpsImg) min_vals = numpy.zeros(n_clumps, dtype=int) infiles = applier.FilenameAssociations() infiles.clumpsImg = clumpsImg infiles.valsImg = valsImg outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() otherargs.min_vals = min_vals otherargs.valBand = valBand - 1 aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.omitPyramids = True aControls.calcStats = False def _findMinVals(info, inputs, outputs, otherargs): """ This is an internal rios function """ unqClumpVals = numpy.unique(inputs.clumpsImg) clumpValsFlat = inputs.clumpsImg.flatten() for clump in unqClumpVals: pxl_vals = inputs.valsImg[otherargs.valBand].flatten()[ clumpValsFlat == clump] pxl_vals = pxl_vals[pxl_vals > 0] if pxl_vals.shape[0] > 0: clump_min_val = pxl_vals.min() if otherargs.min_vals[clump] == 0: otherargs.min_vals[clump] = clump_min_val else: if clump_min_val < otherargs.min_vals[clump]: otherargs.min_vals[clump] = clump_min_val applier.apply(_findMinVals, infiles, outfiles, otherargs, controls=aControls) rsgislib.rastergis.ratutils.setColumnData(clumpsImg, out_col, min_vals)
def rescale_unmixing_results(input_img, output_img, gdalformat, scale_factor=1000): """ A function which rescales an output from a spectral unmixing (e.g., rsgislib.imagecalc.specunmixing.calc_unconstrained_unmixing) so that negative values are removed and each pixel sums to 1. :param inputImg: Input image with the spectral unmixing result (pixels need to range from 0-1) :param outputImg: Output image with the result of the rescaling (pixel values will be in range 0-1) :param gdalformat: the file format of the output file. """ try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.image = input_img outfiles = applier.FilenameAssociations() outfiles.outimage = output_img otherargs = applier.OtherInputs() otherargs.scale_factor = scale_factor aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _applyUnmixRescale(info, inputs, outputs, otherargs): """ This is an internal rios function """ inputs.image[inputs.image < 0] = 0 if otherargs.scale_factor == 1: outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpy.float32) else: outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpy.uint16) for idx in range(inputs.image.shape[0]): outputs.outimage[idx] = (inputs.image[idx] / numpy.sum( inputs.image, axis=0)) * otherargs.scale_factor applier.apply(_applyUnmixRescale, infiles, outfiles, otherargs, controls=aControls)
def apply_sklearn_classifer(classTrainInfo, skClassifier, imgMask, imgMaskVal, imgFileInfo, outputImg, gdalformat, classClrNames=True): """ This function uses a trained classifier and applies it to the provided input image. :param classTrainInfo: dict (where the key is the class name) of rsgislib.classification.ClassSimpleInfoObj objects which will be used to train the classifier (i.e., train_sklearn_classifier()), provide pixel value id and RGB class values. :param skClassifier: a trained instance of a scikit-learn classifier (e.g., use train_sklearn_classifier or train_sklearn_classifer_gridsearch) :param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the valid data regions (rsgislib.imageutils.genValidMask) :param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied. Can be used to create a heirachical classification. :param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to be used for the classification so it adheres to the training data. :param outputImg: output image file with the classification. Note. by default a colour table and class names column is added to the image. If an error is produced use HFA or KEA formats. :param gdalformat: is the output image format - all GDAL supported formats are supported. :param classClrNames: default is True and therefore a colour table will the colours specified in classTrainInfo and a ClassName column (from imgFileInfo) will be added to the output file. """ infiles = applier.FilenameAssociations() infiles.imageMask = imgMask numClassVars = 0 for imgFile in imgFileInfo: infiles.__dict__[imgFile.name] = imgFile.fileName numClassVars = numClassVars + len(imgFile.bands) outfiles = applier.FilenameAssociations() outfiles.outimage = outputImg otherargs = applier.OtherInputs() otherargs.classifier = skClassifier otherargs.mskVal = imgMaskVal otherargs.numClassVars = numClassVars otherargs.imgFileInfo = imgFileInfo try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False # RIOS function to apply classifer def _applySKClassifier(info, inputs, outputs, otherargs): """ Internal function for rios applier. Used within applyClassifer. """ outClassVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint32) if numpy.any(inputs.imageMask == otherargs.mskVal): outClassVals = outClassVals.flatten() imgMaskVals = inputs.imageMask.flatten() classVars = numpy.zeros( (outClassVals.shape[0], otherargs.numClassVars), dtype=numpy.float) # Array index which can be used to populate the output array following masking etc. ID = numpy.arange(imgMaskVals.shape[0]) classVarsIdx = 0 for imgFile in otherargs.imgFileInfo: imgArr = inputs.__dict__[imgFile.name] for band in imgFile.bands: classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten() classVarsIdx = classVarsIdx + 1 classVars = classVars[imgMaskVals == otherargs.mskVal] ID = ID[imgMaskVals == otherargs.mskVal] predClass = otherargs.classifier.predict(classVars) outClassVals[ID] = predClass outClassVals = numpy.expand_dims(outClassVals.reshape( (inputs.imageMask.shape[1], inputs.imageMask.shape[2])), axis=0) outputs.outimage = outClassVals print("Applying the Classifier") applier.apply(_applySKClassifier, infiles, outfiles, otherargs, controls=aControls) print("Completed") rsgislib.rastergis.populateStats(clumps=outputImg, addclrtab=True, calcpyramids=True, ignorezero=True) if classClrNames: ratDataset = gdal.Open(outputImg, gdal.GA_Update) red = rat.readColumn(ratDataset, 'Red') green = rat.readColumn(ratDataset, 'Green') blue = rat.readColumn(ratDataset, 'Blue') ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255')) for classKey in classTrainInfo: print("Apply Colour to class \'" + classKey + "\'") red[classTrainInfo[classKey].id] = classTrainInfo[classKey].red green[classTrainInfo[classKey].id] = classTrainInfo[classKey].green blue[classTrainInfo[classKey].id] = classTrainInfo[classKey].blue ClassName[classTrainInfo[classKey].id] = classKey rat.writeColumn(ratDataset, "Red", red) rat.writeColumn(ratDataset, "Green", green) rat.writeColumn(ratDataset, "Blue", blue) rat.writeColumn(ratDataset, "ClassName", ClassName) ratDataset = None
def apply_keras_pixel_classifier(classTrainInfo, keras_cls_mdl, imgMask, imgMaskVal, imgFileInfo, outClassImg, gdalformat, pred_batch_size=32, classClrNames=True): """ This function applies a trained single pixel keras model to an image. The function train_keras_pixel_classifer can be used to train such as model. The output image will contain the hard membership of the predicted class. :param classTrainInfo: dict (where the key is the class name) of rsgislib.classification.ClassInfoObj objects which will be used to train the classifier (i.e., train_keras_pixel_classifer()), provide pixel value id and RGB class values. :param keras_cls_mdl: a trained keras model object, with a input dimensions equivlent to the number of image bands specified in the imgFileInfo input and output layer which provides an output array of the length of the number of classes. :param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the valid data regions (rsgislib.imageutils.genValidMask) :param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied. Can be used to create a heirachical classification. :param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to be used for the classification so it adheres to the training data. :param outClassImg: Output image which will contain the hard classification. :param gdalformat: is the output image format - all GDAL supported formats are supported. :param pred_batch_size: the batch size used for the classification. :param classClrNames: default is True and therefore a colour table will the colours specified in ClassInfoObj and a ClassName (from classTrainInfo) column will be added to the output file. """ def _applyKerasPxlClassifier(info, inputs, outputs, otherargs): outClassIdVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint16) if numpy.any(inputs.imageMask == otherargs.mskVal): n_pxls = inputs.imageMask.shape[1] * inputs.imageMask.shape[2] outClassIdVals = outClassIdVals.flatten() imgMaskVals = inputs.imageMask.flatten() classVars = numpy.zeros((n_pxls, otherargs.numClassVars), dtype=numpy.float) # Array index which can be used to populate the output array following masking etc. ID = numpy.arange(imgMaskVals.shape[0]) classVarsIdx = 0 for imgFile in otherargs.imgFileInfo: imgArr = inputs.__dict__[imgFile.name] for band in imgFile.bands: classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten() classVarsIdx = classVarsIdx + 1 classVars = classVars[imgMaskVals == otherargs.mskVal] ID = ID[imgMaskVals == otherargs.mskVal] preds_idxs = numpy.argmax(otherargs.classifier.predict( classVars, batch_size=otherargs.pred_batch_size), axis=1) preds_cls_ids = numpy.zeros_like(preds_idxs, dtype=numpy.uint16) for cld_id, idx in zip(otherargs.cls_id_lut, numpy.arange(0, len(otherargs.cls_id_lut))): preds_cls_ids[preds_idxs == idx] = cld_id outClassIdVals[ID] = preds_cls_ids outClassIdVals = numpy.expand_dims(outClassIdVals.reshape( (inputs.imageMask.shape[1], inputs.imageMask.shape[2])), axis=0) outputs.outclsimage = outClassIdVals infiles = applier.FilenameAssociations() infiles.imageMask = imgMask numClassVars = 0 for imgFile in imgFileInfo: infiles.__dict__[imgFile.name] = imgFile.fileName numClassVars = numClassVars + len(imgFile.bands) n_classes = len(classTrainInfo) cls_id_lut = numpy.zeros(n_classes) for clsname in classTrainInfo: if classTrainInfo[clsname].id >= n_classes: raise ( "ClassInfoObj '{}' id ({}) is not consecutive starting from 0." .format(clsname, classTrainInfo[clsname].id)) cls_id_lut[classTrainInfo[clsname].id] = classTrainInfo[clsname].out_id outfiles = applier.FilenameAssociations() outfiles.outclsimage = outClassImg otherargs = applier.OtherInputs() otherargs.classifier = keras_cls_mdl otherargs.pred_batch_size = pred_batch_size otherargs.mskVal = imgMaskVal otherargs.numClassVars = numClassVars otherargs.imgFileInfo = imgFileInfo otherargs.n_classes = n_classes otherargs.cls_id_lut = cls_id_lut try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False print("Applying the Classifier") applier.apply(_applyKerasPxlClassifier, infiles, outfiles, otherargs, controls=aControls) print("Completed Classification") if classClrNames: rsgislib.rastergis.populateStats(outClassImg, addclrtab=True, calcpyramids=True, ignorezero=True) ratDataset = gdal.Open(outClassImg, gdal.GA_Update) red = rat.readColumn(ratDataset, 'Red') green = rat.readColumn(ratDataset, 'Green') blue = rat.readColumn(ratDataset, 'Blue') ClassName = numpy.empty_like(red, dtype=numpy.dtype('a255')) ClassName[...] = "" for classKey in classTrainInfo: print("Apply Colour to class \'" + classKey + "\'") red[classTrainInfo[classKey].out_id] = classTrainInfo[classKey].red green[classTrainInfo[classKey]. out_id] = classTrainInfo[classKey].green blue[classTrainInfo[classKey]. out_id] = classTrainInfo[classKey].blue ClassName[classTrainInfo[classKey].out_id] = classKey rat.writeColumn(ratDataset, "Red", red) rat.writeColumn(ratDataset, "Green", green) rat.writeColumn(ratDataset, "Blue", blue) rat.writeColumn(ratDataset, "ClassName", ClassName) ratDataset = None
def get_ST_masks(json_fp, bands=None, roi_img=None, gdal_format='KEA', num_processes=1, threshold=3): """Main function to run to generate the output masks. Given an input JSON file, generates a mask for each date, for each band where 0=Inlier, 1=High outlier, -1=Low outlier. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. A minimum of 12 observations is required to create the masks. json_fp: Path to JSON file which provides a dictionary where for each date, an input file name and an output file name are provided. gdal_format: Short driver name for GDAL, e.g. KEA, GTiff. num_processes: Number of concurrent processes to use. bands: List of GDAL band numbers to use, e.g. [1, 3, 5]. Defaults to all. threshold: Threshold for screening. Defaults to 3, meaning that observations outside 3*RMSE of the fitted model will be counted as outliers. Lower values will result in more outliers being detected. """ ip_paths = [] op_paths = [] dates = [] try: # Open and read JSON file containing date:filepath pairs with open(json_fp) as json_file: image_list = json.load(json_file) for date in image_list.items(): dates.append([datetime.strptime(date[0], '%Y-%m-%d').toordinal()]) ip_paths.append(date[1]['input']) op_paths.append(date[1]['output']) except FileNotFoundError: print('Could not find the provided JSON file.') sys.exit() except json.decoder.JSONDecodeError as e: print('There is an error in the provided JSON file: {}'.format(e)) sys.exit() # Create object to hold input files infiles = applier.FilenameAssociations() infiles.images = ip_paths # Create object to hold output file outfiles = applier.FilenameAssociations() outfiles.outimage = op_paths # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set window size to 1 because we are working per-pixel app.setWindowXsize(1) app.setWindowYsize(1) # Set progress try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() app.progress = progress_bar # Set output file type app.setOutputDriverName(gdal_format) if roi_img is not None: app.setReferenceImage(roi_img) app.setFootprintType(applier.BOUNDS_FROM_REFERENCE) app.setResampleMethod('near') # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Open first image in list to use as a template template_image = fileinfo.ImageInfo(infiles.images[0]) # Get no data value nodata = template_image.nodataval[0] if not bands: # No bands specified - default to all num_bands = template_image.rasterCount bands = [i for i in range(1, num_bands + 1)] else: # If a list of bands is provided # Number of bands determines things like the size of the output array num_bands = len(bands) # Need to tell the applier to only use the specified bands app.selectInputImageLayers(bands) full_names = [template_image.layerNameFromNumber(i) for i in bands] # Set up output layer name app.setLayerNames(full_names) # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.dates = dates other_args.threshold = threshold other_args.nodata = nodata other_args.num_bands = num_bands template_image = None try: applier.apply(_gen_band_masks, infiles, outfiles, otherArgs=other_args, controls=app) except RuntimeError as e: print('There was an error processing the images: {}'.format(e)) print('Do all images in the JSON file exist?')
def apply_xgboost_binary_classifier(model_file, imgMask, imgMaskVal, imgFileInfo, outProbImg, gdalformat, outClassImg=None, class_thres=5000, nthread=1): """ This function applies a trained binary (i.e., two classes) xgboost model. The function train_xgboost_binary_classifer can be used to train such as model. The output image will contain the probability of membership to the class of interest. You will need to threshold this image to get a final hard classification. Alternative, a hard class output image and threshold can be applied to this image. :param model_file: a trained xgboost binary model which can be loaded with lgb.Booster(model_file=model_file). :param imgMask: is an image file providing a mask to specify where should be classified. Simplest mask is all the valid data regions (rsgislib.imageutils.genValidMask) :param imgMaskVal: the pixel value within the imgMask to limit the region to which the classification is applied. Can be used to create a heirachical classification. :param imgFileInfo: a list of rsgislib.imageutils.ImageBandInfo objects (also used within rsgislib.imageutils.extractZoneImageBandValues2HDF) to identify which images and bands are to be used for the classification so it adheres to the training data. :param outProbImg: output image file with the classification probabilities - this image is scaled by multiplying by 10000. :param gdalformat: is the output image format - all GDAL supported formats are supported. :param outClassImg: Optional output image which will contain the hard classification, defined with a threshold on the probability image. :param class_thres: The threshold used to define the hard classification. Default is 5000 (i.e., probability of 0.5). :param nthread: The number of threads to use for the classifier. """ def _applyXGBClassifier(info, inputs, outputs, otherargs): outClassVals = numpy.zeros_like(inputs.imageMask, dtype=numpy.uint16) if numpy.any(inputs.imageMask == otherargs.mskVal): outClassVals = outClassVals.flatten() imgMaskVals = inputs.imageMask.flatten() classVars = numpy.zeros( (outClassVals.shape[0], otherargs.numClassVars), dtype=numpy.float) # Array index which can be used to populate the output array following masking etc. ID = numpy.arange(imgMaskVals.shape[0]) classVarsIdx = 0 for imgFile in otherargs.imgFileInfo: imgArr = inputs.__dict__[imgFile.name] for band in imgFile.bands: classVars[..., classVarsIdx] = imgArr[(band - 1)].flatten() classVarsIdx = classVarsIdx + 1 classVars = classVars[imgMaskVals == otherargs.mskVal] ID = ID[imgMaskVals == otherargs.mskVal] predClass = numpy.around( otherargs.classifier.predict(xgb.DMatrix(classVars)) * 10000) outClassVals[ID] = predClass outClassVals = numpy.expand_dims(outClassVals.reshape( (inputs.imageMask.shape[1], inputs.imageMask.shape[2])), axis=0) outputs.outimage = outClassVals classifier = xgb.Booster({'nthread': nthread}) classifier.load_model(model_file) infiles = applier.FilenameAssociations() infiles.imageMask = imgMask numClassVars = 0 for imgFile in imgFileInfo: infiles.__dict__[imgFile.name] = imgFile.fileName numClassVars = numClassVars + len(imgFile.bands) outfiles = applier.FilenameAssociations() outfiles.outimage = outProbImg otherargs = applier.OtherInputs() otherargs.classifier = classifier otherargs.mskVal = imgMaskVal otherargs.numClassVars = numClassVars otherargs.imgFileInfo = imgFileInfo try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False print("Applying the Classifier") applier.apply(_applyXGBClassifier, infiles, outfiles, otherargs, controls=aControls) print("Completed") rsgislib.imageutils.popImageStats(outProbImg, usenodataval=True, nodataval=0, calcpyramids=True) if outClassImg is not None: rsgislib.imagecalc.imageMath(outProbImg, outClassImg, 'b1>{}?1:0'.format(class_thres), gdalformat, rsgislib.TYPE_8UINT) rsgislib.rastergis.populateStats(outClassImg, addclrtab=True, calcpyramids=True, ignorezero=True)
def calcImgBasicStats4RefRegion(ref_img, stats_imgs, output_img, gdalformat='KEA'): """ A function which calculates the mean and standard deviation through a series of input images. The region for processing is defined by the reference image and images padded with no-data where no data is present. The output image has twice the number of bands as the input image providing a mean and standard deviation for each input band. If the input images has 2 bands then the output bands will have the following order: 1. band 1 mean 2. band 1 std dev 3. band 2 mean 4. band 2 std dev :param ref_img: reference image which defines the output image :param stats_imgs: a list of input images over which the stats will be calculated. :param output_img: the output image path and file name :param gdalformat: the output image file format. Default KEA. """ import rsgislib.imageutils from rios import applier rsgis_utils = rsgislib.RSGISPyUtils() first = True n_bands = 0 no_data_val = 0 for img in stats_imgs: print(img) if first: n_bands = rsgis_utils.getImageBandCount(img) no_data_val = rsgis_utils.getImageNoDataValue(img) first = False else: if n_bands != rsgis_utils.getImageBandCount(img): raise Exception( "The number of bands must be the same in all input images." ) if no_data_val != rsgis_utils.getImageNoDataValue(img): raise Exception( "The no data value should be the same in all input images." ) # RIOS internal function to calculate mean and standard deviation of the input images def _calcBasicStats(info, inputs, outputs, otherargs): n_imgs = len(inputs.imgs) blk_shp = inputs.imgs[0].shape if blk_shp[0] != otherargs.n_bands: raise Exception( "Block shape and the number of input image bands do not align." ) outputs.output_img = numpy.zeros( (blk_shp[0] * 2, blk_shp[1], blk_shp[2]), dtype=float) band_arr = [] for band in range(blk_shp[0]): band_arr.append( numpy.zeros((n_imgs, blk_shp[1], blk_shp[2]), dtype=float)) img_idx = 0 for img_blk in inputs.imgs: for band in range(blk_shp[0]): band_arr[band][img_idx] = img_blk[band] img_idx = img_idx + 1 for band in range(blk_shp[0]): band_arr[band][band_arr[band] == otherargs.no_data_val] = numpy.nan outputs.output_img[band * 2] = numpy.nanmean(band_arr[band], axis=0) outputs.output_img[band * 2 + 1] = numpy.nanstd(band_arr[band], axis=0) outputs.output_img[band * 2][numpy.isnan( outputs.output_img[band * 2])] = otherargs.no_data_val outputs.output_img[band * 2 + 1][numpy.isnan( outputs.output_img[band * 2 + 1])] = 0.0 try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.imgs = stats_imgs otherargs = applier.OtherInputs() otherargs.n_bands = n_bands otherargs.no_data_val = no_data_val outfiles = applier.FilenameAssociations() outfiles.output_img = output_img aControls = applier.ApplierControls() aControls.referenceImage = ref_img aControls.footprint = applier.BOUNDS_FROM_REFERENCE aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False print("Calculating Stats Image.") applier.apply(_calcBasicStats, infiles, outfiles, otherargs, controls=aControls) print("Completed") rsgislib.imageutils.popImageStats(output_img, usenodataval=True, nodataval=no_data_val, calcpyramids=True)
def perform_simple_unmixing(self, input_image, output_image, gdalformat, endmembers_file, weight=None, scale_factor=1000): """ A function which uses the RIOS to iterate through the input image and perform a simple/standard spectral unmixing on the input image using the calc_abundance function. :param input_image: The file path to a GDAL readable input image. :param output_image: The file path to the GDAL writable output image (Note pixel values will be between 0-1000) :param gdalformat: The output image format to be used. :param endmembers_file: The endmembers for the unmixing in the RSGISLib mtxt format. :param weight: Optional (if None ignored) to provide a weight to implement the approach of Scarth et al (2010) adding a weight to the least squares optimisation to get the abundances to sum to 1. :param scale_factor: Scale factor for integerising the resulting image. If value is 1 then output image will be a floating point image. References: Scarth, P., Roder, A., & Schmidt, M. (2010). Tracking grazing pressure and climate interaction—The role of Landsat fractional cover in time series analysis. Proceedings of the 15th Australasian Remote Sensing and Photogrammetry ConferenceAustralia: Alice Springs. http://dx.doi.org/10.6084/m9.figshare.94250. """ try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() if weight is not None: endmembers_info = self.read_endmembers_mtxt_weight( endmembers_file, weight) else: endmembers_info = self.read_endmembers_mtxt(endmembers_file) print(endmembers_info) infiles = applier.FilenameAssociations() infiles.image = input_image outfiles = applier.FilenameAssociations() outfiles.outimage = output_image otherargs = applier.OtherInputs() otherargs.endmembers_q = endmembers_info[0] otherargs.endmembers_p = endmembers_info[1] otherargs.endmembers_arr = endmembers_info[2] otherargs.weight = weight otherargs.scale_factor = scale_factor aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _simple_unmix(info, inputs, outputs, otherargs): """ This is an internal rios function """ block_shp = inputs.image.shape img_flat = inputs.image.reshape( [block_shp[0], (block_shp[1] * block_shp[2])]).T if otherargs.weight is not None: img_flat_dtype = img_flat.dtype tmp_img_flat = img_flat img_flat = numpy.zeros( (img_flat.shape[0], img_flat.shape[1] + 1), dtype=img_flat_dtype) img_flat[...] = weight img_flat[:, :-1] = tmp_img_flat img_flat_shp = img_flat.shape img_nodata = numpy.where(img_flat == 0, True, False) img_flat_nodata = numpy.all(img_nodata, axis=1) ID = numpy.arange(img_flat_shp[0]) ID = ID[img_flat_nodata == False] img_flat_data = img_flat[img_flat_nodata == False, ...] abundances_arr = self.calc_abundance(img_flat_data, otherargs.endmembers_arr) if otherargs.scale_factor > 1: outarr = numpy.zeros([img_flat_shp[0], otherargs.endmembers_q], dtype=numpy.int16) else: outarr = numpy.zeros([img_flat_shp[0], otherargs.endmembers_q], dtype=numpy.float32) outarr[ID] = (abundances_arr * otherargs.scale_factor) outarr = outarr.T outputs.outimage = outarr.reshape( (otherargs.endmembers_q, block_shp[1], block_shp[2])) applier.apply(_simple_unmix, infiles, outfiles, otherargs, controls=aControls)
def rescaleImgPxlVals(inputImg, outputImg, gdalformat, datatype, bandRescale, trim2Limits=True): """ Function which rescales an input image base on a list of rescaling parameters. :param inputImg: the input image :param outputImg: the output image file name and path (will be same dimensions as the input) :param gdalformat: the GDAL image file format of the output image file. :param bandRescale: list of ImageBandRescale objects :param trim2Limits: whether to trim the output to the output min/max values. """ from rios import applier bandRescaleDict = dict() for rescaleObj in bandRescale: bandRescaleDict[rescaleObj.band - 1] = rescaleObj rsgis_utils = rsgislib.RSGISPyUtils() numpyDT = rsgis_utils.getNumpyDataType(datatype) try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.image = inputImg outfiles = applier.FilenameAssociations() outfiles.outimage = outputImg otherargs = applier.OtherInputs() otherargs.rescaleDict = bandRescaleDict otherargs.trim = trim2Limits otherargs.numpyDT = numpyDT aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _applyRescale(info, inputs, outputs, otherargs): """ This is an internal rios function """ outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpyDT) for idx in range(inputs.image.shape[0]): outputs.outimage[idx] = numpy.where( inputs.image[idx] == otherargs.rescaleDict[idx].inNoData, otherargs.rescaleDict[idx].outNoData, (((inputs.image[idx] - otherargs.rescaleDict[idx].inMin) / (inputs.image[idx] - otherargs.rescaleDict[idx].inMax - inputs.image[idx] - otherargs.rescaleDict[idx].inMin)) * (inputs.image[idx] - otherargs.rescaleDict[idx].outMax - inputs.image[idx] - otherargs.rescaleDict[idx].outMin)) + inputs.image[idx] - otherargs.rescaleDict[idx].outMin) if otherargs.trim: outputs.outimage[idx] = numpy.where( (outputs.outimage[idx] != otherargs.rescaleDict[idx].outNoData) & (outputs.outimage[idx] < otherargs.rescaleDict[idx].outMin), otherargs.rescaleDict[idx].outMin, outputs.outimage[idx]) outputs.outimage[idx] = numpy.where( (outputs.outimage[idx] != otherargs.rescaleDict[idx].outNoData) & (outputs.outimage[idx] > otherargs.rescaleDict[idx].outMax), otherargs.rescaleDict[idx].outMax, outputs.outimage[idx]) applier.apply(_applyRescale, infiles, outfiles, otherargs, controls=aControls)
def perform_analysis(self, scn_db_obj, sen_obj, plgin_objs): logger.info("Processing Scene: {}".format(scn_db_obj.PID)) if scn_db_obj.Invalid: return False, None, False rsgis_utils = rsgislib.RSGISPyUtils() eodd_utils = EODataDownUtils() success = True outputs = False out_dict = None if 'GenChngSummaryFeats' in plgin_objs: if plgin_objs['GenChngSummaryFeats'].Completed and plgin_objs[ 'GenChngSummaryFeats'].Outputs and plgin_objs[ 'GenChngSummaryFeats'].Success: scn_chng_info = plgin_objs['GenChngSummaryFeats'].ExtendedInfo scn_unq_name = sen_obj.get_scn_unq_name_record(scn_db_obj) out_vec_file = os.path.join( self.params['outvecdir'], "{}_chng_vec.gpkg".format(scn_unq_name)) if os.path.exists(out_vec_file): delete_vector_file(out_vec_file) if sen_obj.get_sensor_name() == 'LandsatGOOG': scn_obs_date = scn_db_obj.Sensing_Time elif sen_obj.get_sensor_name() == 'Sentinel2GOOG': scn_obs_date = scn_db_obj.Sensing_Time elif sen_obj.get_sensor_name() == 'Sentinel1ASF': scn_obs_date = scn_db_obj.Acquisition_Date else: raise Exception("Did not recognise the sensor name...") start_date = datetime.datetime(year=2019, month=4, day=30) if scn_obs_date > start_date: try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() drv = gdal.GetDriverByName("GPKG") if drv is None: raise Exception("Driver GPKG is not avaiable.") ds = drv.Create(out_vec_file, 0, 0, 0, gdal.GDT_Unknown) if ds is None: raise Exception( "Could not create output file: {}.".format( out_vec_file)) out_dict = dict() for tile in scn_chng_info: logger.debug("Processing tile {}...".format(tile)) clumps_img = scn_chng_info[tile] in_rats = ratapplier.RatAssociations() out_rats = ratapplier.RatAssociations() in_rats.inrat = ratapplier.RatHandle(clumps_img) lyr = ds.CreateLayer(tile, None, ogr.wkbPoint) if lyr is None: raise Exception( "Could not create output layer: {}.".format( tile)) field_uid_defn = ogr.FieldDefn("uid", ogr.OFTInteger) if lyr.CreateField(field_uid_defn) != 0: raise Exception("Could not create field: 'uid'.") field_prop_chng_defn = ogr.FieldDefn( "prop_chng", ogr.OFTReal) if lyr.CreateField(field_prop_chng_defn) != 0: raise Exception( "Could not create field: 'prop_chng'.") field_score_defn = ogr.FieldDefn( "score", ogr.OFTInteger) if lyr.CreateField(field_score_defn) != 0: raise Exception("Could not create field: 'score'.") # First Observation Date field_firstobsday_defn = ogr.FieldDefn( "firstobsday", ogr.OFTInteger) if lyr.CreateField(field_firstobsday_defn) != 0: raise Exception( "Could not create field: 'firstobsday'.") field_firstobsmonth_defn = ogr.FieldDefn( "firstobsmonth", ogr.OFTInteger) if lyr.CreateField(field_firstobsmonth_defn) != 0: raise Exception( "Could not create field: 'firstobsmonth'.") field_firstobsyear_defn = ogr.FieldDefn( "firstobsyear", ogr.OFTInteger) if lyr.CreateField(field_firstobsyear_defn) != 0: raise Exception( "Could not create field: 'firstobsyear'.") # Last Observation Date field_lastobsday_defn = ogr.FieldDefn( "lastobsday", ogr.OFTInteger) if lyr.CreateField(field_lastobsday_defn) != 0: raise Exception( "Could not create field: 'lastobsday'.") field_lastobsmonth_defn = ogr.FieldDefn( "lastobsmonth", ogr.OFTInteger) if lyr.CreateField(field_lastobsmonth_defn) != 0: raise Exception( "Could not create field: 'lastobsmonth'.") field_lastobsyear_defn = ogr.FieldDefn( "lastobsyear", ogr.OFTInteger) if lyr.CreateField(field_lastobsyear_defn) != 0: raise Exception( "Could not create field: 'lastobsyear'.") # Observation Date Where Score Reached 5 field_scr5obsday_defn = ogr.FieldDefn( "scr5obsday", ogr.OFTInteger) if lyr.CreateField(field_scr5obsday_defn) != 0: raise Exception( "Could not create field: 'scr5obsday'.") field_scr5obsmonth_defn = ogr.FieldDefn( "scr5obsmonth", ogr.OFTInteger) if lyr.CreateField(field_scr5obsmonth_defn) != 0: raise Exception( "Could not create field: 'scr5obsmonth'.") field_scr5obsyear_defn = ogr.FieldDefn( "scr5obsyear", ogr.OFTInteger) if lyr.CreateField(field_scr5obsyear_defn) != 0: raise Exception( "Could not create field: 'scr5obsyear'.") lyr_defn = lyr.GetLayerDefn() otherargs = ratapplier.OtherArguments() otherargs.lyr = lyr otherargs.lyr_defn = lyr_defn ratcontrols = ratapplier.RatApplierControls() ratcontrols.setProgress(progress_bar) ratapplier.apply(_ratapplier_check_string_col_valid, in_rats, out_rats, otherargs, controls=ratcontrols) # Update (create) the JSON LUT file. lut_file_name = "gmw_{}_lut.json".format(tile) lut_file_path = os.path.join(self.params["outlutdir"], lut_file_name) eodd_utils.get_file_lock(lut_file_path, sleep_period=1, wait_iters=120, use_except=True) if os.path.exists(lut_file_path): lut_dict = rsgis_utils.readJSON2Dict(lut_file_path) else: lut_dict = dict() obs_date_iso_str = scn_obs_date.isoformat() lut_dict[obs_date_iso_str] = dict() lut_dict[obs_date_iso_str]["file"] = out_vec_file lut_dict[obs_date_iso_str]["layer"] = tile rsgis_utils.writeDict2JSON(lut_dict, lut_file_path) eodd_utils.release_file_lock(lut_file_path) out_dict[tile] = out_vec_file ds = None outputs = True success = True else: logger.debug( "Scene is within window used to mask change outside of range." ) else: logger.debug( "No change features available as outputs from previous steps..." ) else: logger.debug( "GenChngSummaryFeats was not available so previous step had not run..." ) return success, out_dict, outputs
def calc_unmixing_rmse_residualerr(self, input_img, input_unmix_img, endmembers_file, output_img, gdalformat, scale_factor=1000): """ """ try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() endmembers_info = self.read_endmembers_mtxt(endmembers_file) infiles = applier.FilenameAssociations() infiles.image_orig = input_img infiles.image_unmix = input_unmix_img outfiles = applier.FilenameAssociations() outfiles.outimage = output_img otherargs = applier.OtherInputs() otherargs.endmembers_q = endmembers_info[0] otherargs.endmembers_p = endmembers_info[1] otherargs.endmembers_arr = endmembers_info[2] otherargs.scale_factor = scale_factor aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _calc_unmix_err_rmse(info, inputs, outputs, otherargs): """ This is an internal rios function """ block_refl_shp = inputs.image_orig.shape img_orig_refl_flat = inputs.image_orig.reshape( [block_refl_shp[0], (block_refl_shp[1] * block_refl_shp[2])]).T block_unmix_shp = inputs.image_unmix.shape img_unmix_coef_flat = inputs.image_unmix.reshape([ block_unmix_shp[0], (block_unmix_shp[1] * block_unmix_shp[2]) ]).T img_flat_shp = img_orig_refl_flat.shape img_orig_refl_nodata = numpy.where(img_orig_refl_flat == 0, True, False) img_orig_refl_flat_nodata = numpy.all(img_orig_refl_nodata, axis=1) ID = numpy.arange(img_flat_shp[0]) ID = ID[img_orig_refl_flat_nodata == False] img_orig_refl_flat_data = img_orig_refl_flat[ img_orig_refl_flat_nodata == False, ...] img_orig_refl_flat_data_flt = numpy.zeros_like( img_orig_refl_flat_data, dtype=numpy.float32) img_orig_refl_flat_data_flt[...] = img_orig_refl_flat_data img_unmix_coef_flat_data = img_unmix_coef_flat[ img_orig_refl_flat_nodata == False, ...] img_unmix_coef_flat_data_flt = numpy.zeros_like( img_unmix_coef_flat_data, dtype=numpy.float32) img_unmix_coef_flat_data_flt[ ...] = img_unmix_coef_flat_data / float(scale_factor) img_nodata_flat_shp = img_unmix_coef_flat_data_flt.shape pred_refl_img = numpy.zeros_like(img_orig_refl_flat_data, dtype=float) for i in range(img_nodata_flat_shp[0]): for q in range(otherargs.endmembers_q): pred_refl_img[i] = pred_refl_img[i] + ( img_unmix_coef_flat_data_flt[i, q] * otherargs.endmembers_arr[q]) # Calc Diff diff = img_orig_refl_flat_data_flt - pred_refl_img # Calc RMSE diff_sq = diff * diff mean_sum_diff_sq = numpy.sum(diff_sq, axis=1) / otherargs.endmembers_p rmse_arr = numpy.sqrt(mean_sum_diff_sq) # Calc Residual Error abs_diff = numpy.absolute(diff) residual_arr = numpy.sum(abs_diff, axis=1) / otherargs.endmembers_p outarr = numpy.zeros((img_flat_shp[0], 2)) outarr[ID] = numpy.stack([rmse_arr, residual_arr], axis=-1) outarr = outarr.T outputs.outimage = outarr.reshape( (2, block_refl_shp[1], block_refl_shp[2])) applier.apply(_calc_unmix_err_rmse, infiles, outfiles, otherargs, controls=aControls)
def predict_refl_img_from_simple_unmixing(self, input_unmix_img, endmembers_file, output_img, gdalformat, scale_factor=1000): """ """ try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() endmembers_info = self.read_endmembers_mtxt(endmembers_file) infiles = applier.FilenameAssociations() infiles.image_unmix = input_unmix_img outfiles = applier.FilenameAssociations() outfiles.outimage = output_img otherargs = applier.OtherInputs() otherargs.endmembers_q = endmembers_info[0] otherargs.endmembers_p = endmembers_info[1] otherargs.endmembers_arr = endmembers_info[2] otherargs.scale_factor = scale_factor aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _predict_refl_img(info, inputs, outputs, otherargs): """ This is an internal rios function """ block_unmix_shp = inputs.image_unmix.shape img_unmix_coef_flat = inputs.image_unmix.reshape([ block_unmix_shp[0], (block_unmix_shp[1] * block_unmix_shp[2]) ]).T img_unmix_coef_flat_data_flt = numpy.zeros_like( img_unmix_coef_flat, dtype=numpy.float32) img_unmix_coef_flat_data_flt[ ...] = img_unmix_coef_flat / float(scale_factor) img_nodata_flat_shp = img_unmix_coef_flat_data_flt.shape pred_refl_img = numpy.zeros( (img_unmix_coef_flat.shape[0], otherargs.endmembers_p), dtype=numpy.int16) for i in range(img_unmix_coef_flat.shape[0]): for q in range(otherargs.endmembers_q): pred_refl_img[i] = pred_refl_img[i] + ( img_unmix_coef_flat_data_flt[i, q] * otherargs.endmembers_arr[q]) pred_refl_img = pred_refl_img.T outputs.outimage = pred_refl_img.reshape( (otherargs.endmembers_p, block_unmix_shp[1], block_unmix_shp[2])) applier.apply(_predict_refl_img, infiles, outfiles, otherargs, controls=aControls)
def get_ST_model_coeffs(json_fp, output_fp, gdalformat='KEA', bands=None, num_processes=1, model_type='Lasso', alpha=20, cv=False): """ Main function to run to generate the output image. Given an input JSON file and an output file path, generates a multi-band output image where each pixel contains the model details for that pixel. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. :param json_fp: Path to JSON file of date/filepath pairs. :param output_fp: Path for output file. :param gdalformat: Short driver name for GDAL, e.g. KEA, GTiff. :param bands: List of GDAL band numbers to use in the analysis, e.g. [2, 5, 7]. :param num_processes: Number of concurrent processes to use. :param model_type: Either 'Lasso' or 'OLS'. The type of model fitting to use. OLS will be faster, but more likely to overfit. Both types will adjust the number of model coefficients depending on the number of observations. :param alpha: If using Lasso fitting, the alpha value controls the degree of penalization of the coefficients. The lower the value, the closer the model will fit the data. For surface reflectance, a value of around 20 (the default) is usually OK. :param cv: If using Lasso fitting, you can use cross validation to choose the value of alpha by setting cv=True. However, this is not recommended and will substantially increase run time. """ paths = [] dates = [] try: # Open and read JSON file containing date:filepath pairs with open(json_fp) as json_file: image_list = json.load(json_file) for date, img_path in image_list.items(): dates.append(datetime.strptime(date, '%Y-%m-%d').toordinal()) paths.append(img_path) except FileNotFoundError: print('Could not find the provided JSON file.') sys.exit() except json.decoder.JSONDecodeError as e: print('There is an error in the provided JSON file: {}'.format(e)) sys.exit() # Create object to hold input files infiles = applier.FilenameAssociations() infiles.images = paths # Create object to hold output file outfiles = applier.FilenameAssociations() outfiles.outimage = output_fp # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set window size to 1 because we are working per-pixel app.setWindowXsize(1) app.setWindowYsize(1) # Set output file type app.setOutputDriverName(gdalformat) # Set progress try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() app.progress = progress_bar # Set that pyramids and statistics are not calculated. app.omitPyramids = True app.calcStats = False # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Open first image in list to use as a template template_image = fileinfo.ImageInfo(infiles.images[0]) # Get no data value nodata_val = template_image.nodataval[0] if not bands: # No bands specified - default to all num_bands = template_image.rasterCount bands = [i for i in range(1, num_bands + 1)] else: # If a list of bands is provided # Number of bands determines things like the size of the output array num_bands = len(bands) # Need to tell the applier to only use the specified bands app.selectInputImageLayers(bands) # Create list of actual names full_names = [template_image.layerNameFromNumber(i) for i in bands] template_image = None # Set up output layer names based on band numbers layer_names = gen_layer_names(full_names) app.setLayerNames(layer_names) # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.dates = dates other_args.num_bands = num_bands other_args.nodata_val = nodata_val other_args.model_type = model_type other_args.alpha = alpha other_args.cv = cv try: applier.apply(gen_per_band_models, infiles, outfiles, otherArgs=other_args, controls=app) except RuntimeError as e: print('There was an error processing the images: {}'.format(e)) print('Do all images in the JSON file exist?')
def predict_for_date(date, input_path, output_path, gdalformat='KEA', num_processes=1): """ Main function to generate the predicted image. Given an input image containing per-band model coefficients, outputs a multi-band predicted image over the same area. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. :param date: The date to predict in YYYY-MM-DD format. :param input_path: Path to the input image generated by get_model_coeffs.py. :param output_path: Path for the output image. :param gdalformat: Short driver name for GDAL, e.g. KEA, GTiff. :param num_processes: Number of concurrent processes to use. """ # Create object to hold input files infile = applier.FilenameAssociations() infile.coeff_img = input_path # Create object to hold output file outfile = applier.FilenameAssociations() outfile.output_img = output_path # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set output file type app.setOutputDriverName(gdalformat) # Set progress try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() app.progress = progress_bar # Set that pyramids and statistics are not calculated. app.omitPyramids = True app.calcStats = False # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Convert provided date to ordinal ordinal_date = datetime.strptime(date, '%Y-%m-%d').toordinal() # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.date_to_predict = ordinal_date # Get band names try: input_img = fileinfo.ImageInfo(infile.coeff_img) except: sys.exit('Could not find input image.') layer_names = numpy.unique([name.split('_')[0] for name in input_img.lnames]) app.setLayerNames(layer_names) applier.apply(gen_prediction, infile, outfile, otherArgs=other_args, controls=app)
def find_class_outliers(pyod_obj, img, img_mask, out_lbls_img, out_scores_img=None, img_mask_val=1, img_bands=None, gdalformat="KEA"): """ This function uses the pyod (https://github.com/yzhao062/pyod) library to find outliers within a class. It is assumed that the input images are from a different date than the mask (classification) and therefore the outliners will related to class changes. :param pyod_obj: an instance of a pyod.models (e.g., pyod.models.knn.KNN) pass parameters to the constructor :param img: input image used for analysis :param img_mask: input image mask use to define the region of interest. :param out_lbls_img: output image with pixel over of 1 for within mask but not outlier and 2 for in mask and outlier. :param out_scores_img: output image (optional, None and won't be provided; Default None) providing the probability of each pixel being an outlier :param img_mask_val: the pixel value within the mask image for the class of interest. (Default 1) :param img_bands: the image bands to be used for the analysis. If None then all used (Default: None) :param gdalformat: file format for the output image(s). Default KEA. """ if not haveRIOS: raise Exception( "The rios module is required for this function could not be imported\n\t" + riosErr) rsgis_utils = rsgislib.RSGISPyUtils() if img_bands is not None: if not ((type(img_bands) is list) or (type(img_bands) is tuple)): raise rsgislib.RSGISPyException( "If provided then img_bands should be a list (or None)") else: n_bands = rsgis_utils.getImageBandCount(img) img_bands = numpy.arange(1, n_bands + 1) num_vars = len(img_bands) img_val_no_data = rsgis_utils.getImageNoDataValue(img) msk_arr_vals = rsgislib.imageutils.extractImgPxlValsInMsk( img, img_bands, img_mask, img_mask_val, img_val_no_data) print("There were {} pixels within the mask.".format( msk_arr_vals.shape[0])) print("Fitting oulier detector") pyod_obj.fit(msk_arr_vals) print("Fitted oulier detector") # RIOS function to apply classifer def _applyPyOB(info, inputs, outputs, otherargs): # Internal function for rios applier. Used within find_class_outliers. out_lbls_vals = numpy.zeros_like(inputs.image_mask, dtype=numpy.uint8) if otherargs.out_scores: out_scores_vals = numpy.zeros_like(inputs.image_mask, dtype=numpy.float) if numpy.any(inputs.image_mask == otherargs.msk_val): out_lbls_vals = out_lbls_vals.flatten() img_msk_vals = inputs.image_mask.flatten() if otherargs.out_scores: out_scores_vals = out_scores_vals.flatten() ID = numpy.arange(img_msk_vals.shape[0]) img_shape = inputs.img.shape img_bands = inputs.img.reshape( (img_shape[0], (img_shape[1] * img_shape[2]))) band_lst = [] for band in otherargs.img_bands: if (band > 0) and (band <= img_shape[0]): band_lst.append(img_bands[band - 1]) else: raise Exception( "Band ({}) specified is not within the image".format( band)) img_bands_sel = numpy.stack(band_lst, axis=0) img_bands_trans = numpy.transpose(img_bands_sel) if otherargs.no_data_val is not None: ID = ID[(img_bands_trans != otherargs.no_data_val).all(axis=1)] img_msk_vals = img_msk_vals[( img_bands_trans != otherargs.no_data_val).all(axis=1)] img_bands_trans = img_bands_trans[( img_bands_trans != otherargs.no_data_val).all(axis=1)] ID = ID[img_msk_vals == otherargs.msk_val] img_bands_trans = img_bands_trans[img_msk_vals == otherargs.msk_val] if img_bands_trans.shape[0] > 0: pred_lbls = otherargs.pyod_obj.predict(img_bands_trans) pred_lbls = pred_lbls + 1 out_lbls_vals[ID] = pred_lbls out_lbls_vals = numpy.expand_dims(out_lbls_vals.reshape( (inputs.image_mask.shape[1], inputs.image_mask.shape[2])), axis=0) if otherargs.out_scores: if img_bands_trans.shape[0] > 0: pred_probs = otherargs.pyod_obj.predict_proba( img_bands_trans, method='unify') out_scores_vals[ID] = pred_probs[:, 1] out_scores_vals = numpy.expand_dims(out_scores_vals.reshape( (inputs.image_mask.shape[1], inputs.image_mask.shape[2])), axis=0) outputs.out_lbls_img = out_lbls_vals if otherargs.out_scores: outputs.out_scores_img = out_scores_vals infiles = applier.FilenameAssociations() infiles.image_mask = img_mask infiles.img = img otherargs = applier.OtherInputs() otherargs.pyod_obj = pyod_obj otherargs.msk_val = img_mask_val otherargs.img_bands = img_bands otherargs.out_scores = False otherargs.no_data_val = img_val_no_data outfiles = applier.FilenameAssociations() outfiles.out_lbls_img = out_lbls_img if out_scores_img is not None: outfiles.out_scores_img = out_scores_img otherargs.out_scores = True try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False print("Applying the Outlier Detector") applier.apply(_applyPyOB, infiles, outfiles, otherargs, controls=aControls) print("Completed") rsgislib.rastergis.populateStats(clumps=out_lbls_img, addclrtab=True, calcpyramids=True, ignorezero=True) if out_scores_img is not None: rsgislib.imageutils.popImageStats(out_scores_img, usenodataval=True, nodataval=0, calcpyramids=True)