def makeTOAReflectance(infile, mtlFile, anglesfile, outfile): """ Main routine - does the calculation The eqn for TOA reflectance, p, is p = pi * L * d^2 / E * cos(theta) d = earthSunDistance(date) L = image pixel (radiance) E = exoatmospheric irradiance for the band, and theta = solar zenith angle. Assumes infile is radiance values in DN from USGS. mtlFile is the .mtl file. outfile will be created in the default format that RIOS is configured to use and will be top of atmosphere reflectance values *10000. Also assumes that the angles image file is scaled as radians*100, and has layers for satAzimuth, satZenith, sunAzimuth, sunZenith, in that order. """ mtlInfo = config.readMTLFile(mtlFile) spaceCraft = mtlInfo['SPACECRAFT_ID'] date = mtlInfo['DATE_ACQUIRED'] date = date.replace('-', '') inputs = applier.FilenameAssociations() inputs.infile = infile inputs.angles = anglesfile outputs = applier.FilenameAssociations() outputs.outfile = outfile otherinputs = applier.OtherInputs() otherinputs.earthSunDistance = earthSunDistance(date) otherinputs.earthSunDistanceSq = otherinputs.earthSunDistance * otherinputs.earthSunDistance otherinputs.esun = ESUN_LOOKUP[spaceCraft] gains, offsets = readGainsOffsets(mtlInfo) otherinputs.gains = gains otherinputs.offsets = offsets otherinputs.anglesToRadians = 0.01 otherinputs.outNull = 32767 imginfo = fileinfo.ImageInfo(infile) otherinputs.inNull = imginfo.nodataval[0] controls = applier.ApplierControls() if platform.system() in ["Linux", "Darwin"]: controls.setNumThreads(multiprocessing.cpu_count()) controls.setJobManagerType("multiprocessing") controls.progress = cuiprogress.GDALProgressBar() controls.setStatsIgnore(otherinputs.outNull) controls.setCalcStats(False) applier.apply(riosTOA, inputs, outputs, otherinputs, controls=controls) # Explicitly set the null value in the output ds = gdal.Open(outfile, gdal.GA_Update) for i in range(ds.RasterCount): ds.GetRasterBand(i + 1).SetNoDataValue(otherinputs.outNull)
def runCorrection(insigma, inlinc, inpix, outsigma, thetaref=39.0, nFactor=1.0, filterSize=None): controls = applier.ApplierControls() # Set up input images infiles = applier.FilenameAssociations() infiles.insigma = insigma infiles.inlinc = inlinc infiles.inpix = inpix # Set up output image outfiles = applier.FilenameAssociations() outfiles.outimage = outsigma # Set format for output image outControls = getOutDriver(outsigma) # Set options controls.setOutputDriverName(outControls['gdalFormat']) controls.setCreationOptions(outControls['gdalCOOptions']) controls.setCalcStats(outControls['calcStats']) # Set up parameters otherargs = applier.OtherInputs() otherargs.thetaref = thetaref otherargs.nFactor = nFactor otherargs.filterSize = filterSize # Run correction controls.progress = cuiprogress.CUIProgressBar() applier.apply(castelCorrection, infiles, outfiles, otherargs, controls=controls)
def produceTotal(obs): """ Set up rios to apply images """ controls = applier.ApplierControls() controls.setReferenceImage(obs.inputImages[0]) controls.setOutputDriverName('HFA') infiles = applier.FilenameAssociations() infiles.images = obs.inputImages outfiles = applier.FilenameAssociations() if obs.stage == 'dix': if obs.fraction == 'bare': outStage = 'djl' elif obs.fraction == 'green': outStage = 'djm' elif obs.fraction == 'dry': outStage = 'djn' outfiles.image = qvf.changestage(qvf.changedate(os.path.basename(obs.inputImages[0]),obs.era),outStage) obs.outfile = outfiles.image otherargs = applier.OtherInputs() imginfo = fileinfo.ImageInfo(obs.inputImages[0]) otherargs.coverNull = imginfo.nodataval[0] controls.setStatsIgnore(otherargs.coverNull) applier.apply(model, infiles, outfiles, otherargs, controls=controls) addHistory(obs)
def makeSaturationMask(fmaskConfig, radiancefile, outMask): """ Checks the radianceFile and creates a mask with 1's where there is saturation in one of more visible bands. 0 otherwise. The fmaskConfig parameter should be an instance of :class:`fmask.config.FmaskConfig`. This is used to determine which bands are visible. This mask is advisible since the whiteness test Eqn 2. and Equation 6 are affected by saturated pixels and may determine a pixel is not cloud when it is. The format of outMask will be the current RIOS default format. It is assumed that the input radianceFile has values in the range 0-255 and saturated pixels are set to 255. """ inputs = applier.FilenameAssociations() inputs.radiance = radiancefile outputs = applier.FilenameAssociations() outputs.mask = outMask otherargs = applier.OtherInputs() otherargs.radianceBands = fmaskConfig.bands controls = applier.ApplierControls() controls.progress = cuiprogress.GDALProgressBar() applier.apply(riosSaturationMask, inputs, outputs, otherargs, controls=controls)
def run(data): # Load the machine learning estimators (greenEstimator,nonGreenEstimator,bareEstimator) = load_estimators() # Read in the FC ML Models otherargs = applier.OtherInputs() otherargs.greenEstimator = greenEstimator otherargs.nonGreenEstimator = nonGreenEstimator otherargs.bareEstimator = bareEstimator # Setup file name associations infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() # Loop for each image for feature in data.get("features"): print("Found GeoJSON feature:\n%s" % feature) # Find the filename rel_image_path = feature["properties"].get(AOICLIPPED) print("Processing %s" % rel_image_path) infiles.nbar = "/tmp/input/%s" % rel_image_path outfiles.fc = os.path.splitext("/tmp/output/%s" % rel_image_path)[0] + "_fc.tif" # Set up processing controls controls = applier.ApplierControls() controls.setStatsIgnore(0) controls.setOutputDriverName("GTIFF") controls.setCreationOptions(["COMPRESS=DEFLATE", "ZLEVEL=9", "BIGTIFF=YES", "TILED=YES", "INTERLEAVE=BAND", "NUM_THREADS=ALL_CPUS"]) # Run the model applier.apply(unmixfc, infiles, outfiles, otherargs, controls=controls) print("Feature has an FC image at %s" % outfiles.fc) return data
def isBandAllZeroes(filename, band=0): """ Checks the specified band within a file to see if it is all zeroes. band should be 0 based. Returns True if all zeroes, False otherwise. This function firstly checks the stats if present and uses this and assumes that they are correct. If they are not present, then the band is read and the values determined. """ info = fileinfo.ImageFileStats(filename) maxVal = info[band].max if maxVal is not None: # we have valid stats return maxVal == 0 # otherwise read the thing with rios infiles = applier.FilenameAssociations() infiles.input = filename outfiles = applier.FilenameAssociations() otherArgs = applier.OtherInputs() otherArgs.nonZeroFound = False otherArgs.band = band applier.apply(riosAllZeroes, infiles, outfiles, otherArgs) return not otherArgs.nonZeroFound
def makeAnglesImage(templateimg, outfile, nadirLine, extentSunAngles, satAzimuth, imgInfo): """ Make a single output image file of the sun and satellite angles for every pixel in the template image. """ imgInfo = fileinfo.ImageInfo(templateimg) infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() infiles.img = templateimg outfiles.angles = outfile (ctrLat, ctrLong) = getCtrLatLong(imgInfo) otherargs.R = localRadius(ctrLat) otherargs.nadirLine = nadirLine otherargs.xMin = imgInfo.xMin otherargs.xMax = imgInfo.xMax otherargs.yMin = imgInfo.yMin otherargs.yMax = imgInfo.yMax otherargs.extentSunAngles = extentSunAngles otherargs.satAltitude = 705000 # Landsat nominal altitude in metres otherargs.satAzimuth = satAzimuth otherargs.radianScale = 100 # Store pixel values as (radians * radianScale) controls.setStatsIgnore(500) applier.apply(makeAngles, infiles, outfiles, otherargs, controls=controls)
def main(): metadataFile = sys.argv[1] imgFile = sys.argv[2] toaFile = sys.argv[3] d = float(sys.argv[4]) outfile = "%s_envi.exp" % (metadataFile.split('.')[0]) if os.path.exists(toaFile): os.remove(toaFile) (calFactors, solarZenithAngle, eSun, layerNames) = getParams(metadataFile) infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() infiles.raw = imgFile outfiles.toaFile = toaFile otherargs.calFactors = calFactors otherargs.solarZenithAngle = solarZenithAngle otherargs.pi = 3.14159265358979323846 otherargs.d = d otherargs.eSun = eSun info = fileinfo.ImageInfo(imgFile) xMin = np.floor(info.xMin) xMax = np.ceil(info.xMax) yMin = np.floor(info.yMin) yMax = np.ceil(info.yMax) proj = info.projection transform = info.transform xRes = info.xRes yRes = info.yRes otherargs.outNull = 0 print(otherargs.outNull) controls.setStatsIgnore(otherargs.outNull) controls.setOutputDriverName('GTiff') controls.setCreationOptions(['COMPRESS=LZW']) controls.setCreationOptions(['BIGTIFF=IF_SAFER']) pixgrid = pixelgrid.PixelGridDefn(geotransform=transform, xMin=xMin, xMax=xMax, yMin=yMin, yMax=yMax, xRes=xRes, yRes=yRes, projection=proj) print(pixgrid) controls.setReferencePixgrid(pixgrid) controls.setLayerNames(layerNames) controls.setWindowXsize(20) controls.setWindowYsize(20) progress = cuiprogress.GDALProgressBar() controls.setProgress(progress) applier.apply(doTOA, infiles, outfiles, otherargs, controls=controls)
def main(): infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() infiles.img = 'crap.img' #outfiles.outimg = 'crapz.img' outfiles.outimg = "/apollo/imagery/rsc/landsat/landsat57tm/wrs2/090_079/2015/201512/l7tmpa_p090r079_20151228_da0m6.img" applier.apply(doit, infiles, outfiles)
def calcImgsPxlMode(inputImgs, outputImg, gdalformat, no_data_val=0): """ Function which calculates the mode of a group of images. Warning, this function can be very slow. Use rsgislib.imagecalc.imagePixelColumnSummary :param inputImgs: the list of images :param outputImg: the output image file name and path (will be same dimensions as the input) :param gdalformat: the GDAL image file format of the output image file. """ import scipy.stats from rios import applier rsgis_utils = rsgislib.RSGISPyUtils() datatype = rsgis_utils.getRSGISLibDataTypeFromImg(inputImgs[0]) numpyDT = rsgis_utils.getNumpyDataType(datatype) try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.images = inputImgs outfiles = applier.FilenameAssociations() outfiles.outimage = outputImg otherargs = applier.OtherInputs() otherargs.no_data_val = no_data_val otherargs.numpyDT = numpyDT aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _applyCalcMode(info, inputs, outputs, otherargs): """ This is an internal rios function """ image_data = numpy.concatenate(inputs.images, axis=0).astype(numpy.float32) image_data[image_data == otherargs.no_data_val] = numpy.nan mode_arr, count_arr = scipy.stats.mode(image_data, axis=0, nan_policy='omit') outputs.outimage = mode_arr.astype(otherargs.numpyDT) applier.apply(_applyCalcMode, infiles, outfiles, otherargs, controls=aControls)
def update_uid_image(uid_img, chng_img, nxt_scr5_img, clrsky_img, obs_day_since_base, tmp_uid_tile): from rios import applier try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.uid_img = uid_img infiles.chng_img = chng_img infiles.nxt_scr5_img = nxt_scr5_img infiles.clrsky_img = clrsky_img outfiles = applier.FilenameAssociations() outfiles.uid_img_out = tmp_uid_tile otherargs = applier.OtherInputs() otherargs.obs_day_since_base = obs_day_since_base aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.omitPyramids = True aControls.calcStats = False def _update_uid_img(info, inputs, outputs, otherargs): """ This is an internal rios function """ uid_img_arr = numpy.array(inputs.uid_img, dtype=numpy.uint32, copy=True) start_year = inputs.uid_img[0] chng_feats = inputs.chng_img[0] new_chng_pxls = numpy.zeros_like(start_year) new_chng_pxls[(start_year == 0) & (chng_feats == 1)] = 1 chng_scr5_year = inputs.uid_img[4] new_scr5_pxls = numpy.zeros_like(chng_scr5_year) new_scr5_pxls[(chng_scr5_year == 0) & (inputs.nxt_scr5_img[0] == 1)] = 1 uid_img_arr[0, new_chng_pxls == 1] = 1970 uid_img_arr[1, new_chng_pxls == 1] = otherargs.obs_day_since_base uid_img_arr[2, inputs.clrsky_img[0] == 1] = 1970 uid_img_arr[3, inputs.clrsky_img[0] == 1] = otherargs.obs_day_since_base uid_img_arr[4, new_scr5_pxls == 1] = 1970 uid_img_arr[5, new_scr5_pxls == 1] = otherargs.obs_day_since_base outputs.uid_img_out = uid_img_arr applier.apply(_update_uid_img, infiles, outfiles, otherargs, controls=aControls)
def map_worker(self, input_image, output_image): print("Running map function...") infiles = applier.FilenameAssociations() infiles.input_image = input_image outfiles = applier.FilenameAssociations() outfiles.output_image = output_image applier.apply(self.map_func, infiles, outfiles, controls=self.get_controls())
def calcAverage(file1, file2, avgfile): """ Use RIOS to calculate the average of two files. """ infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() infiles.img = [file1, file2] outfiles.avg = avgfile applier.apply(doAvg, infiles, outfiles)
def predict_for_date(date, input_path, output_path, output_driver='KEA', num_processes=1): """Main function to generate the predicted image. Given an input image containing per-band model coefficients, outputs a multi-band predicted image over the same area. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. date: The date to predict in YYYY-MM-DD format. input_path: Path to the input image generated by get_model_coeffs.py. output_path: Path for the output image. output_driver: Short driver name for GDAL, e.g. KEA, GTiff. num_processes: Number of concurrent processes to use.""" # Create object to hold input files infile = applier.FilenameAssociations() infile.coeff_img = input_path # Create object to hold output file outfile = applier.FilenameAssociations() outfile.output_img = output_path # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set output file type app.setOutputDriverName(output_driver) # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Convert provided date to ordinal ordinal_date = datetime.strptime(date, '%Y-%m-%d').toordinal() # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.date_to_predict = ordinal_date # Get band names try: input_img = fileinfo.ImageInfo(infile.coeff_img) except: sys.exit('Could not find input image.') layer_names = np.unique([name.split('_')[0] for name in input_img.lnames]) app.setLayerNames(layer_names) applier.apply(gen_prediction, infile, outfile, otherArgs=other_args, controls=app)
def run(): """ Run the test """ riostestutils.reportStart(TESTNAME) # Create a multi-band file with some data in it. tstfile = 'multilayer.img' numBands = 5 ds = riostestutils.createTestFile(tstfile, numBands=numBands) onelayerArr = riostestutils.genRampArray() lyrList = [] for i in range(numBands): lyr = (onelayerArr + 1) band = ds.GetRasterBand(i + 1) band.WriteArray(lyr) lyrList.append(lyr) del ds stack = numpy.array(lyrList) # Sum of all pixels in bands 2 & 4 layerList = [2, 4] correctSum = sum([(stack[i - 1].astype(numpy.float64)).sum() for i in layerList]) # Now do it using RIOS infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() infiles.img = tstfile controls.selectInputImageLayers(layerList) otherargs.total = 0 # We will use this to check the number of layers being read otherargs.numLayers = len(layerList) otherargs.numLayersIsOK = True applier.apply(doSum, infiles, outfiles, otherargs, controls=controls) if correctSum != otherargs.total: riostestutils.report( TESTNAME, "Totals do not match: %s != %s" % (correctSum, otherargs.total)) ok = False else: riostestutils.report(TESTNAME, "Passed") ok = True os.remove(tstfile) return ok
def reduce_worker(self, input_images, output_image): print("Running reduce function...") infiles = applier.FilenameAssociations() infiles.input_images = input_images outfiles = applier.FilenameAssociations() outfiles.output_image = output_image applier.apply(self.reduce_func, infiles, outfiles, controls=self.get_controls()) return output_image
def apply_gain_to_img(input_img, output_img, gdal_format, gain, np_dtype, in_no_data, out_no_data): """ Apply a gain scale factor to an input image and save as an integer dataset. :param input_img: Input image file :param output_img: Output image file :param gdal_format: GDAL image format for output image :param gain: scale factor to be applied to the input image :param np_dtype: the numpy datatype for the output data (e.g., numpy.int16 or numpy.uint16) :param in_no_data: the input image no data value :param out_no_data: the no data value to be used for the output image """ try: import tqdm progress_bar = TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() def _apply_img_gain(info, inputs, outputs, otherargs): # Internal Function... out_img_arr = numpy.zeros_like(inputs.image, dtype=otherargs.np_dtype) out_img_arr[...] = numpy.around((inputs.image * otherargs.gain), 0) out_img_arr[inputs.image == otherargs.in_no_data] = otherargs.out_no_data outputs.outimage = out_img_arr infiles = applier.FilenameAssociations() infiles.image = input_img outfiles = applier.FilenameAssociations() outfiles.outimage = output_img otherargs = applier.OtherInputs() otherargs.gain = gain otherargs.np_dtype = np_dtype otherargs.in_no_data = in_no_data otherargs.out_no_data = out_no_data aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdal_format aControls.omitPyramids = True aControls.calcStats = False if gdal_format == 'GTIFF': aControls.creationoptions = GTIFF_CREATION_OPTS applier.apply(_apply_img_gain, infiles, outfiles, otherargs, controls=aControls)
def call_min_max_filter(infile, outfile, min=-1, max=1, block_size=512, n_threads=5): """ Function caller to min_max_filter which set values below the minimum or above the maximum to nan Args: infile: full path to the input file outfile: full path to the output file min: minimum allowed value max: maximum allowed value block_size: block size for RIOS block wise processing n_threads: number of threads Returns: GeoTiff with extreme values set to nan """ # inputs inputs = applier.FilenameAssociations() inputs.image = infile # parameters otherargs = applier.OtherInputs() otherargs.min = min otherargs.max = max # controls controls = applier.ApplierControls() controls.setWindowXsize(block_size) controls.setWindowYsize(block_size) controls.progress = cuiprogress.CUIProgressBar() controls.setOutputDriverName("GTiff") controls.setCreationOptions(["COMPRESS=NONE"]) controls.setCreationOptions(["INTERLEAVE=PIXEL"]) # parallelization currently not working due to issues with pickeling if n_threads > 1: # parallel.jobmanager.getAvailableJobManagerTypes() controls.setNumThreads(n_threads) controls.setJobManagerType('subproc') # output file outfiles = applier.FilenameAssociations() outfiles.outimage = outfile applier.apply(min_max_filter, inputs, outfiles, otherargs, controls=controls)
def calcWSG84PixelArea(img, out_img, scale=10000, gdalformat='KEA'): """ A function which calculates the area (in metres) of the pixel projected in WGS84. :param img: input image, for which the per-pixel area will be calculated. :param out_img: output image file. :param scale: scale the output area to unit of interest. Scale=10000(Ha), Scale=1(sq m), Scale=1000000(sq km), Scale=4046.856(Acre), Scale=2590000(sq miles), Scale=0.0929022668(sq feet) """ import rsgislib.tools from rios import applier try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() rsgis_utils = rsgislib.RSGISPyUtils() x_res, y_res = rsgis_utils.getImageRes(img) infiles = applier.FilenameAssociations() infiles.img = img outfiles = applier.FilenameAssociations() outfiles.outimage = out_img otherargs = applier.OtherInputs() otherargs.x_res = x_res otherargs.y_res = y_res otherargs.scale = float(scale) aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = False aControls.calcStats = False def _calcPixelArea(info, inputs, outputs, otherargs): xBlock, yBlock = info.getBlockCoordArrays() x_res_arr = numpy.zeros_like(yBlock, dtype=float) x_res_arr[...] = otherargs.x_res y_res_arr = numpy.zeros_like(yBlock, dtype=float) y_res_arr[...] = otherargs.y_res x_res_arr_m, y_res_arr_m = rsgislib.tools.degrees_to_metres( yBlock, x_res_arr, y_res_arr) outputs.outimage = numpy.expand_dims( (x_res_arr_m * y_res_arr_m) / otherargs.scale, axis=0) applier.apply(_calcPixelArea, infiles, outfiles, otherargs, controls=aControls)
def rescale_unmixing_results(input_img, output_img, gdalformat, scale_factor=1000): """ A function which rescales an output from a spectral unmixing (e.g., rsgislib.imagecalc.specunmixing.calc_unconstrained_unmixing) so that negative values are removed and each pixel sums to 1. :param inputImg: Input image with the spectral unmixing result (pixels need to range from 0-1) :param outputImg: Output image with the result of the rescaling (pixel values will be in range 0-1) :param gdalformat: the file format of the output file. """ try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() infiles = applier.FilenameAssociations() infiles.image = input_img outfiles = applier.FilenameAssociations() outfiles.outimage = output_img otherargs = applier.OtherInputs() otherargs.scale_factor = scale_factor aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _applyUnmixRescale(info, inputs, outputs, otherargs): """ This is an internal rios function """ inputs.image[inputs.image < 0] = 0 if otherargs.scale_factor == 1: outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpy.float32) else: outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpy.uint16) for idx in range(inputs.image.shape[0]): outputs.outimage[idx] = (inputs.image[idx] / numpy.sum( inputs.image, axis=0)) * otherargs.scale_factor applier.apply(_applyUnmixRescale, infiles, outfiles, otherargs, controls=aControls)
def findminclumpval(clumpsImg, valsImg, valBand, out_col): import rsgislib.rastergis import rsgislib.rastergis.ratutils from rios import applier import numpy try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: from rios import cuiprogress progress_bar = cuiprogress.GDALProgressBar() n_clumps = rsgislib.rastergis.getRATLength(clumpsImg) min_vals = numpy.zeros(n_clumps, dtype=int) infiles = applier.FilenameAssociations() infiles.clumpsImg = clumpsImg infiles.valsImg = valsImg outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() otherargs.min_vals = min_vals otherargs.valBand = valBand - 1 aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.omitPyramids = True aControls.calcStats = False def _findMinVals(info, inputs, outputs, otherargs): """ This is an internal rios function """ unqClumpVals = numpy.unique(inputs.clumpsImg) clumpValsFlat = inputs.clumpsImg.flatten() for clump in unqClumpVals: pxl_vals = inputs.valsImg[otherargs.valBand].flatten()[ clumpValsFlat == clump] pxl_vals = pxl_vals[pxl_vals > 0] if pxl_vals.shape[0] > 0: clump_min_val = pxl_vals.min() if otherargs.min_vals[clump] == 0: otherargs.min_vals[clump] = clump_min_val else: if clump_min_val < otherargs.min_vals[clump]: otherargs.min_vals[clump] = clump_min_val applier.apply(_findMinVals, infiles, outfiles, otherargs, controls=aControls) rsgislib.rastergis.ratutils.setColumnData(clumpsImg, out_col, min_vals)
def exec(input_dir, model_filepath, output_filepath): ts_utils.mkdirp(ts_utils.basedir(output_filepath)) images = ts_utils.get_filenames(input_dir, \ filter_prefix='ts_b', filter_suffix='.vrt') model = load_model(model_filepath) infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() otherargs.n_images = len(images) for i in range(0, otherargs.n_images): key = 'img' + str(i) setattr(infiles, key, images[i]) outfiles.result = output_filepath otherargs.model = model def genmap(info, inputs, outputs, otherargs): size_x, size_y = info.getBlockSize() outputs.result = np.empty((1, size_y, size_x), dtype=np.uint16) print("Processing status " + str(info.getPercent()) + "%") input_ts = [] for i in range(0, otherargs.n_images): key = 'img' + str(i) input_img = getattr(inputs, key) input_img = np.transpose(input_img) shape = input_img.shape input_img = input_img.reshape(shape[0] * shape[1], shape[2]) input_ts.append(input_img) input_ts = np.dstack(input_ts) predicted = np.argmax(otherargs.model.predict(input_ts), axis=1).astype(np.uint16) outputs.result = np.transpose( np.reshape(predicted, (size_x, size_y, 1))) controls = applier.ApplierControls() controls.setNumThreads(1) controls.setJobManagerType('multiprocessing') applier.apply(genmap, infiles, outfiles, otherargs, controls=controls)
def calcAverage(file1, file2, avgfile): """ Use RIOS to calculate the average of two files. """ infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() infiles.img = [file1, file2] outfiles.avg = avgfile controls = applier.ApplierControls() controls.setNumThreads(TEST_NCPUS) controls.setJobManagerType("multiprocessing") applier.apply(doAvg, infiles, outfiles, controls=controls)
def main(): """ Main routine """ cmdargs = getCmdargs() infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() controls = applier.ApplierControls() infiles.inimg = readList(cmdargs.infile) outfiles.avg = cmdargs.outfile applier.apply(doAnalysis, infiles, outfiles, controls=controls)
def calc_ratio_img(vv_img, vh_img, out_img, gdal_format): """ A function which calculates the ratio image of the VV/HV polarisations. :param vv_img: GDAL image with VV polarisation :param vh_img: GDAL image with VH polarisation :param out_img: Output image file. :param gdal_format: Output image file format. """ try: import tqdm progress_bar = TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() def _apply_calc_pol_ratio(info, inputs, outputs, otherargs): # Internal Function... ratio_img_arr = numpy.where( (numpy.isfinite(inputs.vv_img) & (inputs.vv_img > 0.0) & numpy.isfinite(inputs.vh_img) & (inputs.vh_img > 0.0)), inputs.vv_img / inputs.vh_img, 0.0) ratio_img_arr[numpy.isnan(ratio_img_arr)] = 0.0 ratio_img_arr[numpy.isinf(ratio_img_arr)] = 0.0 outputs.outimage = ratio_img_arr infiles = applier.FilenameAssociations() infiles.vv_img = vv_img infiles.vh_img = vh_img outfiles = applier.FilenameAssociations() outfiles.outimage = out_img otherargs = applier.OtherInputs() aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdal_format aControls.omitPyramids = True aControls.calcStats = False if gdal_format == 'GTIFF': aControls.creationoptions = GTIFF_CREATION_OPTS applier.apply(_apply_calc_pol_ratio, infiles, outfiles, otherargs, controls=aControls) logger.debug("Created ratio output image file: {}".format(out_img))
def main(): """ Main routine """ cmdargs = getCmdargs() infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() controls = applier.ApplierControls() infiles.counts = cmdargs.infile outfiles.percent = cmdargs.outfile controls.setStatsIgnore(PCNT_NULL) applier.apply(doPcnt, infiles, outfiles, controls=controls)
def calcAverage(file1, file2, avgfile): """ Use RIOS to calculate the average of two files. Allows nearest-neighbour resampling of the second file. """ infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() infiles.img = [file1, file2] outfiles.avg = avgfile controls = applier.ApplierControls() controls.setReferenceImage(file1) controls.setResampleMethod('near') controls.setProgress(cuiprogress.CUIProgressBar()) applier.apply(doAvg, infiles, outfiles, controls=controls)
def calc_valid_msk(input_img, output_img, gdal_format, no_data_val): """ A function to create a valid data mask image. :param input_img: Input image file :param output_img: Output image file :param gdal_format: GDAL image format for output image :param no_data_val: the input image no data value """ try: import tqdm progress_bar = TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() def _calc_valid_msk(info, inputs, outputs, otherargs): # Internal Function... input_shp = inputs.image.shape outputs.output_img = numpy.zeros((1, input_shp[1], input_shp[2]), dtype=numpy.uint8) for n in range(input_shp[0]): outputs.output_img[0] = numpy.where( inputs.image[n] != otherargs.no_data_val, 1, outputs.output_img[0]) infiles = applier.FilenameAssociations() infiles.image = input_img outfiles = applier.FilenameAssociations() outfiles.output_img = output_img otherargs = applier.OtherInputs() otherargs.no_data_val = no_data_val aControls = applier.ApplierControls() aControls.progress = progress_bar aControls.drivername = gdal_format aControls.omitPyramids = True aControls.calcStats = False if gdal_format == 'GTIFF': aControls.creationoptions = GTIFF_CREATION_OPTS applier.apply(_calc_valid_msk, infiles, outfiles, otherargs, controls=aControls)
def apply_rf_image(in_data_stack, out_image, rf_model, nodata_vals): """ Apply Random Forests model generated by scikit-learn to an input data stack and output image Requires: * in_data_stack - stack of all layers * out_image - output image * rf_model - model produced by scikit-learn * nodata_vals - array with a no-data value for each band Returns the mean and standard deviation of the output (predicted) image. """ # Apply to image infiles = applier.FilenameAssociations() infiles.inimage = in_data_stack outfiles = applier.FilenameAssociations() outfiles.outimage = out_image otherargs = applier.OtherInputs() otherargs.rf = rf_model otherargs.predict_sm = None # Array to store output SM # Pass in list of no data values for each layer otherargs.nodata_vals_list = nodata_vals controls = applier.ApplierControls() controls.setOutputDriverName( upscaling_utilities.get_gdal_format(out_image)) controls.setCalcStats(False) controls.progress = cuiprogress.CUIProgressBar() applier.apply(_rios_apply_rf_image, infiles, outfiles, otherargs, controls=controls) average_sm_predict = otherargs.predict_sm.mean() sd_sm_predict = otherargs.predict_sm.std() return average_sm_predict, sd_sm_predict
def createPlots(imageA, imageB, outPlot, bandA=1, bandB=1, labelA=None, labelB=None, plotMin=None, plotMax=None, scale=1, independentXY=False): controls = applier.ApplierControls() # Set up input images infiles = applier.FilenameAssociations() infiles.imageA = imageA infiles.imageB = imageB outfiles = applier.FilenameAssociations() # Set up parameters otherargs = applier.OtherInputs() otherargs.bandA = bandA - 1 otherargs.bandB = bandB - 1 otherargs.outdataA = numpy.array([], dtype=numpy.float32) otherargs.outdataB = numpy.array([], dtype=numpy.float32) otherargs.scale = scale # Get data print('Extracting data') controls.progress = cuiprogress.CUIProgressBar() # If necessary reproject image B to image A controls.setReferenceImage(infiles.imageA) controls.setResampleMethod('near') applier.apply(getData, infiles, outfiles, otherargs, controls=controls) # Produce plot print('Saving plot') plotData(otherargs.outdataA, otherargs.outdataB, outPlot, labelA, labelB, plotMin, plotMax, independentXY)
def main(): """ Main routine """ cmdargs = getCmdargs() infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() controls = applier.ApplierControls() controls.setOutputDriverName('GTiff') options = ['COMPRESS=LZW', 'BIGTIFF=YES', 'TILED=YES', 'INTERLEAVE=BAND', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256'] controls.setCreationOptions(options) controls.setWindowXsize(256) #set the rios block size to match the tiff tile size controls.setWindowYsize(256) controls.setStatsIgnore(255) infiles.inFrac = cmdargs.inFrac infiles.inDec = cmdargs.inDecile outfiles.comp = cmdargs.outfile #controls.setStatsIgnore(-110,imagename='comp') applier.apply(doAnalysis, infiles, outfiles, controls=controls)
def mainRoutine(): """ Main routine """ cmdargs = getCmdargs() infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() infiles.inimg = cmdargs.infile infiles.firemask = cmdargs.maskfile outfiles.outimg = cmdargs.outfile controls.setBurnAttribute('ID') imginfo = gdalcommon.info(cmdargs.infile) otherargs.nullval = imginfo.nodataval[0] controls.setStatsIgnore(otherargs.nullval) applier.apply(doMask, infiles, outfiles, otherargs, controls=controls) DNscaling.copyDNscaling(cmdargs.infile, cmdargs.outfile)
def findImgCorners(img, imgInfo): """ Find the corners of the data within the given template image Return a numpy array of (x, y) coordinates. The array has 2 columns, for X and Y. Each row is a corner, in the order: top-left, top-right, bottom-left, bottom-right. Uses RIOS to pass through the image searching for non-null data, and find the extremes. Assumes we are working with a full-swathe Landsat image. Each list element is a numpy array of (x, y) """ infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() infiles.img = img otherargs.tl = None otherargs.tr = None otherargs.bl = None otherargs.br = None otherargs.nullVal = imgInfo.nodataval[0] if otherargs.nullVal is None: otherargs.nullVal = 0 applier.apply(findCorners, infiles, outfiles, otherargs) corners = numpy.array([ otherargs.tl, otherargs.tr, otherargs.bl, otherargs.br, ]) return corners
def run(): """ Run the test """ allOK = True riostestutils.reportStart(TESTNAME) ramp1 = 'ramp1.img' ramp2 = 'ramp2.img' riostestutils.genRampImageFile(ramp1) # Second file is same as first, but shifted 100 pixels right and down. xLeft = riostestutils.DEFAULT_XLEFT + OFFSET_2ND_IMAGE yTop = riostestutils.DEFAULT_YTOP - OFFSET_2ND_IMAGE riostestutils.genRampImageFile(ramp2, xLeft=xLeft, yTop=yTop) # Set up some RIOS calls infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() # Simplest check. Can we get back the coordinates from a single input file. infiles.img = ramp1 otherargs.tstPixList = [] otherargs.centresList = [] applier.apply(getCoords, infiles, outfiles, otherargs, controls=controls) ok = checkCoords('1 file, overlap=0', otherargs, TSTPIX_1FILE_NOOVERLAP, CENTRES_1FILE_NOOVERLAP) if not ok: allOK = False # Single input file, with an overlap set otherargs.tstPixList = [] otherargs.centresList = [] controls.setOverlap(OLAP) applier.apply(getCoords, infiles, outfiles, otherargs, controls=controls) ok = checkCoords('1 file, overlap=2', otherargs, TSTPIX_1FILE_OVERLAP2, CENTRES_1FILE_OVERLAP2) if not ok: allOK = False # Two input files, with an overlap set infiles.img = [ramp1, ramp2] otherargs.tstPixList = [] otherargs.centresList = [] controls.setOverlap(OLAP) applier.apply(getCoords, infiles, outfiles, otherargs, controls=controls) ok = checkCoords('2 files, overlap=2', otherargs, TSTPIX_2FILE_OVERLAP2, CENTRES_2FILE_OVERLAP2) if not ok: allOK = False # Clean up for filename in [ramp1, ramp2]: os.remove(filename) if allOK: riostestutils.report(TESTNAME, "Passed") return allOK
def rescaleUnmixingResults(inputImg, outputImg, gdalformat): """ A function which rescales an output from a spectral unmixing (e.g., rsgislib.imagecalc.conSum1LinearSpecUnmix) so that negative values are removed and each pixel sums to 1. :param inputImg: Input image with the spectral unmixing result (pixels need to range from 0-1) :param outputImg: Output image with the result of the rescaling (pixel values will be in range 0-1) :param gdalformat: the file format of the output file. """ infiles = applier.FilenameAssociations() infiles.image = inputImg outfiles = applier.FilenameAssociations() outfiles.outimage = outputImg otherargs = applier.OtherInputs() aControls = applier.ApplierControls() aControls.progress = cuiprogress.CUIProgressBar() aControls.drivername = gdalformat aControls.omitPyramids = True aControls.calcStats = False def _applyUnmixRescale(info, inputs, outputs, otherargs): """ This is an internal rios function """ inputs.image[inputs.image < 0] = 0 outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpy.float32) for idx in range(inputs.image.shape[0]): outputs.outimage[idx] = inputs.image[idx] / numpy.sum(inputs.image, axis=0) applier.apply(_applyUnmixRescale, infiles, outfiles, otherargs, controls=aControls)
parser = argparse.ArgumentParser() parser.add_argument("-i", "--inimage", type=str, help="Input image",required=True) parser.add_argument("-o", "--outimage", type=str, help="Output image", required=True) args = parser.parse_args() infiles = applier.FilenameAssociations() infiles.inimage = args.inimage outfiles = applier.FilenameAssociations() outfiles.outimage = args.outimage controls = applier.ApplierControls() controls.progress = cuiprogress.CUIProgressBar() controls.setCalcStats(True) # Set up depths otherargs = applier.OtherInputs() otherargs.depths = np.arange(0,1.05,0.05) outBandNames = [] for depth in otherargs.depths: name = str(int(depth*100.)) + ' cm' outBandNames.append(name) controls.setLayerNames(outBandNames) applier.apply(genSM, infiles, outfiles, otherargs, controls=controls)
def main(): """ Main routine """ cmdargs = getCmdargs() infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() controls = applier.ApplierControls() controls = applier.ApplierControls() controls.setOutputDriverName('GTiff') options = ['COMPRESS=LZW', 'BIGTIFF=YES', 'TILED=YES', 'INTERLEAVE=BAND', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256'] controls.setCreationOptions(options) controls.setWindowXsize(256) #set the rios block size to match the tiff tile size controls.setWindowYsize(256) allWinter, allAnnual, allSummer = doSeason(cmdargs.imageList, cmdargs.inDir) #pdb.set_trace() for i in allWinter: infiles.inimg = i year = i[0] year = year[-10:-6] outDirName = cmdargs.outDir + "winter"+ "_" + year +".tif" outfiles.comp = outDirName applier.apply(doAnalysis, infiles, outfiles, controls=controls) for i in allSummer: infiles.inimg = i year = i[0] year = year[-10:-6] outDirName = cmdargs.outDir + "summer"+ "_" + year +".tif" outfiles.comp = outDirName applier.apply(doAnalysis, infiles, outfiles, controls=controls) for i in allAnnual: infiles.inimg = i year = i[0] year = year[-10:-6] outDirName = cmdargs.outDir + "annual"+ "_" + year +".tif" outfiles.comp = outDirName applier.apply(doAnalysis, infiles, outfiles, controls=controls)
def applyMasks(processedList): """ Do masking of output - create generic model to be used for all seasonal products. Uses cloud and shadow masks to mask image. Uses RSC derived masks if available and fmask if not. """ cloudList = [] shadowList = [] sceneList =[] maskImageList = [] processedListClipped = [] for scene in processedList: processedListClipped.append(qvf.changestage(scene, 'tmp')) if not os.path.exists(qvf.changestage(scene, 'tmp')): fMaskCloudImage = qvf.changestage(scene, 'dgr') fMaskShadowImage = qvf.changestage(scene, 'dgs') if qv.existsonfilestore(fMaskCloudImage): cloudMaskImage = fMaskCloudImage if qv.existsonfilestore(fMaskShadowImage): shadowMaskImage = fMaskShadowImage if qv.existsonfilestore(cloudMaskImage) and qv.existsonfilestore(shadowMaskImage) and qv.existsonfilestore(scene): cloudList.append(cloudMaskImage) shadowList.append(shadowMaskImage) sceneList.append(scene) maskImageList.append([scene,cloudMaskImage,shadowMaskImage]) if len(maskImageList)!=0: recallList(cloudList, qvf.stage(cloudList[0])) recallList(shadowList, qvf.stage(shadowList[0])) recallList(sceneList, qvf.stage(sceneList[0])) print 'Applying masks...' controls = applier.ApplierControls() controls.setStatsIgnore(0) for (image1,image2,image3) in maskImageList: tmpscene = qvf.changestage(image1, 'tmp') print tmpscene if not os.path.exists(tmpscene) and os.path.exists(image1) and os.path.exists(image2): infiles = applier.FilenameAssociations() infiles.image1 = image1 infiles.image2 = image2 infiles.image3 = image3 outfiles = applier.FilenameAssociations() outfiles.outimage = tmpscene if not os.path.exists(outfiles.outimage): applier.apply(createMaskedImage, infiles, outfiles, controls=controls) if os.path.exists(outfiles.outimage): os.remove(image1) os.remove(image2) os.remove(image3) return processedListClipped