def makeTOAReflectance(infile, mtlFile, anglesfile, outfile):
"""
Main routine - does the calculation
The eqn for TOA reflectance, p, is
p = pi * L * d^2 / E * cos(theta)
d = earthSunDistance(date)
L = image pixel (radiance)
E = exoatmospheric irradiance for the band, and
theta = solar zenith angle.
Assumes infile is radiance values in DN from USGS.
mtlFile is the .mtl file.
outfile will be created in the default format that RIOS
is configured to use and will be top of atmosphere
reflectance values *10000. Also assumes that the
angles image file is scaled as radians*100, and has layers for
satAzimuth, satZenith, sunAzimuth, sunZenith, in that order.
"""
mtlInfo = config.readMTLFile(mtlFile)
spaceCraft = mtlInfo['SPACECRAFT_ID']
date = mtlInfo['DATE_ACQUIRED']
date = date.replace('-', '')
inputs = applier.FilenameAssociations()
inputs.infile = infile
inputs.angles = anglesfile
outputs = applier.FilenameAssociations()
outputs.outfile = outfile
otherinputs = applier.OtherInputs()
otherinputs.earthSunDistance = earthSunDistance(date)
otherinputs.earthSunDistanceSq = otherinputs.earthSunDistance * otherinputs.earthSunDistance
otherinputs.esun = ESUN_LOOKUP[spaceCraft]
gains, offsets = readGainsOffsets(mtlInfo)
otherinputs.gains = gains
otherinputs.offsets = offsets
otherinputs.anglesToRadians = 0.01
otherinputs.outNull = 32767
imginfo = fileinfo.ImageInfo(infile)
otherinputs.inNull = imginfo.nodataval[0]
controls = applier.ApplierControls()
if platform.system() in ["Linux", "Darwin"]:
controls.setNumThreads(multiprocessing.cpu_count())
controls.setJobManagerType("multiprocessing")
controls.progress = cuiprogress.GDALProgressBar()
controls.setStatsIgnore(otherinputs.outNull)
controls.setCalcStats(False)
applier.apply(riosTOA, inputs, outputs, otherinputs, controls=controls)
# Explicitly set the null value in the output
ds = gdal.Open(outfile, gdal.GA_Update)
for i in range(ds.RasterCount):
ds.GetRasterBand(i + 1).SetNoDataValue(otherinputs.outNull)
def runCorrection(insigma, inlinc, inpix, outsigma, thetaref=39.0, nFactor=1.0, filterSize=None):
controls = applier.ApplierControls()
# Set up input images
infiles = applier.FilenameAssociations()
infiles.insigma = insigma
infiles.inlinc = inlinc
infiles.inpix = inpix
# Set up output image
outfiles = applier.FilenameAssociations()
outfiles.outimage = outsigma
# Set format for output image
outControls = getOutDriver(outsigma)
# Set options
controls.setOutputDriverName(outControls['gdalFormat'])
controls.setCreationOptions(outControls['gdalCOOptions'])
controls.setCalcStats(outControls['calcStats'])
# Set up parameters
otherargs = applier.OtherInputs()
otherargs.thetaref = thetaref
otherargs.nFactor = nFactor
otherargs.filterSize = filterSize
# Run correction
controls.progress = cuiprogress.CUIProgressBar()
applier.apply(castelCorrection, infiles, outfiles, otherargs, controls=controls)
def produceTotal(obs):
"""
Set up rios to apply images
"""
controls = applier.ApplierControls()
controls.setReferenceImage(obs.inputImages[0])
controls.setOutputDriverName('HFA')
infiles = applier.FilenameAssociations()
infiles.images = obs.inputImages
outfiles = applier.FilenameAssociations()
if obs.stage == 'dix':
if obs.fraction == 'bare':
outStage = 'djl'
elif obs.fraction == 'green':
outStage = 'djm'
elif obs.fraction == 'dry':
outStage = 'djn'
outfiles.image = qvf.changestage(qvf.changedate(os.path.basename(obs.inputImages[0]),obs.era),outStage)
obs.outfile = outfiles.image
otherargs = applier.OtherInputs()
imginfo = fileinfo.ImageInfo(obs.inputImages[0])
otherargs.coverNull = imginfo.nodataval[0]
controls.setStatsIgnore(otherargs.coverNull)
applier.apply(model, infiles, outfiles, otherargs, controls=controls)
addHistory(obs)
def makeSaturationMask(fmaskConfig, radiancefile, outMask):
"""
Checks the radianceFile and creates a mask with
1's where there is saturation in one of more visible bands.
0 otherwise.
The fmaskConfig parameter should be an instance of
:class:`fmask.config.FmaskConfig`. This is used to determine
which bands are visible.
This mask is advisible since the whiteness test Eqn 2.
and Equation 6 are affected by saturated pixels and
may determine a pixel is not cloud when it is.
The format of outMask will be the current
RIOS default format.
It is assumed that the input radianceFile has values in the
range 0-255 and saturated pixels are set to 255.
"""
inputs = applier.FilenameAssociations()
inputs.radiance = radiancefile
outputs = applier.FilenameAssociations()
outputs.mask = outMask
otherargs = applier.OtherInputs()
otherargs.radianceBands = fmaskConfig.bands
controls = applier.ApplierControls()
controls.progress = cuiprogress.GDALProgressBar()
applier.apply(riosSaturationMask, inputs, outputs,
otherargs, controls=controls)
def run(data):
# Load the machine learning estimators
(greenEstimator,nonGreenEstimator,bareEstimator) = load_estimators()
# Read in the FC ML Models
otherargs = applier.OtherInputs()
otherargs.greenEstimator = greenEstimator
otherargs.nonGreenEstimator = nonGreenEstimator
otherargs.bareEstimator = bareEstimator
# Setup file name associations
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
# Loop for each image
for feature in data.get("features"):
print("Found GeoJSON feature:\n%s" % feature)
# Find the filename
rel_image_path = feature["properties"].get(AOICLIPPED)
print("Processing %s" % rel_image_path)
infiles.nbar = "/tmp/input/%s" % rel_image_path
outfiles.fc = os.path.splitext("/tmp/output/%s" % rel_image_path)[0] + "_fc.tif"
# Set up processing controls
controls = applier.ApplierControls()
controls.setStatsIgnore(0)
controls.setOutputDriverName("GTIFF")
controls.setCreationOptions(["COMPRESS=DEFLATE",
"ZLEVEL=9",
"BIGTIFF=YES",
"TILED=YES",
"INTERLEAVE=BAND",
"NUM_THREADS=ALL_CPUS"])
# Run the model
applier.apply(unmixfc, infiles, outfiles, otherargs, controls=controls)
print("Feature has an FC image at %s" % outfiles.fc)
return data
def isBandAllZeroes(filename, band=0):
"""
Checks the specified band within a file to see if it is all zeroes.
band should be 0 based.
Returns True if all zeroes, False otherwise.
This function firstly checks the stats if present and uses this
and assumes that they are correct.
If they are not present, then the band is read and the values determined.
"""
info = fileinfo.ImageFileStats(filename)
maxVal = info[band].max
if maxVal is not None:
# we have valid stats
return maxVal == 0
# otherwise read the thing with rios
infiles = applier.FilenameAssociations()
infiles.input = filename
outfiles = applier.FilenameAssociations()
otherArgs = applier.OtherInputs()
otherArgs.nonZeroFound = False
otherArgs.band = band
applier.apply(riosAllZeroes, infiles, outfiles, otherArgs)
return not otherArgs.nonZeroFound
def makeAnglesImage(templateimg, outfile, nadirLine, extentSunAngles,
satAzimuth, imgInfo):
"""
Make a single output image file of the sun and satellite angles for every
pixel in the template image.
"""
imgInfo = fileinfo.ImageInfo(templateimg)
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
infiles.img = templateimg
outfiles.angles = outfile
(ctrLat, ctrLong) = getCtrLatLong(imgInfo)
otherargs.R = localRadius(ctrLat)
otherargs.nadirLine = nadirLine
otherargs.xMin = imgInfo.xMin
otherargs.xMax = imgInfo.xMax
otherargs.yMin = imgInfo.yMin
otherargs.yMax = imgInfo.yMax
otherargs.extentSunAngles = extentSunAngles
otherargs.satAltitude = 705000 # Landsat nominal altitude in metres
otherargs.satAzimuth = satAzimuth
otherargs.radianScale = 100 # Store pixel values as (radians * radianScale)
controls.setStatsIgnore(500)
applier.apply(makeAngles, infiles, outfiles, otherargs, controls=controls)
def main():
metadataFile = sys.argv[1]
imgFile = sys.argv[2]
toaFile = sys.argv[3]
d = float(sys.argv[4])
outfile = "%s_envi.exp" % (metadataFile.split('.')[0])
if os.path.exists(toaFile):
os.remove(toaFile)
(calFactors, solarZenithAngle, eSun, layerNames) = getParams(metadataFile)
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
infiles.raw = imgFile
outfiles.toaFile = toaFile
otherargs.calFactors = calFactors
otherargs.solarZenithAngle = solarZenithAngle
otherargs.pi = 3.14159265358979323846
otherargs.d = d
otherargs.eSun = eSun
info = fileinfo.ImageInfo(imgFile)
xMin = np.floor(info.xMin)
xMax = np.ceil(info.xMax)
yMin = np.floor(info.yMin)
yMax = np.ceil(info.yMax)
proj = info.projection
transform = info.transform
xRes = info.xRes
yRes = info.yRes
otherargs.outNull = 0
print(otherargs.outNull)
controls.setStatsIgnore(otherargs.outNull)
controls.setOutputDriverName('GTiff')
controls.setCreationOptions(['COMPRESS=LZW'])
controls.setCreationOptions(['BIGTIFF=IF_SAFER'])
pixgrid = pixelgrid.PixelGridDefn(geotransform=transform,
xMin=xMin,
xMax=xMax,
yMin=yMin,
yMax=yMax,
xRes=xRes,
yRes=yRes,
projection=proj)
print(pixgrid)
controls.setReferencePixgrid(pixgrid)
controls.setLayerNames(layerNames)
controls.setWindowXsize(20)
controls.setWindowYsize(20)
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
applier.apply(doTOA, infiles, outfiles, otherargs, controls=controls)
def main():
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.img = 'crap.img'
#outfiles.outimg = 'crapz.img'
outfiles.outimg = "/apollo/imagery/rsc/landsat/landsat57tm/wrs2/090_079/2015/201512/l7tmpa_p090r079_20151228_da0m6.img"
applier.apply(doit, infiles, outfiles)
def calcImgsPxlMode(inputImgs, outputImg, gdalformat, no_data_val=0):
"""
Function which calculates the mode of a group of images.
Warning, this function can be very slow. Use rsgislib.imagecalc.imagePixelColumnSummary
:param inputImgs: the list of images
:param outputImg: the output image file name and path (will be same dimensions as the input)
:param gdalformat: the GDAL image file format of the output image file.
"""
import scipy.stats
from rios import applier
rsgis_utils = rsgislib.RSGISPyUtils()
datatype = rsgis_utils.getRSGISLibDataTypeFromImg(inputImgs[0])
numpyDT = rsgis_utils.getNumpyDataType(datatype)
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.images = inputImgs
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
otherargs.no_data_val = no_data_val
otherargs.numpyDT = numpyDT
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyCalcMode(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
image_data = numpy.concatenate(inputs.images,
axis=0).astype(numpy.float32)
image_data[image_data == otherargs.no_data_val] = numpy.nan
mode_arr, count_arr = scipy.stats.mode(image_data,
axis=0,
nan_policy='omit')
outputs.outimage = mode_arr.astype(otherargs.numpyDT)
applier.apply(_applyCalcMode,
infiles,
outfiles,
otherargs,
controls=aControls)
def update_uid_image(uid_img, chng_img, nxt_scr5_img, clrsky_img,
obs_day_since_base, tmp_uid_tile):
from rios import applier
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.uid_img = uid_img
infiles.chng_img = chng_img
infiles.nxt_scr5_img = nxt_scr5_img
infiles.clrsky_img = clrsky_img
outfiles = applier.FilenameAssociations()
outfiles.uid_img_out = tmp_uid_tile
otherargs = applier.OtherInputs()
otherargs.obs_day_since_base = obs_day_since_base
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.omitPyramids = True
aControls.calcStats = False
def _update_uid_img(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
uid_img_arr = numpy.array(inputs.uid_img,
dtype=numpy.uint32,
copy=True)
start_year = inputs.uid_img[0]
chng_feats = inputs.chng_img[0]
new_chng_pxls = numpy.zeros_like(start_year)
new_chng_pxls[(start_year == 0) & (chng_feats == 1)] = 1
chng_scr5_year = inputs.uid_img[4]
new_scr5_pxls = numpy.zeros_like(chng_scr5_year)
new_scr5_pxls[(chng_scr5_year == 0)
& (inputs.nxt_scr5_img[0] == 1)] = 1
uid_img_arr[0, new_chng_pxls == 1] = 1970
uid_img_arr[1, new_chng_pxls == 1] = otherargs.obs_day_since_base
uid_img_arr[2, inputs.clrsky_img[0] == 1] = 1970
uid_img_arr[3,
inputs.clrsky_img[0] == 1] = otherargs.obs_day_since_base
uid_img_arr[4, new_scr5_pxls == 1] = 1970
uid_img_arr[5, new_scr5_pxls == 1] = otherargs.obs_day_since_base
outputs.uid_img_out = uid_img_arr
applier.apply(_update_uid_img,
infiles,
outfiles,
otherargs,
controls=aControls)
def map_worker(self, input_image, output_image):
print("Running map function...")
infiles = applier.FilenameAssociations()
infiles.input_image = input_image
outfiles = applier.FilenameAssociations()
outfiles.output_image = output_image
applier.apply(self.map_func, infiles, outfiles,
controls=self.get_controls())
def calcAverage(file1, file2, avgfile):
"""
Use RIOS to calculate the average of two files.
"""
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.img = [file1, file2]
outfiles.avg = avgfile
applier.apply(doAvg, infiles, outfiles)
def predict_for_date(date,
input_path,
output_path,
output_driver='KEA',
num_processes=1):
"""Main function to generate the predicted image. Given an input image containing
per-band model coefficients, outputs a multi-band predicted image over the same area.
Opening/closing of files, generation of blocks and use of multiprocessing is
all handled by RIOS.
date: The date to predict in YYYY-MM-DD format.
input_path: Path to the input image generated by get_model_coeffs.py.
output_path: Path for the output image.
output_driver: Short driver name for GDAL, e.g. KEA, GTiff.
num_processes: Number of concurrent processes to use."""
# Create object to hold input files
infile = applier.FilenameAssociations()
infile.coeff_img = input_path
# Create object to hold output file
outfile = applier.FilenameAssociations()
outfile.output_img = output_path
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set output file type
app.setOutputDriverName(output_driver)
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Convert provided date to ordinal
ordinal_date = datetime.strptime(date, '%Y-%m-%d').toordinal()
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.date_to_predict = ordinal_date
# Get band names
try:
input_img = fileinfo.ImageInfo(infile.coeff_img)
except:
sys.exit('Could not find input image.')
layer_names = np.unique([name.split('_')[0] for name in input_img.lnames])
app.setLayerNames(layer_names)
applier.apply(gen_prediction,
infile,
outfile,
otherArgs=other_args,
controls=app)
def run():
"""
Run the test
"""
riostestutils.reportStart(TESTNAME)
# Create a multi-band file with some data in it.
tstfile = 'multilayer.img'
numBands = 5
ds = riostestutils.createTestFile(tstfile, numBands=numBands)
onelayerArr = riostestutils.genRampArray()
lyrList = []
for i in range(numBands):
lyr = (onelayerArr + 1)
band = ds.GetRasterBand(i + 1)
band.WriteArray(lyr)
lyrList.append(lyr)
del ds
stack = numpy.array(lyrList)
# Sum of all pixels in bands 2 & 4
layerList = [2, 4]
correctSum = sum([(stack[i - 1].astype(numpy.float64)).sum()
for i in layerList])
# Now do it using RIOS
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
infiles.img = tstfile
controls.selectInputImageLayers(layerList)
otherargs.total = 0
# We will use this to check the number of layers being read
otherargs.numLayers = len(layerList)
otherargs.numLayersIsOK = True
applier.apply(doSum, infiles, outfiles, otherargs, controls=controls)
if correctSum != otherargs.total:
riostestutils.report(
TESTNAME,
"Totals do not match: %s != %s" % (correctSum, otherargs.total))
ok = False
else:
riostestutils.report(TESTNAME, "Passed")
ok = True
os.remove(tstfile)
return ok
def reduce_worker(self, input_images, output_image):
print("Running reduce function...")
infiles = applier.FilenameAssociations()
infiles.input_images = input_images
outfiles = applier.FilenameAssociations()
outfiles.output_image = output_image
applier.apply(self.reduce_func, infiles, outfiles,
controls=self.get_controls())
return output_image
def apply_gain_to_img(input_img, output_img, gdal_format, gain, np_dtype,
in_no_data, out_no_data):
"""
Apply a gain scale factor to an input image and save as an integer dataset.
:param input_img: Input image file
:param output_img: Output image file
:param gdal_format: GDAL image format for output image
:param gain: scale factor to be applied to the input image
:param np_dtype: the numpy datatype for the output data (e.g., numpy.int16 or numpy.uint16)
:param in_no_data: the input image no data value
:param out_no_data: the no data value to be used for the output image
"""
try:
import tqdm
progress_bar = TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
def _apply_img_gain(info, inputs, outputs, otherargs):
# Internal Function...
out_img_arr = numpy.zeros_like(inputs.image, dtype=otherargs.np_dtype)
out_img_arr[...] = numpy.around((inputs.image * otherargs.gain), 0)
out_img_arr[inputs.image ==
otherargs.in_no_data] = otherargs.out_no_data
outputs.outimage = out_img_arr
infiles = applier.FilenameAssociations()
infiles.image = input_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.gain = gain
otherargs.np_dtype = np_dtype
otherargs.in_no_data = in_no_data
otherargs.out_no_data = out_no_data
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdal_format
aControls.omitPyramids = True
aControls.calcStats = False
if gdal_format == 'GTIFF':
aControls.creationoptions = GTIFF_CREATION_OPTS
applier.apply(_apply_img_gain,
infiles,
outfiles,
otherargs,
controls=aControls)
def call_min_max_filter(infile,
outfile,
min=-1,
max=1,
block_size=512,
n_threads=5):
"""
Function caller to min_max_filter which set values below the minimum or above the maximum to nan
Args:
infile: full path to the input file
outfile: full path to the output file
min: minimum allowed value
max: maximum allowed value
block_size: block size for RIOS block wise processing
n_threads: number of threads
Returns: GeoTiff with extreme values set to nan
"""
# inputs
inputs = applier.FilenameAssociations()
inputs.image = infile
# parameters
otherargs = applier.OtherInputs()
otherargs.min = min
otherargs.max = max
# controls
controls = applier.ApplierControls()
controls.setWindowXsize(block_size)
controls.setWindowYsize(block_size)
controls.progress = cuiprogress.CUIProgressBar()
controls.setOutputDriverName("GTiff")
controls.setCreationOptions(["COMPRESS=NONE"])
controls.setCreationOptions(["INTERLEAVE=PIXEL"])
# parallelization currently not working due to issues with pickeling
if n_threads > 1:
# parallel.jobmanager.getAvailableJobManagerTypes()
controls.setNumThreads(n_threads)
controls.setJobManagerType('subproc')
# output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = outfile
applier.apply(min_max_filter,
inputs,
outfiles,
otherargs,
controls=controls)
def calcWSG84PixelArea(img, out_img, scale=10000, gdalformat='KEA'):
"""
A function which calculates the area (in metres) of the pixel projected in WGS84.
:param img: input image, for which the per-pixel area will be calculated.
:param out_img: output image file.
:param scale: scale the output area to unit of interest. Scale=10000(Ha), Scale=1(sq m), Scale=1000000(sq km), Scale=4046.856(Acre), Scale=2590000(sq miles), Scale=0.0929022668(sq feet)
"""
import rsgislib.tools
from rios import applier
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
rsgis_utils = rsgislib.RSGISPyUtils()
x_res, y_res = rsgis_utils.getImageRes(img)
infiles = applier.FilenameAssociations()
infiles.img = img
outfiles = applier.FilenameAssociations()
outfiles.outimage = out_img
otherargs = applier.OtherInputs()
otherargs.x_res = x_res
otherargs.y_res = y_res
otherargs.scale = float(scale)
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = False
aControls.calcStats = False
def _calcPixelArea(info, inputs, outputs, otherargs):
xBlock, yBlock = info.getBlockCoordArrays()
x_res_arr = numpy.zeros_like(yBlock, dtype=float)
x_res_arr[...] = otherargs.x_res
y_res_arr = numpy.zeros_like(yBlock, dtype=float)
y_res_arr[...] = otherargs.y_res
x_res_arr_m, y_res_arr_m = rsgislib.tools.degrees_to_metres(
yBlock, x_res_arr, y_res_arr)
outputs.outimage = numpy.expand_dims(
(x_res_arr_m * y_res_arr_m) / otherargs.scale, axis=0)
applier.apply(_calcPixelArea,
infiles,
outfiles,
otherargs,
controls=aControls)
def rescale_unmixing_results(input_img,
output_img,
gdalformat,
scale_factor=1000):
"""
A function which rescales an output from a spectral unmixing
(e.g., rsgislib.imagecalc.specunmixing.calc_unconstrained_unmixing) so that
negative values are removed and each pixel sums to 1.
:param inputImg: Input image with the spectral unmixing result (pixels need to range from 0-1)
:param outputImg: Output image with the result of the rescaling (pixel values will be in range 0-1)
:param gdalformat: the file format of the output file.
"""
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
infiles = applier.FilenameAssociations()
infiles.image = input_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_img
otherargs = applier.OtherInputs()
otherargs.scale_factor = scale_factor
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyUnmixRescale(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
inputs.image[inputs.image < 0] = 0
if otherargs.scale_factor == 1:
outputs.outimage = numpy.zeros_like(inputs.image,
dtype=numpy.float32)
else:
outputs.outimage = numpy.zeros_like(inputs.image,
dtype=numpy.uint16)
for idx in range(inputs.image.shape[0]):
outputs.outimage[idx] = (inputs.image[idx] / numpy.sum(
inputs.image, axis=0)) * otherargs.scale_factor
applier.apply(_applyUnmixRescale,
infiles,
outfiles,
otherargs,
controls=aControls)
def findminclumpval(clumpsImg, valsImg, valBand, out_col):
import rsgislib.rastergis
import rsgislib.rastergis.ratutils
from rios import applier
import numpy
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
from rios import cuiprogress
progress_bar = cuiprogress.GDALProgressBar()
n_clumps = rsgislib.rastergis.getRATLength(clumpsImg)
min_vals = numpy.zeros(n_clumps, dtype=int)
infiles = applier.FilenameAssociations()
infiles.clumpsImg = clumpsImg
infiles.valsImg = valsImg
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
otherargs.min_vals = min_vals
otherargs.valBand = valBand - 1
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.omitPyramids = True
aControls.calcStats = False
def _findMinVals(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
unqClumpVals = numpy.unique(inputs.clumpsImg)
clumpValsFlat = inputs.clumpsImg.flatten()
for clump in unqClumpVals:
pxl_vals = inputs.valsImg[otherargs.valBand].flatten()[
clumpValsFlat == clump]
pxl_vals = pxl_vals[pxl_vals > 0]
if pxl_vals.shape[0] > 0:
clump_min_val = pxl_vals.min()
if otherargs.min_vals[clump] == 0:
otherargs.min_vals[clump] = clump_min_val
else:
if clump_min_val < otherargs.min_vals[clump]:
otherargs.min_vals[clump] = clump_min_val
applier.apply(_findMinVals,
infiles,
outfiles,
otherargs,
controls=aControls)
rsgislib.rastergis.ratutils.setColumnData(clumpsImg, out_col, min_vals)
def exec(input_dir, model_filepath, output_filepath):
ts_utils.mkdirp(ts_utils.basedir(output_filepath))
images = ts_utils.get_filenames(input_dir, \
filter_prefix='ts_b', filter_suffix='.vrt')
model = load_model(model_filepath)
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
otherargs.n_images = len(images)
for i in range(0, otherargs.n_images):
key = 'img' + str(i)
setattr(infiles, key, images[i])
outfiles.result = output_filepath
otherargs.model = model
def genmap(info, inputs, outputs, otherargs):
size_x, size_y = info.getBlockSize()
outputs.result = np.empty((1, size_y, size_x), dtype=np.uint16)
print("Processing status " + str(info.getPercent()) + "%")
input_ts = []
for i in range(0, otherargs.n_images):
key = 'img' + str(i)
input_img = getattr(inputs, key)
input_img = np.transpose(input_img)
shape = input_img.shape
input_img = input_img.reshape(shape[0] * shape[1], shape[2])
input_ts.append(input_img)
input_ts = np.dstack(input_ts)
predicted = np.argmax(otherargs.model.predict(input_ts),
axis=1).astype(np.uint16)
outputs.result = np.transpose(
np.reshape(predicted, (size_x, size_y, 1)))
controls = applier.ApplierControls()
controls.setNumThreads(1)
controls.setJobManagerType('multiprocessing')
applier.apply(genmap, infiles, outfiles, otherargs, controls=controls)
def calcAverage(file1, file2, avgfile):
"""
Use RIOS to calculate the average of two files.
"""
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.img = [file1, file2]
outfiles.avg = avgfile
controls = applier.ApplierControls()
controls.setNumThreads(TEST_NCPUS)
controls.setJobManagerType("multiprocessing")
applier.apply(doAvg, infiles, outfiles, controls=controls)
def main():
"""
Main routine
"""
cmdargs = getCmdargs()
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
controls = applier.ApplierControls()
infiles.inimg = readList(cmdargs.infile)
outfiles.avg = cmdargs.outfile
applier.apply(doAnalysis, infiles, outfiles, controls=controls)
def calc_ratio_img(vv_img, vh_img, out_img, gdal_format):
"""
A function which calculates the ratio image of the VV/HV polarisations.
:param vv_img: GDAL image with VV polarisation
:param vh_img: GDAL image with VH polarisation
:param out_img: Output image file.
:param gdal_format: Output image file format.
"""
try:
import tqdm
progress_bar = TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
def _apply_calc_pol_ratio(info, inputs, outputs, otherargs):
# Internal Function...
ratio_img_arr = numpy.where(
(numpy.isfinite(inputs.vv_img) &
(inputs.vv_img > 0.0) & numpy.isfinite(inputs.vh_img) &
(inputs.vh_img > 0.0)), inputs.vv_img / inputs.vh_img, 0.0)
ratio_img_arr[numpy.isnan(ratio_img_arr)] = 0.0
ratio_img_arr[numpy.isinf(ratio_img_arr)] = 0.0
outputs.outimage = ratio_img_arr
infiles = applier.FilenameAssociations()
infiles.vv_img = vv_img
infiles.vh_img = vh_img
outfiles = applier.FilenameAssociations()
outfiles.outimage = out_img
otherargs = applier.OtherInputs()
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdal_format
aControls.omitPyramids = True
aControls.calcStats = False
if gdal_format == 'GTIFF':
aControls.creationoptions = GTIFF_CREATION_OPTS
applier.apply(_apply_calc_pol_ratio,
infiles,
outfiles,
otherargs,
controls=aControls)
logger.debug("Created ratio output image file: {}".format(out_img))
def main():
"""
Main routine
"""
cmdargs = getCmdargs()
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
controls = applier.ApplierControls()
infiles.counts = cmdargs.infile
outfiles.percent = cmdargs.outfile
controls.setStatsIgnore(PCNT_NULL)
applier.apply(doPcnt, infiles, outfiles, controls=controls)
def calcAverage(file1, file2, avgfile):
"""
Use RIOS to calculate the average of two files. Allows
nearest-neighbour resampling of the second file.
"""
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.img = [file1, file2]
outfiles.avg = avgfile
controls = applier.ApplierControls()
controls.setReferenceImage(file1)
controls.setResampleMethod('near')
controls.setProgress(cuiprogress.CUIProgressBar())
applier.apply(doAvg, infiles, outfiles, controls=controls)
def calc_valid_msk(input_img, output_img, gdal_format, no_data_val):
"""
A function to create a valid data mask image.
:param input_img: Input image file
:param output_img: Output image file
:param gdal_format: GDAL image format for output image
:param no_data_val: the input image no data value
"""
try:
import tqdm
progress_bar = TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
def _calc_valid_msk(info, inputs, outputs, otherargs):
# Internal Function...
input_shp = inputs.image.shape
outputs.output_img = numpy.zeros((1, input_shp[1], input_shp[2]),
dtype=numpy.uint8)
for n in range(input_shp[0]):
outputs.output_img[0] = numpy.where(
inputs.image[n] != otherargs.no_data_val, 1,
outputs.output_img[0])
infiles = applier.FilenameAssociations()
infiles.image = input_img
outfiles = applier.FilenameAssociations()
outfiles.output_img = output_img
otherargs = applier.OtherInputs()
otherargs.no_data_val = no_data_val
aControls = applier.ApplierControls()
aControls.progress = progress_bar
aControls.drivername = gdal_format
aControls.omitPyramids = True
aControls.calcStats = False
if gdal_format == 'GTIFF':
aControls.creationoptions = GTIFF_CREATION_OPTS
applier.apply(_calc_valid_msk,
infiles,
outfiles,
otherargs,
controls=aControls)
def apply_rf_image(in_data_stack, out_image, rf_model, nodata_vals):
"""
Apply Random Forests model generated by scikit-learn
to an input data stack and output image
Requires:
* in_data_stack - stack of all layers
* out_image - output image
* rf_model - model produced by scikit-learn
* nodata_vals - array with a no-data value for each band
Returns the mean and standard deviation of the output (predicted)
image.
"""
# Apply to image
infiles = applier.FilenameAssociations()
infiles.inimage = in_data_stack
outfiles = applier.FilenameAssociations()
outfiles.outimage = out_image
otherargs = applier.OtherInputs()
otherargs.rf = rf_model
otherargs.predict_sm = None # Array to store output SM
# Pass in list of no data values for each layer
otherargs.nodata_vals_list = nodata_vals
controls = applier.ApplierControls()
controls.setOutputDriverName(
upscaling_utilities.get_gdal_format(out_image))
controls.setCalcStats(False)
controls.progress = cuiprogress.CUIProgressBar()
applier.apply(_rios_apply_rf_image,
infiles,
outfiles,
otherargs,
controls=controls)
average_sm_predict = otherargs.predict_sm.mean()
sd_sm_predict = otherargs.predict_sm.std()
return average_sm_predict, sd_sm_predict
def createPlots(imageA,
imageB,
outPlot,
bandA=1,
bandB=1,
labelA=None,
labelB=None,
plotMin=None,
plotMax=None,
scale=1,
independentXY=False):
controls = applier.ApplierControls()
# Set up input images
infiles = applier.FilenameAssociations()
infiles.imageA = imageA
infiles.imageB = imageB
outfiles = applier.FilenameAssociations()
# Set up parameters
otherargs = applier.OtherInputs()
otherargs.bandA = bandA - 1
otherargs.bandB = bandB - 1
otherargs.outdataA = numpy.array([], dtype=numpy.float32)
otherargs.outdataB = numpy.array([], dtype=numpy.float32)
otherargs.scale = scale
# Get data
print('Extracting data')
controls.progress = cuiprogress.CUIProgressBar()
# If necessary reproject image B to image A
controls.setReferenceImage(infiles.imageA)
controls.setResampleMethod('near')
applier.apply(getData, infiles, outfiles, otherargs, controls=controls)
# Produce plot
print('Saving plot')
plotData(otherargs.outdataA, otherargs.outdataB, outPlot, labelA, labelB,
plotMin, plotMax, independentXY)
def main():
"""
Main routine
"""
cmdargs = getCmdargs()
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
controls = applier.ApplierControls()
controls.setOutputDriverName('GTiff')
options = ['COMPRESS=LZW', 'BIGTIFF=YES', 'TILED=YES', 'INTERLEAVE=BAND', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256']
controls.setCreationOptions(options)
controls.setWindowXsize(256) #set the rios block size to match the tiff tile size
controls.setWindowYsize(256)
controls.setStatsIgnore(255)
infiles.inFrac = cmdargs.inFrac
infiles.inDec = cmdargs.inDecile
outfiles.comp = cmdargs.outfile
#controls.setStatsIgnore(-110,imagename='comp')
applier.apply(doAnalysis, infiles, outfiles, controls=controls)
def mainRoutine():
"""
Main routine
"""
cmdargs = getCmdargs()
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
infiles.inimg = cmdargs.infile
infiles.firemask = cmdargs.maskfile
outfiles.outimg = cmdargs.outfile
controls.setBurnAttribute('ID')
imginfo = gdalcommon.info(cmdargs.infile)
otherargs.nullval = imginfo.nodataval[0]
controls.setStatsIgnore(otherargs.nullval)
applier.apply(doMask, infiles, outfiles, otherargs, controls=controls)
DNscaling.copyDNscaling(cmdargs.infile, cmdargs.outfile)
def findImgCorners(img, imgInfo):
"""
Find the corners of the data within the given template image
Return a numpy array of (x, y) coordinates. The array has 2 columns, for X and Y.
Each row is a corner, in the order:
top-left, top-right, bottom-left, bottom-right.
Uses RIOS to pass through the image searching for non-null data,
and find the extremes. Assumes we are working with a full-swathe Landsat
image.
Each list element is a numpy array of (x, y)
"""
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
infiles.img = img
otherargs.tl = None
otherargs.tr = None
otherargs.bl = None
otherargs.br = None
otherargs.nullVal = imgInfo.nodataval[0]
if otherargs.nullVal is None:
otherargs.nullVal = 0
applier.apply(findCorners, infiles, outfiles, otherargs)
corners = numpy.array([
otherargs.tl,
otherargs.tr,
otherargs.bl,
otherargs.br,
])
return corners
def run():
"""
Run the test
"""
allOK = True
riostestutils.reportStart(TESTNAME)
ramp1 = 'ramp1.img'
ramp2 = 'ramp2.img'
riostestutils.genRampImageFile(ramp1)
# Second file is same as first, but shifted 100 pixels right and down.
xLeft = riostestutils.DEFAULT_XLEFT + OFFSET_2ND_IMAGE
yTop = riostestutils.DEFAULT_YTOP - OFFSET_2ND_IMAGE
riostestutils.genRampImageFile(ramp2, xLeft=xLeft, yTop=yTop)
# Set up some RIOS calls
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
# Simplest check. Can we get back the coordinates from a single input file.
infiles.img = ramp1
otherargs.tstPixList = []
otherargs.centresList = []
applier.apply(getCoords, infiles, outfiles, otherargs, controls=controls)
ok = checkCoords('1 file, overlap=0', otherargs, TSTPIX_1FILE_NOOVERLAP, CENTRES_1FILE_NOOVERLAP)
if not ok: allOK = False
# Single input file, with an overlap set
otherargs.tstPixList = []
otherargs.centresList = []
controls.setOverlap(OLAP)
applier.apply(getCoords, infiles, outfiles, otherargs, controls=controls)
ok = checkCoords('1 file, overlap=2', otherargs, TSTPIX_1FILE_OVERLAP2, CENTRES_1FILE_OVERLAP2)
if not ok: allOK = False
# Two input files, with an overlap set
infiles.img = [ramp1, ramp2]
otherargs.tstPixList = []
otherargs.centresList = []
controls.setOverlap(OLAP)
applier.apply(getCoords, infiles, outfiles, otherargs, controls=controls)
ok = checkCoords('2 files, overlap=2', otherargs, TSTPIX_2FILE_OVERLAP2, CENTRES_2FILE_OVERLAP2)
if not ok: allOK = False
# Clean up
for filename in [ramp1, ramp2]:
os.remove(filename)
if allOK:
riostestutils.report(TESTNAME, "Passed")
return allOK
def rescaleUnmixingResults(inputImg, outputImg, gdalformat):
"""
A function which rescales an output from a spectral unmixing
(e.g., rsgislib.imagecalc.conSum1LinearSpecUnmix) so that
negative values are removed and each pixel sums to 1.
:param inputImg: Input image with the spectral unmixing result (pixels need to range from 0-1)
:param outputImg: Output image with the result of the rescaling (pixel values will be in range 0-1)
:param gdalformat: the file format of the output file.
"""
infiles = applier.FilenameAssociations()
infiles.image = inputImg
outfiles = applier.FilenameAssociations()
outfiles.outimage = outputImg
otherargs = applier.OtherInputs()
aControls = applier.ApplierControls()
aControls.progress = cuiprogress.CUIProgressBar()
aControls.drivername = gdalformat
aControls.omitPyramids = True
aControls.calcStats = False
def _applyUnmixRescale(info, inputs, outputs, otherargs):
"""
This is an internal rios function
"""
inputs.image[inputs.image < 0] = 0
outputs.outimage = numpy.zeros_like(inputs.image, dtype=numpy.float32)
for idx in range(inputs.image.shape[0]):
outputs.outimage[idx] = inputs.image[idx] / numpy.sum(inputs.image,
axis=0)
applier.apply(_applyUnmixRescale,
infiles,
outfiles,
otherargs,
controls=aControls)
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--inimage", type=str, help="Input image",required=True)
parser.add_argument("-o", "--outimage", type=str, help="Output image", required=True)
args = parser.parse_args()
infiles = applier.FilenameAssociations()
infiles.inimage = args.inimage
outfiles = applier.FilenameAssociations()
outfiles.outimage = args.outimage
controls = applier.ApplierControls()
controls.progress = cuiprogress.CUIProgressBar()
controls.setCalcStats(True)
# Set up depths
otherargs = applier.OtherInputs()
otherargs.depths = np.arange(0,1.05,0.05)
outBandNames = []
for depth in otherargs.depths:
name = str(int(depth*100.)) + ' cm'
outBandNames.append(name)
controls.setLayerNames(outBandNames)
applier.apply(genSM, infiles, outfiles, otherargs, controls=controls)
def main():
"""
Main routine
"""
cmdargs = getCmdargs()
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
controls = applier.ApplierControls()
controls = applier.ApplierControls()
controls.setOutputDriverName('GTiff')
options = ['COMPRESS=LZW', 'BIGTIFF=YES', 'TILED=YES', 'INTERLEAVE=BAND', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256']
controls.setCreationOptions(options)
controls.setWindowXsize(256) #set the rios block size to match the tiff tile size
controls.setWindowYsize(256)
allWinter, allAnnual, allSummer = doSeason(cmdargs.imageList, cmdargs.inDir)
#pdb.set_trace()
for i in allWinter:
infiles.inimg = i
year = i[0]
year = year[-10:-6]
outDirName = cmdargs.outDir + "winter"+ "_" + year +".tif"
outfiles.comp = outDirName
applier.apply(doAnalysis, infiles, outfiles, controls=controls)
for i in allSummer:
infiles.inimg = i
year = i[0]
year = year[-10:-6]
outDirName = cmdargs.outDir + "summer"+ "_" + year +".tif"
outfiles.comp = outDirName
applier.apply(doAnalysis, infiles, outfiles, controls=controls)
for i in allAnnual:
infiles.inimg = i
year = i[0]
year = year[-10:-6]
outDirName = cmdargs.outDir + "annual"+ "_" + year +".tif"
outfiles.comp = outDirName
applier.apply(doAnalysis, infiles, outfiles, controls=controls)
def applyMasks(processedList):
"""
Do masking of output - create generic model to be used for all seasonal products.
Uses cloud and shadow masks to mask image. Uses RSC derived masks if available
and fmask if not.
"""
cloudList = []
shadowList = []
sceneList =[]
maskImageList = []
processedListClipped = []
for scene in processedList:
processedListClipped.append(qvf.changestage(scene, 'tmp'))
if not os.path.exists(qvf.changestage(scene, 'tmp')):
fMaskCloudImage = qvf.changestage(scene, 'dgr')
fMaskShadowImage = qvf.changestage(scene, 'dgs')
if qv.existsonfilestore(fMaskCloudImage):
cloudMaskImage = fMaskCloudImage
if qv.existsonfilestore(fMaskShadowImage):
shadowMaskImage = fMaskShadowImage
if qv.existsonfilestore(cloudMaskImage) and qv.existsonfilestore(shadowMaskImage) and qv.existsonfilestore(scene):
cloudList.append(cloudMaskImage)
shadowList.append(shadowMaskImage)
sceneList.append(scene)
maskImageList.append([scene,cloudMaskImage,shadowMaskImage])
if len(maskImageList)!=0:
recallList(cloudList, qvf.stage(cloudList[0]))
recallList(shadowList, qvf.stage(shadowList[0]))
recallList(sceneList, qvf.stage(sceneList[0]))
print 'Applying masks...'
controls = applier.ApplierControls()
controls.setStatsIgnore(0)
for (image1,image2,image3) in maskImageList:
tmpscene = qvf.changestage(image1, 'tmp')
print tmpscene
if not os.path.exists(tmpscene) and os.path.exists(image1) and os.path.exists(image2):
infiles = applier.FilenameAssociations()
infiles.image1 = image1
infiles.image2 = image2
infiles.image3 = image3
outfiles = applier.FilenameAssociations()
outfiles.outimage = tmpscene
if not os.path.exists(outfiles.outimage):
applier.apply(createMaskedImage, infiles, outfiles, controls=controls)
if os.path.exists(outfiles.outimage):
os.remove(image1)
os.remove(image2)
os.remove(image3)
return processedListClipped
