def checkAnglesFile(inputAnglesFile, toafile): """ Check that the resolution of the input angles file matches that of the input TOA reflectance file. If not, make a VRT file which will resample it on-the-fly. Only checks the resolution, assumes that if these match, then everything else will match too. Return the name of the angles file to use. """ toaImgInfo = fileinfo.ImageInfo(toafile) anglesImgInfo = fileinfo.ImageInfo(inputAnglesFile) outputAnglesFile = inputAnglesFile if (toaImgInfo.xRes != anglesImgInfo.xRes) or (toaImgInfo.yRes != anglesImgInfo.yRes): (fd, vrtName) = tempfile.mkstemp(prefix='angles', suffix='.vrt') os.close(fd) subprocess.check_call([ GDALWARPCMD, '-q', '-of', 'VRT', '-tr', str(toaImgInfo.xRes), str(toaImgInfo.yRes), '-te', str(toaImgInfo.xMin), str(toaImgInfo.yMin), str(toaImgInfo.xMax), str(toaImgInfo.yMax), '-r', 'near', inputAnglesFile, vrtName ]) outputAnglesFile = vrtName return outputAnglesFile
def checkAnglesFile(inputAnglesFile, toafile): """ Check that the resolution of the input angles file matches that of the input TOA reflectance file. If not, make a VRT file which will resample it on-the-fly. Only checks the resolution, assumes that if these match, then everything else will match too. Return the name of the angles file to use. """ toaImgInfo = fileinfo.ImageInfo(toafile) anglesImgInfo = fileinfo.ImageInfo(inputAnglesFile) outputAnglesFile = inputAnglesFile if (toaImgInfo.xRes != anglesImgInfo.xRes) or (toaImgInfo.yRes != anglesImgInfo.yRes): (fd, vrtName) = tempfile.mkstemp(prefix="angles", suffix=".vrt") os.close(fd) cmdFmt = ( "gdalwarp -q -of VRT -tr {xres} {yres} -te {xmin} {ymin} {xmax} {ymax} " + "-r near {infile} {outfile} ") cmd = cmdFmt.format( xres=toaImgInfo.xRes, yres=toaImgInfo.yRes, xmin=toaImgInfo.xMin, ymin=toaImgInfo.yMin, xmax=toaImgInfo.xMax, ymax=toaImgInfo.yMax, outfile=vrtName, infile=inputAnglesFile, ) os.system(cmd) outputAnglesFile = vrtName return outputAnglesFile
def makeAnglesImage(templateimg, outfile, nadirLine, extentSunAngles, satAzimuth, imgInfo): """ Make a single output image file of the sun and satellite angles for every pixel in the template image. """ imgInfo = fileinfo.ImageInfo(templateimg) infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() infiles.img = templateimg outfiles.angles = outfile (ctrLat, ctrLong) = getCtrLatLong(imgInfo) otherargs.R = localRadius(ctrLat) otherargs.nadirLine = nadirLine otherargs.xMin = imgInfo.xMin otherargs.xMax = imgInfo.xMax otherargs.yMin = imgInfo.yMin otherargs.yMax = imgInfo.yMax otherargs.extentSunAngles = extentSunAngles otherargs.satAltitude = 705000 # Landsat nominal altitude in metres otherargs.satAzimuth = satAzimuth otherargs.radianScale = 100 # Store pixel values as (radians * radianScale) controls.setStatsIgnore(500) applier.apply(makeAngles, infiles, outfiles, otherargs, controls=controls)
def LandsatFmaskRoutine(MTLfile,toafile='toa.img',themalfile='thermal.img', anglesfile='angles.img',outfile='cloud.img', keepintermediates=False,verbose=True, tempdir='.',mincloudsize=0, cloudprobthreshold=100 * fmask.config.FmaskConfig.Eqn17CloudProbThresh, nirsnowthreshold=fmask.config.FmaskConfig.Eqn20NirSnowThresh, greensnowthreshold=fmask.config.FmaskConfig.Eqn20GreenSnowThresh, cloudbufferdistance=300,shadowbufferdistance=300): thermalInfo = fmask.config.readThermalInfoFromLandsatMTL(MTLfile) anglesInfo = fmask.config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) mtlInfo = fmask.config.readMTLFile(MTLfile) sensor=GetLandsatSensor(MTLfile) fmaskFilenames = fmask.config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(toafile) fmaskFilenames.setThermalFile(themalfile) fmaskFilenames.setOutputCloudMaskFile(outfile) fmaskConfig = fmask.config.FmaskConfig(sensor) saturationcheck.makeSaturationMask(fmaskConfig,'ref.img','saturationmask.img') fmaskFilenames.setSaturationMask('saturationmask.img') fmaskConfig.setThermalInfo(thermalInfo) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(keepintermediates) fmaskConfig.setVerbose(verbose) fmaskConfig.setTempDir(tempdir) fmaskConfig.setMinCloudSize(mincloudsize) fmaskConfig.setEqn17CloudProbThresh(cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(greensnowthreshold) toaImgInfo = fileinfo.ImageInfo(toafile) fmaskConfig.setCloudBufferSize(int(cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize(int(shadowbufferdistance / toaImgInfo.xRes)) fmask.fmask.doFmask(fmaskFilenames, fmaskConfig)
def makeTOAReflectance(infile, mtlFile, anglesfile, outfile): """ Main routine - does the calculation The eqn for TOA reflectance, p, is p = pi * L * d^2 / E * cos(theta) d = earthSunDistance(date) L = image pixel (radiance) E = exoatmospheric irradiance for the band, and theta = solar zenith angle. Assumes infile is radiance values in DN from USGS. mtlFile is the .mtl file. outfile will be created in the default format that RIOS is configured to use and will be top of atmosphere reflectance values *10000. Also assumes that the angles image file is scaled as radians*100, and has layers for satAzimuth, satZenith, sunAzimuth, sunZenith, in that order. """ mtlInfo = config.readMTLFile(mtlFile) spaceCraft = mtlInfo['SPACECRAFT_ID'] date = mtlInfo['DATE_ACQUIRED'] date = date.replace('-', '') inputs = applier.FilenameAssociations() inputs.infile = infile inputs.angles = anglesfile outputs = applier.FilenameAssociations() outputs.outfile = outfile otherinputs = applier.OtherInputs() otherinputs.earthSunDistance = earthSunDistance(date) otherinputs.earthSunDistanceSq = otherinputs.earthSunDistance * otherinputs.earthSunDistance otherinputs.esun = ESUN_LOOKUP[spaceCraft] gains, offsets = readGainsOffsets(mtlInfo) otherinputs.gains = gains otherinputs.offsets = offsets otherinputs.anglesToRadians = 0.01 otherinputs.outNull = 32767 imginfo = fileinfo.ImageInfo(infile) otherinputs.inNull = imginfo.nodataval[0] controls = applier.ApplierControls() if platform.system() in ["Linux", "Darwin"]: controls.setNumThreads(multiprocessing.cpu_count()) controls.setJobManagerType("multiprocessing") controls.progress = cuiprogress.GDALProgressBar() controls.setStatsIgnore(otherinputs.outNull) controls.setCalcStats(False) applier.apply(riosTOA, inputs, outputs, otherinputs, controls=controls) # Explicitly set the null value in the output ds = gdal.Open(outfile, gdal.GA_Update) for i in range(ds.RasterCount): ds.GetRasterBand(i + 1).SetNoDataValue(otherinputs.outNull)
def main(): metadataFile = sys.argv[1] imgFile = sys.argv[2] toaFile = sys.argv[3] d = float(sys.argv[4]) outfile = "%s_envi.exp" % (metadataFile.split('.')[0]) if os.path.exists(toaFile): os.remove(toaFile) (calFactors, solarZenithAngle, eSun, layerNames) = getParams(metadataFile) infiles = applier.FilenameAssociations() outfiles = applier.FilenameAssociations() otherargs = applier.OtherInputs() controls = applier.ApplierControls() infiles.raw = imgFile outfiles.toaFile = toaFile otherargs.calFactors = calFactors otherargs.solarZenithAngle = solarZenithAngle otherargs.pi = 3.14159265358979323846 otherargs.d = d otherargs.eSun = eSun info = fileinfo.ImageInfo(imgFile) xMin = np.floor(info.xMin) xMax = np.ceil(info.xMax) yMin = np.floor(info.yMin) yMax = np.ceil(info.yMax) proj = info.projection transform = info.transform xRes = info.xRes yRes = info.yRes otherargs.outNull = 0 print(otherargs.outNull) controls.setStatsIgnore(otherargs.outNull) controls.setOutputDriverName('GTiff') controls.setCreationOptions(['COMPRESS=LZW']) controls.setCreationOptions(['BIGTIFF=IF_SAFER']) pixgrid = pixelgrid.PixelGridDefn(geotransform=transform, xMin=xMin, xMax=xMax, yMin=yMin, yMax=yMax, xRes=xRes, yRes=yRes, projection=proj) print(pixgrid) controls.setReferencePixgrid(pixgrid) controls.setLayerNames(layerNames) controls.setWindowXsize(20) controls.setWindowYsize(20) progress = cuiprogress.GDALProgressBar() controls.setProgress(progress) applier.apply(doTOA, infiles, outfiles, otherargs, controls=controls)
def get_angle_images(self, DST=None): """ :param DST: Optional name of the output tif containing all angles images :return: set self.angles_file Following band order : SAT_AZ , SAT_ZENITH, SUN_AZ, SUN_ZENITH ') The unit is RADIANS """ # downsample factor F = 10 if DST is not None: out_file = DST else: out_file = os.path.join(self.product_path, 'tie_points.tif') if self.ang_filename != 'not found' and sys.platform == 'linux2': self.ang_filename = os.path.join(self.product_path, self.ang_filename) ls8_angles_exe = os.path.join(BINDIR, 'l8_angles', 'l8_angles') args = [ ls8_angles_exe, os.path.abspath(self.ang_filename), 'SATELLITE {} -b 1,2,3,4,5,6,7'.format(F) ] subprocess.check_call(' '.join(args), shell=True, cwd=os.path.dirname(out_file)) args = [ ls8_angles_exe, os.path.abspath(self.ang_filename), 'SOLAR {} -b 1'.format(F) ] subprocess.check_call(' '.join(args), shell=True, cwd=os.path.dirname(out_file)) mtlInfo = config.readMTLFile(self.mtl_file_name) image = self.reflective_band_list[0] # downsample image for angle computation dirname = os.path.dirname(out_file) if not os.path.exists(dirname): os.makedirs(dirname) coarseResImage = os.path.join(dirname, 'tie_points_coarseResImage.tif') gdal.Translate(coarseResImage, image, xRes=30 * F, yRes=30 * F) imgInfo = fileinfo.ImageInfo(coarseResImage) corners = landsatangles.findImgCorners(coarseResImage, imgInfo) nadirLine = landsatangles.findNadirLine(corners) extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) satAzimuth = landsatangles.satAzLeftRight(nadirLine) # do not use fmask function but internal custom function self.makeAnglesImage(coarseResImage, out_file, nadirLine, extentSunAngles, satAzimuth, imgInfo) log.info('SAT_AZ , SAT_ZENITH, SUN_AZ, SUN_ZENITH ') log.info('UNIT = DEGREES (scale: x100) :') log.info(' ' + out_file) self.angles_file = out_file
def mainRoutine(cmdargs): """ Main routine that calls fmask """ # 1040nm thermal band should always be the first (or only) band in a # stack of Landsat thermal bands thermalInfo = config.readThermalInfoFromLandsatMTL(cmdargs.mtl) anglesfile = cmdargs.anglesfile anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) mtlInfo = config.readMTLFile(cmdargs.mtl) landsat = mtlInfo["SPACECRAFT_ID"][-1] if landsat == "4": sensor = config.FMASK_LANDSAT47 elif landsat == "5": sensor = config.FMASK_LANDSAT47 elif landsat == "7": sensor = config.FMASK_LANDSAT47 elif landsat == "8": sensor = config.FMASK_LANDSAT8 else: raise SystemExit("Unsupported Landsat sensor") fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) fmaskFilenames.setThermalFile(cmdargs.thermal) fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) if cmdargs.saturation is not None: fmaskFilenames.setSaturationMask(cmdargs.saturation) else: print( "saturation mask not supplied - see fmask_usgsLandsatSaturationMask.py" ) fmaskConfig = config.FmaskConfig(sensor) fmaskConfig.setThermalInfo(thermalInfo) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) fmaskConfig.setVerbose(cmdargs.verbose) fmaskConfig.setTempDir(cmdargs.tempdir) fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) fmaskConfig.setCloudBufferSize( int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize( int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) fmask.doFmask(fmaskFilenames, fmaskConfig)
def predict_for_date(date, input_path, output_path, output_driver='KEA', num_processes=1): """Main function to generate the predicted image. Given an input image containing per-band model coefficients, outputs a multi-band predicted image over the same area. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. date: The date to predict in YYYY-MM-DD format. input_path: Path to the input image generated by get_model_coeffs.py. output_path: Path for the output image. output_driver: Short driver name for GDAL, e.g. KEA, GTiff. num_processes: Number of concurrent processes to use.""" # Create object to hold input files infile = applier.FilenameAssociations() infile.coeff_img = input_path # Create object to hold output file outfile = applier.FilenameAssociations() outfile.output_img = output_path # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set output file type app.setOutputDriverName(output_driver) # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Convert provided date to ordinal ordinal_date = datetime.strptime(date, '%Y-%m-%d').toordinal() # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.date_to_predict = ordinal_date # Get band names try: input_img = fileinfo.ImageInfo(infile.coeff_img) except: sys.exit('Could not find input image.') layer_names = np.unique([name.split('_')[0] for name in input_img.lnames]) app.setLayerNames(layer_names) applier.apply(gen_prediction, infile, outfile, otherArgs=other_args, controls=app)
def mainRoutine(argv=None): """ Main routine that calls fmask If argv is given, it should be a list of pairs of parameter and arguments like in command line. See mainRoutine(['-h']) for details on available parameters. Example: mainRoutine(['--safedir', '<.SAFE directory>', '-o', '<output file>']) If argv is None or not given, command line sys.args are used, see argparse.parse_args. """ cmdargs = getCmdargs(argv) tempStack = False if cmdargs.safedir is not None or cmdargs.granuledir is not None: tempStack = True resampledBands = makeStackAndAngles(cmdargs) anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa) anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) fmaskConfig.setVerbose(cmdargs.verbose) fmaskConfig.setTempDir(cmdargs.tempdir) fmaskConfig.setTOARefScaling(10000.0) fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest) # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) fmaskConfig.setCloudBufferSize( int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize( int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) fmask.doFmask(fmaskFilenames, fmaskConfig) if (anglesfile != cmdargs.anglesfile): # Must have been a temporary, so remove it os.remove(anglesfile) if tempStack and not cmdargs.keepintermediates: for fn in [cmdargs.toa, cmdargs.anglesfile]: if os.path.exists(fn): os.remove(fn)
def chooseResampleMethod(outpixsize, inBandImg): """ Choose the right resample method, given the image and the desired output pixel size """ imginfo = fileinfo.ImageInfo(inBandImg) inPixsize = imginfo.xRes if outpixsize == inPixsize: resample = "near" elif outpixsize > inPixsize: resample = "average" else: resample = "cubic" return resample
def makeAngles(mtlfile, templateimg, outfile): """ Callable main routine """ mtlInfo = config.readMTLFile(mtlfile) imgInfo = fileinfo.ImageInfo(templateimg) corners = landsatangles.findImgCorners(templateimg, imgInfo) nadirLine = landsatangles.findNadirLine(corners) extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) satAzimuth = landsatangles.satAzLeftRight(nadirLine) landsatangles.makeAnglesImage(templateimg, outfile, nadirLine, extentSunAngles, satAzimuth, imgInfo)
def mainRoutine(): """ Main routine that calls fmask """ cmdargs = getCmdargs() tempStack = False if cmdargs.safedir is not None or cmdargs.granuledir is not None: tempStack = True resampledBands = makeStackAndAngles(cmdargs) anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa) anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) fmaskConfig.setVerbose(cmdargs.verbose) fmaskConfig.setTempDir(cmdargs.tempdir) fmaskConfig.setTOARefScaling(10000.0) fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest) # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) fmaskConfig.setCloudBufferSize( int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize( int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) fmask.doFmask(fmaskFilenames, fmaskConfig) if (anglesfile != cmdargs.anglesfile): # Must have been a temporary, so remove it os.remove(anglesfile) if tempStack and not cmdargs.keepintermediates: for fn in [cmdargs.toa, cmdargs.anglesfile]: if os.path.exists(fn): os.remove(fn)
def mainRoutine(cmdargs): """ Main routine """ mtlInfo = config.readMTLFile(cmdargs.mtl) imgInfo = fileinfo.ImageInfo(cmdargs.templateimg) corners = landsatangles.findImgCorners(cmdargs.templateimg, imgInfo) nadirLine = landsatangles.findNadirLine(corners) extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) satAzimuth = landsatangles.satAzLeftRight(nadirLine) landsatangles.makeAnglesImage(cmdargs.templateimg, cmdargs.outfile, nadirLine, extentSunAngles, satAzimuth, imgInfo)
def autofmask(dirname): os.chdir(dirname) MTLfile = glob(os.path.join(dirname, '*MTL.TXT')) refname=os.path.join(dirname,'ref.img') themalname=os.path.join(dirname,'thermal.img') srcReflist=os.path.join(dirname,'L*_B[1,2,3,4,5,7].TIF') srcReflist=glob(srcReflist) srcThemal=os.path.join(dirname,'L*_B6.TIF') srcThemal=glob(srcThemal) anglesname=os.path.join(dirname,'angles.img') toaname=os.path.join(dirname,'toa.img') # 合并文件 refMergeArgv = ['', '-separate', '-of', 'HFA', '-co', 'COMPRESSED=YES', '-o', refname] refMergeArgv.extend(srcReflist) themalMergeArgv = ['', '-separate', '-of', 'HFA', '-co', 'COMPRESSED=YES', '-o',themalname] themalMergeArgv.extend(srcThemal) if not os.path.exists(refname): gdal_merge.main(refMergeArgv) else: print('跳过组合多光谱') if not os.path.exists(themalname): gdal_merge.main(themalMergeArgv) else: print('跳过组合热红外') # 生成角度文件 # 读取文件信息 MTLfile = MTLfile[0] mtlInfo = fmask.config.readMTLFile(MTLfile) if not os.path.exists(anglesname): imgInfo = fileinfo.ImageInfo(refname) corners = landsatangles.findImgCorners(refname, imgInfo) nadirLine = landsatangles.findNadirLine(corners) extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) satAzimuth = landsatangles.satAzLeftRight(nadirLine) landsatangles.makeAnglesImage(refname, anglesname, nadirLine, extentSunAngles, satAzimuth, imgInfo) # 生成辅助临时文件:反射率 if not os.path.exists(toaname): fmask.landsatTOA.makeTOAReflectance(refname, MTLfile, anglesname, toaname) print("begin this") LandsatFmaskRoutine(MTLfile)
def mainRoutine(): """ Main routine that calls fmask """ cmdargs = getCmdargs() anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa) anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) fmaskConfig.setVerbose(cmdargs.verbose) fmaskConfig.setTempDir(cmdargs.tempdir) fmaskConfig.setTOARefScaling(10000.0) fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) fmaskConfig.setCloudBufferSize( int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize( int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) fmask.doFmask(fmaskFilenames, fmaskConfig) if anglesfile != cmdargs.anglesfile: # Must have been a temporary vrt, so remove it os.remove(anglesfile)
def makeAnglesImage(self, template_img, outfile, nadirLine, extentSunAngles, satAzimuth, imgInfo): """ Make a single output image file of the sun and satellite angles for every pixel in the template image. """ imgInfo = fileinfo.ImageInfo(template_img) infiles = landsatangles.applier.FilenameAssociations() outfiles = landsatangles.applier.FilenameAssociations() otherargs = landsatangles.applier.OtherInputs() controls = landsatangles.applier.ApplierControls() infiles.img = template_img outfiles.angles = outfile (ctrLat, ctrLong) = landsatangles.getCtrLatLong(imgInfo) otherargs.R = landsatangles.localRadius(ctrLat) otherargs.nadirLine = nadirLine otherargs.xMin = imgInfo.xMin otherargs.xMax = imgInfo.xMax otherargs.yMin = imgInfo.yMin otherargs.yMax = imgInfo.yMax otherargs.extentSunAngles = extentSunAngles otherargs.satAltitude = 705000 # Landsat nominal altitude in metres otherargs.satAzimuth = satAzimuth otherargs.radianScale = 100 * 180 / np.pi # Store pixel values in degrees and scale factor of 100 controls.setStatsIgnore(500) controls.setCalcStats(False) controls.setOutputDriverName('GTiff') landsatangles.applier.apply(landsatangles.makeAngles, infiles, outfiles, otherargs, controls=controls)
def do_fmask(mtl_file, filters_enabled, tmp_dir, min_cloud_size=0, cloud_prob_thresh=0.225, cloud_buffer_size=4, shadow_buffer_size=6, cirrus_prob_ratio=0.04, nir_fill_thresh=0.02, swir2_thresh=0.03, whiteness_thresh=0.7, swir2_water_test=0.03, nir_snow_thresh=0.11, green_snow_thresh=0.1): print("Fmask:") input_dir = os.path.dirname(mtl_file) # parser mtl_file_parse = mtl2dict(mtl_file) # get the landsat version landsat_version = int(mtl_file_parse['SPACECRAFT_ID'][-1]) # set bands for reflective and thermal if landsat_version in [4, 5]: # get the reflective file names bands reflective_bands = [ os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)]) for N in [1, 2, 3, 4, 5, 7] ] # get the thermal file names bands thermal_bands = [ os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)]) for N in [6] ] # set bands for reflective and thermal if landsat_version == 7: # get the reflective file names bands reflective_bands = [ os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)]) for N in [1, 2, 3, 4, 5, 7] ] # get the thermal file names bands thermal_bands = [ os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_6_VCID_' + str(N)]) for N in [1, 2] ] # set bands for reflective and thermal if landsat_version == 8: # get the reflective file names bands reflective_bands = [ os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)]) for N in [1, 2, 3, 4, 5, 6, 7, 9] ] # get the thermal file names bands thermal_bands = [ os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)]) for N in [10, 11] ] # set the prefer file name band for process reflective_bands = [ get_prefer_name(file_path) for file_path in reflective_bands ] thermal_bands = [get_prefer_name(file_path) for file_path in thermal_bands] ######################################## # reflective bands stack # tmp file for reflective bands stack reflective_stack_file = os.path.join(tmp_dir, "reflective_stack.tif") if not os.path.isfile(reflective_stack_file): gdal_merge.main( ["", "-separate", "-of", "GTiff", "-o", reflective_stack_file] + reflective_bands) ######################################## # thermal bands stack # tmp file for reflective bands stack thermal_stack_file = os.path.join(tmp_dir, "thermal_stack.tif") if not os.path.isfile(thermal_stack_file): gdal_merge.main( ["", "-separate", "-of", "GTiff", "-o", thermal_stack_file] + thermal_bands) ######################################## # estimates of per-pixel angles for sun # and satellite azimuth and zenith # # fmask_usgsLandsatMakeAnglesImage.py # tmp file for angles angles_file = os.path.join(tmp_dir, "angles.tif") mtlInfo = config.readMTLFile(mtl_file) imgInfo = fileinfo.ImageInfo(reflective_stack_file) corners = landsatangles.findImgCorners(reflective_stack_file, imgInfo) nadirLine = landsatangles.findNadirLine(corners) extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) satAzimuth = landsatangles.satAzLeftRight(nadirLine) landsatangles.makeAnglesImage(reflective_stack_file, angles_file, nadirLine, extentSunAngles, satAzimuth, imgInfo) ######################################## # saturation mask # # fmask_usgsLandsatSaturationMask.py # tmp file for angles saturationmask_file = os.path.join(tmp_dir, "saturationmask.tif") if landsat_version == 4: sensor = config.FMASK_LANDSAT47 elif landsat_version == 5: sensor = config.FMASK_LANDSAT47 elif landsat_version == 7: sensor = config.FMASK_LANDSAT47 elif landsat_version == 8: sensor = config.FMASK_LANDSAT8 # needed so the saturation function knows which # bands are visible etc. fmaskConfig = config.FmaskConfig(sensor) saturationcheck.makeSaturationMask(fmaskConfig, reflective_stack_file, saturationmask_file) ######################################## # top of Atmosphere reflectance # # fmask_usgsLandsatTOA.py # tmp file for toa toa_file = os.path.join(tmp_dir, "toa.tif") landsatTOA.makeTOAReflectance(reflective_stack_file, mtl_file, angles_file, toa_file) ######################################## # cloud mask # # fmask_usgsLandsatStacked.py # tmp file for cloud cloud_fmask_file = os.path.join(tmp_dir, "fmask.tif") # 1040nm thermal band should always be the first (or only) band in a # stack of Landsat thermal bands thermalInfo = config.readThermalInfoFromLandsatMTL(mtl_file) anglesInfo = config.AnglesFileInfo(angles_file, 3, angles_file, 2, angles_file, 1, angles_file, 0) if landsat_version == 4: sensor = config.FMASK_LANDSAT47 elif landsat_version == 5: sensor = config.FMASK_LANDSAT47 elif landsat_version == 7: sensor = config.FMASK_LANDSAT47 elif landsat_version == 8: sensor = config.FMASK_LANDSAT8 fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(toa_file) fmaskFilenames.setThermalFile(thermal_stack_file) fmaskFilenames.setOutputCloudMaskFile(cloud_fmask_file) fmaskFilenames.setSaturationMask(saturationmask_file) # TODO: optional fmaskConfig = config.FmaskConfig(sensor) fmaskConfig.setThermalInfo(thermalInfo) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(False) fmaskConfig.setVerbose(False) fmaskConfig.setTempDir(tmp_dir) # Set the settings fmask filters from widget to FmaskConfig fmaskConfig.setMinCloudSize(min_cloud_size) fmaskConfig.setEqn17CloudProbThresh(cloud_prob_thresh) fmaskConfig.setCloudBufferSize(int(cloud_buffer_size)) fmaskConfig.setShadowBufferSize(int(shadow_buffer_size)) fmaskConfig.setCirrusProbRatio(cirrus_prob_ratio) fmaskConfig.setEqn19NIRFillThresh(nir_fill_thresh) fmaskConfig.setEqn1Swir2Thresh(swir2_thresh) fmaskConfig.setEqn2WhitenessThresh(whiteness_thresh) fmaskConfig.setEqn7Swir2Thresh(swir2_water_test) fmaskConfig.setEqn20NirSnowThresh(nir_snow_thresh) fmaskConfig.setEqn20GreenSnowThresh(green_snow_thresh) # set to 1 for all Fmask filters disabled if filters_enabled["Fmask Cloud"]: fmask.OUTCODE_CLOUD = 2 else: fmask.OUTCODE_CLOUD = 1 if filters_enabled["Fmask Shadow"]: fmask.OUTCODE_SHADOW = 3 else: fmask.OUTCODE_SHADOW = 1 if filters_enabled["Fmask Snow"]: fmask.OUTCODE_SNOW = 4 else: fmask.OUTCODE_SNOW = 1 if filters_enabled["Fmask Water"]: fmask.OUTCODE_WATER = 5 else: fmask.OUTCODE_WATER = 1 # process Fmask fmask.doFmask(fmaskFilenames, fmaskConfig) return cloud_fmask_file
def do_fmask(self, filters_enabled, min_cloud_size=0, cloud_prob_thresh=0.225, cloud_buffer_size=4, shadow_buffer_size=6, cirrus_prob_ratio=0.04, nir_fill_thresh=0.02, swir2_thresh=0.03, whiteness_thresh=0.7, swir2_water_test=0.03, nir_snow_thresh=0.11, green_snow_thresh=0.1): ######################################## # reflective bands stack # tmp file for reflective bands stack self.reflective_stack_file = os.path.join(self.tmp_dir, "reflective_stack.tif") if not os.path.isfile(self.reflective_stack_file): update_process_bar(self.process_bar, 10, self.process_status, self.tr("Making reflective bands stack...")) gdal_merge.main(["", "-separate", "-of", "GTiff", "-o", self.reflective_stack_file] + self.reflective_bands) ######################################## # thermal bands stack # tmp file for reflective bands stack self.thermal_stack_file = os.path.join(self.tmp_dir, "thermal_stack.tif") if not os.path.isfile(self.thermal_stack_file): update_process_bar(self.process_bar, 20, self.process_status, self.tr("Making thermal bands stack...")) gdal_merge.main(["", "-separate", "-of", "GTiff", "-o", self.thermal_stack_file] + self.thermal_bands) ######################################## # clipping the reflective bands stack (only if is activated selected area or shape area) self.reflective_stack_clip_file = os.path.join(self.tmp_dir, "reflective_stack_clip.tif") self.reflective_stack_for_process = self.clip(self.reflective_stack_file, self.reflective_stack_clip_file) ######################################## # clipping the thermal bands stack (only if is activated selected area or shape area) self.thermal_stack_clip_file = os.path.join(self.tmp_dir, "thermal_stack_clip.tif") self.thermal_stack_for_process = self.clip(self.thermal_stack_file, self.thermal_stack_clip_file) ######################################## # estimates of per-pixel angles for sun # and satellite azimuth and zenith # # fmask_usgsLandsatMakeAnglesImage.py # tmp file for angles self.angles_file = os.path.join(self.tmp_dir, "angles.tif") update_process_bar(self.process_bar, 30, self.process_status, self.tr("Making fmask angles file...")) mtlInfo = config.readMTLFile(self.mtl_path) imgInfo = fileinfo.ImageInfo(self.reflective_stack_for_process) corners = landsatangles.findImgCorners(self.reflective_stack_for_process, imgInfo) nadirLine = landsatangles.findNadirLine(corners) extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo) satAzimuth = landsatangles.satAzLeftRight(nadirLine) landsatangles.makeAnglesImage(self.reflective_stack_for_process, self.angles_file, nadirLine, extentSunAngles, satAzimuth, imgInfo) ######################################## # saturation mask # # fmask_usgsLandsatSaturationMask.py # tmp file for angles self.saturationmask_file = os.path.join(self.tmp_dir, "saturationmask.tif") update_process_bar(self.process_bar, 40, self.process_status, self.tr("Making saturation mask file...")) if self.landsat_version == 4: sensor = config.FMASK_LANDSAT47 elif self.landsat_version == 5: sensor = config.FMASK_LANDSAT47 elif self.landsat_version == 7: sensor = config.FMASK_LANDSAT47 elif self.landsat_version == 8: sensor = config.FMASK_LANDSAT8 # needed so the saturation function knows which # bands are visible etc. fmaskConfig = config.FmaskConfig(sensor) saturationcheck.makeSaturationMask(fmaskConfig, self.reflective_stack_for_process, self.saturationmask_file) ######################################## # top of Atmosphere reflectance # # fmask_usgsLandsatTOA.py # tmp file for toa self.toa_file = os.path.join(self.tmp_dir, "toa.tif") update_process_bar(self.process_bar, 50, self.process_status, self.tr("Making top of Atmosphere ref...")) landsatTOA.makeTOAReflectance(self.reflective_stack_for_process, self.mtl_path, self.angles_file, self.toa_file) ######################################## # cloud mask # # fmask_usgsLandsatStacked.py # tmp file for cloud self.cloud_fmask_file = os.path.join(self.tmp_dir, "cloud_fmask_{}.tif".format(datetime.now().strftime('%H%M%S'))) update_process_bar(self.process_bar, 70, self.process_status, self.tr("Making cloud mask with fmask...")) # 1040nm thermal band should always be the first (or only) band in a # stack of Landsat thermal bands thermalInfo = config.readThermalInfoFromLandsatMTL(self.mtl_path) anglesInfo = config.AnglesFileInfo(self.angles_file, 3, self.angles_file, 2, self.angles_file, 1, self.angles_file, 0) if self.landsat_version == 4: sensor = config.FMASK_LANDSAT47 elif self.landsat_version == 5: sensor = config.FMASK_LANDSAT47 elif self.landsat_version == 7: sensor = config.FMASK_LANDSAT47 elif self.landsat_version == 8: sensor = config.FMASK_LANDSAT8 fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(self.toa_file) fmaskFilenames.setThermalFile(self.thermal_stack_for_process) fmaskFilenames.setOutputCloudMaskFile(self.cloud_fmask_file) fmaskFilenames.setSaturationMask(self.saturationmask_file) # TODO: optional fmaskConfig = config.FmaskConfig(sensor) fmaskConfig.setThermalInfo(thermalInfo) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(False) fmaskConfig.setVerbose(True) fmaskConfig.setTempDir(self.tmp_dir) # Set the settings fmask filters from widget to FmaskConfig fmaskConfig.setMinCloudSize(min_cloud_size) fmaskConfig.setEqn17CloudProbThresh(cloud_prob_thresh) fmaskConfig.setCloudBufferSize(int(cloud_buffer_size)) fmaskConfig.setShadowBufferSize(int(shadow_buffer_size)) fmaskConfig.setCirrusProbRatio(cirrus_prob_ratio) fmaskConfig.setEqn19NIRFillThresh(nir_fill_thresh) fmaskConfig.setEqn1Swir2Thresh(swir2_thresh) fmaskConfig.setEqn2WhitenessThresh(whiteness_thresh) fmaskConfig.setEqn7Swir2Thresh(swir2_water_test) fmaskConfig.setEqn20NirSnowThresh(nir_snow_thresh) fmaskConfig.setEqn20GreenSnowThresh(green_snow_thresh) # set to 1 for all Fmask filters disabled if filters_enabled["Fmask Cloud"]: fmask.OUTCODE_CLOUD = 2 else: fmask.OUTCODE_CLOUD = 1 if filters_enabled["Fmask Shadow"]: fmask.OUTCODE_SHADOW = 3 else: fmask.OUTCODE_SHADOW = 1 if filters_enabled["Fmask Snow"]: fmask.OUTCODE_SNOW = 4 else: fmask.OUTCODE_SNOW = 1 if filters_enabled["Fmask Water"]: fmask.OUTCODE_WATER = 5 else: fmask.OUTCODE_WATER = 1 # process Fmask fmask.doFmask(fmaskFilenames, fmaskConfig) # save final result of masking self.cloud_masking_files.append(self.cloud_fmask_file) ### ending fmask process update_process_bar(self.process_bar, 100, self.process_status, self.tr("DONE"))
def get_ST_masks(json_fp, bands=None, roi_img=None, gdal_format='KEA', num_processes=1, threshold=3): """Main function to run to generate the output masks. Given an input JSON file, generates a mask for each date, for each band where 0=Inlier, 1=High outlier, -1=Low outlier. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. A minimum of 12 observations is required to create the masks. json_fp: Path to JSON file which provides a dictionary where for each date, an input file name and an output file name are provided. gdal_format: Short driver name for GDAL, e.g. KEA, GTiff. num_processes: Number of concurrent processes to use. bands: List of GDAL band numbers to use, e.g. [1, 3, 5]. Defaults to all. threshold: Threshold for screening. Defaults to 3, meaning that observations outside 3*RMSE of the fitted model will be counted as outliers. Lower values will result in more outliers being detected. """ ip_paths = [] op_paths = [] dates = [] try: # Open and read JSON file containing date:filepath pairs with open(json_fp) as json_file: image_list = json.load(json_file) for date in image_list.items(): dates.append([datetime.strptime(date[0], '%Y-%m-%d').toordinal()]) ip_paths.append(date[1]['input']) op_paths.append(date[1]['output']) except FileNotFoundError: print('Could not find the provided JSON file.') sys.exit() except json.decoder.JSONDecodeError as e: print('There is an error in the provided JSON file: {}'.format(e)) sys.exit() # Create object to hold input files infiles = applier.FilenameAssociations() infiles.images = ip_paths # Create object to hold output file outfiles = applier.FilenameAssociations() outfiles.outimage = op_paths # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set window size to 1 because we are working per-pixel app.setWindowXsize(1) app.setWindowYsize(1) # Set progress try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() app.progress = progress_bar # Set output file type app.setOutputDriverName(gdal_format) if roi_img is not None: app.setReferenceImage(roi_img) app.setFootprintType(applier.BOUNDS_FROM_REFERENCE) app.setResampleMethod('near') # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Open first image in list to use as a template template_image = fileinfo.ImageInfo(infiles.images[0]) # Get no data value nodata = template_image.nodataval[0] if not bands: # No bands specified - default to all num_bands = template_image.rasterCount bands = [i for i in range(1, num_bands + 1)] else: # If a list of bands is provided # Number of bands determines things like the size of the output array num_bands = len(bands) # Need to tell the applier to only use the specified bands app.selectInputImageLayers(bands) full_names = [template_image.layerNameFromNumber(i) for i in bands] # Set up output layer name app.setLayerNames(full_names) # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.dates = dates other_args.threshold = threshold other_args.nodata = nodata other_args.num_bands = num_bands template_image = None try: applier.apply(_gen_band_masks, infiles, outfiles, otherArgs=other_args, controls=app) except RuntimeError as e: print('There was an error processing the images: {}'.format(e)) print('Do all images in the JSON file exist?')
se = [180, -90] if shapefile: # Get a Layer's Extent; shapefile has to use lat/long inShapefile = shapefile inDriver = ogr.GetDriverByName("ESRI Shapefile") inDataSource = inDriver.Open(inShapefile, 0) inLayer = inDataSource.GetLayer() ext = inLayer.GetExtent() nw = [ext[0], ext[3]] se = [ext[1], ext[2]] if debug: print(nw) if debug: print(se) # get nodata value of input dataset; has to be set explicitely for gdalwarp info = fileinfo.ImageInfo(infile) dstnodata = info.nodataval print(dstnodata[0]) if dstnodata[0] is None: print("nodata value for input dataset not defined. Using 0") dstnodata = [255] # create 'standard' DGGS based on WGS84 and center meridian at 52 deg and corresponding WKT string n_square = 1 s_square = 3 a = 6378137 central_meridian = 52 E = Ellipsoid(lon_0=central_meridian) rddgs = RHEALPixDGGS(ellipsoid=E, north_square=n_square, south_square=s_square,
def main(): # create directory for cloudmasks if _input.endswith(".SAFE/"): paths = _input.split(os.sep) cloudmaskdir = os.path.join(os.sep, *paths[0:-2], "cloudmasks") os.makedirs(cloudmaskdir, exist_ok=True) elif _input.endswith(".SAFE"): paths = _input.split(os.sep) cloudmaskdir = os.path.join(os.sep, *paths[0:-1], "cloudmasks") os.makedirs(cloudmaskdir, exist_ok=True) else: cloudmaskdir = os.path.join(_input, "cloudmasks") os.makedirs(cloudmaskdir, exist_ok=True) # a dictionary containing all the default arguments from the argument parser args = { 'safedir': None, 'granuledir': None, 'toa': None, 'anglesfile': None, 'output': None, 'verbose': True, 'pixsize': 20, 'keepintermediates': False, 'tempdir': '.', 'mincloudsize': 0, 'cloudbufferdistance': 150.0, 'shadowbufferdistance': 300.0, 'cloudprobthreshold': 100 * config.FmaskConfig.Eqn17CloudProbThresh, 'nirsnowthreshold': config.FmaskConfig.Eqn20NirSnowThresh, 'greensnowthreshold': config.FmaskConfig.Eqn20GreenSnowThresh, 'parallaxtest': False } # a helper class so that the arguments can be called with args.argument instead of args['argument'] class Cmdargs(object): def __init__(self, args): for key, value in args.items(): setattr(self, key, value) for i in safe_dir: cmdargs = Cmdargs(args) if not i.endswith(os.sep): safe = i.split(os.sep)[-1] else: safe = i.split(os.sep)[-2] names = safe.split("_") out_name = "cloudmask_" + names[0] + "_" + names[2] + "_" + names[ 5] + "_.img" print(os.path.join(cloudmaskdir, out_name)) cmdargs.output = os.path.join(cloudmaskdir, out_name) cmdargs.safedir = i tempStack = False if cmdargs.safedir is not None or cmdargs.granuledir is not None: tempStack = True resampledBands = makeStackAndAngles(cmdargs) anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa) anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) fmaskConfig.setVerbose(cmdargs.verbose) fmaskConfig.setTempDir(cmdargs.tempdir) fmaskConfig.setTOARefScaling(10000.0) fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) fmaskConfig.setEqn17CloudProbThresh( cmdargs.cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest) # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) fmaskConfig.setCloudBufferSize( int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize( int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) fmask.doFmask(fmaskFilenames, fmaskConfig) if anglesfile != cmdargs.anglesfile: # Must have been a temporary, so remove it os.remove(anglesfile) if tempStack and not cmdargs.keepintermediates: for fn in [cmdargs.toa, cmdargs.anglesfile]: if os.path.exists(fn): os.remove(fn)
def get_ST_model_coeffs(json_fp, output_fp, gdalformat='KEA', bands=None, num_processes=1, model_type='Lasso', alpha=20, cv=False): """ Main function to run to generate the output image. Given an input JSON file and an output file path, generates a multi-band output image where each pixel contains the model details for that pixel. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. :param json_fp: Path to JSON file of date/filepath pairs. :param output_fp: Path for output file. :param gdalformat: Short driver name for GDAL, e.g. KEA, GTiff. :param bands: List of GDAL band numbers to use in the analysis, e.g. [2, 5, 7]. :param num_processes: Number of concurrent processes to use. :param model_type: Either 'Lasso' or 'OLS'. The type of model fitting to use. OLS will be faster, but more likely to overfit. Both types will adjust the number of model coefficients depending on the number of observations. :param alpha: If using Lasso fitting, the alpha value controls the degree of penalization of the coefficients. The lower the value, the closer the model will fit the data. For surface reflectance, a value of around 20 (the default) is usually OK. :param cv: If using Lasso fitting, you can use cross validation to choose the value of alpha by setting cv=True. However, this is not recommended and will substantially increase run time. """ paths = [] dates = [] try: # Open and read JSON file containing date:filepath pairs with open(json_fp) as json_file: image_list = json.load(json_file) for date, img_path in image_list.items(): dates.append(datetime.strptime(date, '%Y-%m-%d').toordinal()) paths.append(img_path) except FileNotFoundError: print('Could not find the provided JSON file.') sys.exit() except json.decoder.JSONDecodeError as e: print('There is an error in the provided JSON file: {}'.format(e)) sys.exit() # Create object to hold input files infiles = applier.FilenameAssociations() infiles.images = paths # Create object to hold output file outfiles = applier.FilenameAssociations() outfiles.outimage = output_fp # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set window size to 1 because we are working per-pixel app.setWindowXsize(1) app.setWindowYsize(1) # Set output file type app.setOutputDriverName(gdalformat) # Set progress try: import tqdm progress_bar = rsgislib.TQDMProgressBar() except: progress_bar = cuiprogress.GDALProgressBar() app.progress = progress_bar # Set that pyramids and statistics are not calculated. app.omitPyramids = True app.calcStats = False # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Open first image in list to use as a template template_image = fileinfo.ImageInfo(infiles.images[0]) # Get no data value nodata_val = template_image.nodataval[0] if not bands: # No bands specified - default to all num_bands = template_image.rasterCount bands = [i for i in range(1, num_bands + 1)] else: # If a list of bands is provided # Number of bands determines things like the size of the output array num_bands = len(bands) # Need to tell the applier to only use the specified bands app.selectInputImageLayers(bands) # Create list of actual names full_names = [template_image.layerNameFromNumber(i) for i in bands] template_image = None # Set up output layer names based on band numbers layer_names = gen_layer_names(full_names) app.setLayerNames(layer_names) # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.dates = dates other_args.num_bands = num_bands other_args.nodata_val = nodata_val other_args.model_type = model_type other_args.alpha = alpha other_args.cv = cv try: applier.apply(gen_per_band_models, infiles, outfiles, otherArgs=other_args, controls=app) except RuntimeError as e: print('There was an error processing the images: {}'.format(e)) print('Do all images in the JSON file exist?')
def run_tmask(json_fp, gdal_format='KEA', num_processes=1, green_band=2, nir_band=4, swir_band=5, threshold=40): """ Main function to run to generate the output masks. Given an input JSON file, generates a mask for each date where 1=cloud/cloud shadow/snow and 0=clear. Opening/closing of files, generation of blocks and use of multiprocessing is all handled by RIOS. A minimum of 12 observations is required to create the masks. :param json_fp: Path to JSON file which provides a dictionary where for each date, an input file name and an output file name are provided. :param gdal_format: The file format of the output image (e.g., KEA, GTIFF). (Default: KEA) :param num_processes: Number of concurrent processes to use. (Default: 1) :param green_band: GDAL band number for green spectral band. Defaults to 2. :param nir_band: GDAL band number for NIR spectral band. Defaults to 4. :param swir_band: GDAL band number for SWIR spectral band. Defaults to 5. :param threshold: Numerical threshold for screening out cloud, cloud shadow, and snow. Defaults to 40. See Zhu & Woodcock (2014) for details. """ ip_paths = [] op_paths = [] dates = [] try: # Open and read JSON file containing date:filepath pairs with open(json_fp) as json_file: image_list = json.load(json_file) for date in image_list.items(): dates.append( [datetime.strptime(date[0], '%Y-%m-%d').toordinal()]) ip_paths.append(date[1]['input']) op_paths.append(date[1]['output']) except FileNotFoundError: print('Could not find the provided JSON file.') sys.exit() except json.decoder.JSONDecodeError as e: print('There is an error in the provided JSON file: {}'.format(e)) sys.exit() # Create object to hold input files infiles = applier.FilenameAssociations() infiles.images = ip_paths # Create object to hold output file outfiles = applier.FilenameAssociations() outfiles.outimage = op_paths # ApplierControls object holds details on how processing should be done app = applier.ApplierControls() # Set window size to 1 because we are working per-pixel app.setWindowXsize(1) app.setWindowYsize(1) # Set output file type app.setOutputDriverName(gdal_format) # Use Python's multiprocessing module app.setJobManagerType('multiprocessing') app.setNumThreads(num_processes) # Open first image in list to use as a template template_image = fileinfo.ImageInfo(infiles.images[0]) # Get no data value nodata = template_image.nodataval[0] # Need to tell the applier to only use the specified bands app.selectInputImageLayers([green_band, nir_band, swir_band]) # Set up output layer name app.setLayerNames(['tmask']) # Additional arguments - have to be passed as a single object other_args = applier.OtherInputs() other_args.dates = dates other_args.threshold = threshold other_args.nodata = nodata try: applier.apply(_gen_tmask, infiles, outfiles, otherArgs=other_args, controls=app) except RuntimeError as e: print('There was an error processing the images: {}'.format(e)) print('Do all images in the JSON file exist?')
def doMinimum(info, inputs, outputs, otherargs): "Called from RIOS. Average the input files" minimum = numpy.zeros(inputs.imgs[0].shape, dtype=numpy.float32) for img in inputs.imgs: img[numpy.isnan(img)] = otherargs.noDataVal imgNonNull = (img != otherargs.noDataVal) minNull = (minimum == otherargs.noDataVal) minimum[minNull] = img[minNull] newMin = (imgNonNull & ~minNull & (img < minimum)) minimum[newMin] = img[newMin] outputs.min = minimum.astype(img.dtype) infiles = applier.FilenameAssociations() # names of imput images infiles.imgs = sys.argv[1:] otherargs = applier.OtherInputs() otherargs.noDataVal = float(fileinfo.ImageInfo(infiles.imgs[0]).nodataval[0]) print(otherargs.noDataVal) # Last name given is the output outfiles = applier.FilenameAssociations() outfiles.min = "outfile18.img" controls = applier.ApplierControls() controls.setFootprintType(applier.UNION) applier.apply(doMinimum, infiles, outfiles, otherargs, controls=controls)
def mainRoutine(): """ Main routine that calls fmask """ cmdargs = getCmdargs() tempStack = False if cmdargs.scenedir is not None: tempStack = True makeStacksAndAngles(cmdargs) if (cmdargs.thermal is None or cmdargs.anglesfile is None or cmdargs.mtl is None is None or cmdargs.output is None or cmdargs.toa is None): raise SystemExit('Not all required input parameters supplied') # 1040nm thermal band should always be the first (or only) band in a # stack of Landsat thermal bands thermalInfo = config.readThermalInfoFromLandsatMTL(cmdargs.mtl) anglesfile = cmdargs.anglesfile anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0) mtlInfo = config.readMTLFile(cmdargs.mtl) landsat = mtlInfo['SPACECRAFT_ID'][-1] if landsat == '4': sensor = config.FMASK_LANDSAT47 elif landsat == '5': sensor = config.FMASK_LANDSAT47 elif landsat == '7': sensor = config.FMASK_LANDSAT47 elif landsat == '8': sensor = config.FMASK_LANDSAT8 else: raise SystemExit('Unsupported Landsat sensor') fmaskFilenames = config.FmaskFilenames() fmaskFilenames.setTOAReflectanceFile(cmdargs.toa) fmaskFilenames.setThermalFile(cmdargs.thermal) fmaskFilenames.setOutputCloudMaskFile(cmdargs.output) if cmdargs.saturation is not None: fmaskFilenames.setSaturationMask(cmdargs.saturation) else: print('saturation mask not supplied - see fmask_usgsLandsatSaturationMask.py') fmaskConfig = config.FmaskConfig(sensor) fmaskConfig.setThermalInfo(thermalInfo) fmaskConfig.setAnglesInfo(anglesInfo) fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates) fmaskConfig.setVerbose(cmdargs.verbose) fmaskConfig.setTempDir(cmdargs.tempdir) fmaskConfig.setMinCloudSize(cmdargs.mincloudsize) fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold) fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold) # Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image toaImgInfo = fileinfo.ImageInfo(cmdargs.toa) fmaskConfig.setCloudBufferSize(int(cmdargs.cloudbufferdistance / toaImgInfo.xRes)) fmaskConfig.setShadowBufferSize(int(cmdargs.shadowbufferdistance / toaImgInfo.xRes)) fmask.doFmask(fmaskFilenames, fmaskConfig) if tempStack and not cmdargs.keepintermediates: for fn in [cmdargs.refstack, cmdargs.thermal, cmdargs.anglesfile, cmdargs.saturation, cmdargs.toa]: if os.path.exists(fn): os.remove(fn)