def checkAnglesFile(inputAnglesFile, toafile):
"""
Check that the resolution of the input angles file matches that of the input
TOA reflectance file. If not, make a VRT file which will resample it
on-the-fly. Only checks the resolution, assumes that if these match, then everything
else will match too.
Return the name of the angles file to use.
"""
toaImgInfo = fileinfo.ImageInfo(toafile)
anglesImgInfo = fileinfo.ImageInfo(inputAnglesFile)
outputAnglesFile = inputAnglesFile
if (toaImgInfo.xRes != anglesImgInfo.xRes) or (toaImgInfo.yRes !=
anglesImgInfo.yRes):
(fd, vrtName) = tempfile.mkstemp(prefix='angles', suffix='.vrt')
os.close(fd)
subprocess.check_call([
GDALWARPCMD, '-q', '-of', 'VRT', '-tr',
str(toaImgInfo.xRes),
str(toaImgInfo.yRes), '-te',
str(toaImgInfo.xMin),
str(toaImgInfo.yMin),
str(toaImgInfo.xMax),
str(toaImgInfo.yMax), '-r', 'near', inputAnglesFile, vrtName
])
outputAnglesFile = vrtName
return outputAnglesFile
def checkAnglesFile(inputAnglesFile, toafile):
"""
Check that the resolution of the input angles file matches that of the input
TOA reflectance file. If not, make a VRT file which will resample it
on-the-fly. Only checks the resolution, assumes that if these match, then everything
else will match too.
Return the name of the angles file to use.
"""
toaImgInfo = fileinfo.ImageInfo(toafile)
anglesImgInfo = fileinfo.ImageInfo(inputAnglesFile)
outputAnglesFile = inputAnglesFile
if (toaImgInfo.xRes != anglesImgInfo.xRes) or (toaImgInfo.yRes !=
anglesImgInfo.yRes):
(fd, vrtName) = tempfile.mkstemp(prefix="angles", suffix=".vrt")
os.close(fd)
cmdFmt = (
"gdalwarp -q -of VRT -tr {xres} {yres} -te {xmin} {ymin} {xmax} {ymax} "
+ "-r near {infile} {outfile} ")
cmd = cmdFmt.format(
xres=toaImgInfo.xRes,
yres=toaImgInfo.yRes,
xmin=toaImgInfo.xMin,
ymin=toaImgInfo.yMin,
xmax=toaImgInfo.xMax,
ymax=toaImgInfo.yMax,
outfile=vrtName,
infile=inputAnglesFile,
)
os.system(cmd)
outputAnglesFile = vrtName
return outputAnglesFile
def makeAnglesImage(templateimg, outfile, nadirLine, extentSunAngles,
satAzimuth, imgInfo):
"""
Make a single output image file of the sun and satellite angles for every
pixel in the template image.
"""
imgInfo = fileinfo.ImageInfo(templateimg)
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
infiles.img = templateimg
outfiles.angles = outfile
(ctrLat, ctrLong) = getCtrLatLong(imgInfo)
otherargs.R = localRadius(ctrLat)
otherargs.nadirLine = nadirLine
otherargs.xMin = imgInfo.xMin
otherargs.xMax = imgInfo.xMax
otherargs.yMin = imgInfo.yMin
otherargs.yMax = imgInfo.yMax
otherargs.extentSunAngles = extentSunAngles
otherargs.satAltitude = 705000 # Landsat nominal altitude in metres
otherargs.satAzimuth = satAzimuth
otherargs.radianScale = 100 # Store pixel values as (radians * radianScale)
controls.setStatsIgnore(500)
applier.apply(makeAngles, infiles, outfiles, otherargs, controls=controls)
def LandsatFmaskRoutine(MTLfile,toafile='toa.img',themalfile='thermal.img',
anglesfile='angles.img',outfile='cloud.img',
keepintermediates=False,verbose=True,
tempdir='.',mincloudsize=0,
cloudprobthreshold=100 * fmask.config.FmaskConfig.Eqn17CloudProbThresh,
nirsnowthreshold=fmask.config.FmaskConfig.Eqn20NirSnowThresh,
greensnowthreshold=fmask.config.FmaskConfig.Eqn20GreenSnowThresh,
cloudbufferdistance=300,shadowbufferdistance=300):
thermalInfo = fmask.config.readThermalInfoFromLandsatMTL(MTLfile)
anglesInfo = fmask.config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0)
mtlInfo = fmask.config.readMTLFile(MTLfile)
sensor=GetLandsatSensor(MTLfile)
fmaskFilenames = fmask.config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(toafile)
fmaskFilenames.setThermalFile(themalfile)
fmaskFilenames.setOutputCloudMaskFile(outfile)
fmaskConfig = fmask.config.FmaskConfig(sensor)
saturationcheck.makeSaturationMask(fmaskConfig,'ref.img','saturationmask.img')
fmaskFilenames.setSaturationMask('saturationmask.img')
fmaskConfig.setThermalInfo(thermalInfo)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(keepintermediates)
fmaskConfig.setVerbose(verbose)
fmaskConfig.setTempDir(tempdir)
fmaskConfig.setMinCloudSize(mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(cloudprobthreshold / 100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(greensnowthreshold)
toaImgInfo = fileinfo.ImageInfo(toafile)
fmaskConfig.setCloudBufferSize(int(cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(int(shadowbufferdistance / toaImgInfo.xRes))
fmask.fmask.doFmask(fmaskFilenames, fmaskConfig)
def makeTOAReflectance(infile, mtlFile, anglesfile, outfile):
"""
Main routine - does the calculation
The eqn for TOA reflectance, p, is
p = pi * L * d^2 / E * cos(theta)
d = earthSunDistance(date)
L = image pixel (radiance)
E = exoatmospheric irradiance for the band, and
theta = solar zenith angle.
Assumes infile is radiance values in DN from USGS.
mtlFile is the .mtl file.
outfile will be created in the default format that RIOS
is configured to use and will be top of atmosphere
reflectance values *10000. Also assumes that the
angles image file is scaled as radians*100, and has layers for
satAzimuth, satZenith, sunAzimuth, sunZenith, in that order.
"""
mtlInfo = config.readMTLFile(mtlFile)
spaceCraft = mtlInfo['SPACECRAFT_ID']
date = mtlInfo['DATE_ACQUIRED']
date = date.replace('-', '')
inputs = applier.FilenameAssociations()
inputs.infile = infile
inputs.angles = anglesfile
outputs = applier.FilenameAssociations()
outputs.outfile = outfile
otherinputs = applier.OtherInputs()
otherinputs.earthSunDistance = earthSunDistance(date)
otherinputs.earthSunDistanceSq = otherinputs.earthSunDistance * otherinputs.earthSunDistance
otherinputs.esun = ESUN_LOOKUP[spaceCraft]
gains, offsets = readGainsOffsets(mtlInfo)
otherinputs.gains = gains
otherinputs.offsets = offsets
otherinputs.anglesToRadians = 0.01
otherinputs.outNull = 32767
imginfo = fileinfo.ImageInfo(infile)
otherinputs.inNull = imginfo.nodataval[0]
controls = applier.ApplierControls()
if platform.system() in ["Linux", "Darwin"]:
controls.setNumThreads(multiprocessing.cpu_count())
controls.setJobManagerType("multiprocessing")
controls.progress = cuiprogress.GDALProgressBar()
controls.setStatsIgnore(otherinputs.outNull)
controls.setCalcStats(False)
applier.apply(riosTOA, inputs, outputs, otherinputs, controls=controls)
# Explicitly set the null value in the output
ds = gdal.Open(outfile, gdal.GA_Update)
for i in range(ds.RasterCount):
ds.GetRasterBand(i + 1).SetNoDataValue(otherinputs.outNull)
def main():
metadataFile = sys.argv[1]
imgFile = sys.argv[2]
toaFile = sys.argv[3]
d = float(sys.argv[4])
outfile = "%s_envi.exp" % (metadataFile.split('.')[0])
if os.path.exists(toaFile):
os.remove(toaFile)
(calFactors, solarZenithAngle, eSun, layerNames) = getParams(metadataFile)
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
otherargs = applier.OtherInputs()
controls = applier.ApplierControls()
infiles.raw = imgFile
outfiles.toaFile = toaFile
otherargs.calFactors = calFactors
otherargs.solarZenithAngle = solarZenithAngle
otherargs.pi = 3.14159265358979323846
otherargs.d = d
otherargs.eSun = eSun
info = fileinfo.ImageInfo(imgFile)
xMin = np.floor(info.xMin)
xMax = np.ceil(info.xMax)
yMin = np.floor(info.yMin)
yMax = np.ceil(info.yMax)
proj = info.projection
transform = info.transform
xRes = info.xRes
yRes = info.yRes
otherargs.outNull = 0
print(otherargs.outNull)
controls.setStatsIgnore(otherargs.outNull)
controls.setOutputDriverName('GTiff')
controls.setCreationOptions(['COMPRESS=LZW'])
controls.setCreationOptions(['BIGTIFF=IF_SAFER'])
pixgrid = pixelgrid.PixelGridDefn(geotransform=transform,
xMin=xMin,
xMax=xMax,
yMin=yMin,
yMax=yMax,
xRes=xRes,
yRes=yRes,
projection=proj)
print(pixgrid)
controls.setReferencePixgrid(pixgrid)
controls.setLayerNames(layerNames)
controls.setWindowXsize(20)
controls.setWindowYsize(20)
progress = cuiprogress.GDALProgressBar()
controls.setProgress(progress)
applier.apply(doTOA, infiles, outfiles, otherargs, controls=controls)
def get_angle_images(self, DST=None):
"""
:param DST: Optional name of the output tif containing all angles images
:return: set self.angles_file
Following band order : SAT_AZ , SAT_ZENITH, SUN_AZ, SUN_ZENITH ')
The unit is RADIANS
"""
# downsample factor
F = 10
if DST is not None:
out_file = DST
else:
out_file = os.path.join(self.product_path, 'tie_points.tif')
if self.ang_filename != 'not found' and sys.platform == 'linux2':
self.ang_filename = os.path.join(self.product_path,
self.ang_filename)
ls8_angles_exe = os.path.join(BINDIR, 'l8_angles', 'l8_angles')
args = [
ls8_angles_exe,
os.path.abspath(self.ang_filename),
'SATELLITE {} -b 1,2,3,4,5,6,7'.format(F)
]
subprocess.check_call(' '.join(args),
shell=True,
cwd=os.path.dirname(out_file))
args = [
ls8_angles_exe,
os.path.abspath(self.ang_filename), 'SOLAR {} -b 1'.format(F)
]
subprocess.check_call(' '.join(args),
shell=True,
cwd=os.path.dirname(out_file))
mtlInfo = config.readMTLFile(self.mtl_file_name)
image = self.reflective_band_list[0]
# downsample image for angle computation
dirname = os.path.dirname(out_file)
if not os.path.exists(dirname):
os.makedirs(dirname)
coarseResImage = os.path.join(dirname, 'tie_points_coarseResImage.tif')
gdal.Translate(coarseResImage, image, xRes=30 * F, yRes=30 * F)
imgInfo = fileinfo.ImageInfo(coarseResImage)
corners = landsatangles.findImgCorners(coarseResImage, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
# do not use fmask function but internal custom function
self.makeAnglesImage(coarseResImage, out_file, nadirLine,
extentSunAngles, satAzimuth, imgInfo)
log.info('SAT_AZ , SAT_ZENITH, SUN_AZ, SUN_ZENITH ')
log.info('UNIT = DEGREES (scale: x100) :')
log.info(' ' + out_file)
self.angles_file = out_file
def mainRoutine(cmdargs):
"""
Main routine that calls fmask
"""
# 1040nm thermal band should always be the first (or only) band in a
# stack of Landsat thermal bands
thermalInfo = config.readThermalInfoFromLandsatMTL(cmdargs.mtl)
anglesfile = cmdargs.anglesfile
anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2,
anglesfile, 1, anglesfile, 0)
mtlInfo = config.readMTLFile(cmdargs.mtl)
landsat = mtlInfo["SPACECRAFT_ID"][-1]
if landsat == "4":
sensor = config.FMASK_LANDSAT47
elif landsat == "5":
sensor = config.FMASK_LANDSAT47
elif landsat == "7":
sensor = config.FMASK_LANDSAT47
elif landsat == "8":
sensor = config.FMASK_LANDSAT8
else:
raise SystemExit("Unsupported Landsat sensor")
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(cmdargs.toa)
fmaskFilenames.setThermalFile(cmdargs.thermal)
fmaskFilenames.setOutputCloudMaskFile(cmdargs.output)
if cmdargs.saturation is not None:
fmaskFilenames.setSaturationMask(cmdargs.saturation)
else:
print(
"saturation mask not supplied - see fmask_usgsLandsatSaturationMask.py"
)
fmaskConfig = config.FmaskConfig(sensor)
fmaskConfig.setThermalInfo(thermalInfo)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates)
fmaskConfig.setVerbose(cmdargs.verbose)
fmaskConfig.setTempDir(cmdargs.tempdir)
fmaskConfig.setMinCloudSize(cmdargs.mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold /
100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold)
# Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image
toaImgInfo = fileinfo.ImageInfo(cmdargs.toa)
fmaskConfig.setCloudBufferSize(
int(cmdargs.cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(
int(cmdargs.shadowbufferdistance / toaImgInfo.xRes))
fmask.doFmask(fmaskFilenames, fmaskConfig)
def predict_for_date(date,
input_path,
output_path,
output_driver='KEA',
num_processes=1):
"""Main function to generate the predicted image. Given an input image containing
per-band model coefficients, outputs a multi-band predicted image over the same area.
Opening/closing of files, generation of blocks and use of multiprocessing is
all handled by RIOS.
date: The date to predict in YYYY-MM-DD format.
input_path: Path to the input image generated by get_model_coeffs.py.
output_path: Path for the output image.
output_driver: Short driver name for GDAL, e.g. KEA, GTiff.
num_processes: Number of concurrent processes to use."""
# Create object to hold input files
infile = applier.FilenameAssociations()
infile.coeff_img = input_path
# Create object to hold output file
outfile = applier.FilenameAssociations()
outfile.output_img = output_path
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set output file type
app.setOutputDriverName(output_driver)
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Convert provided date to ordinal
ordinal_date = datetime.strptime(date, '%Y-%m-%d').toordinal()
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.date_to_predict = ordinal_date
# Get band names
try:
input_img = fileinfo.ImageInfo(infile.coeff_img)
except:
sys.exit('Could not find input image.')
layer_names = np.unique([name.split('_')[0] for name in input_img.lnames])
app.setLayerNames(layer_names)
applier.apply(gen_prediction,
infile,
outfile,
otherArgs=other_args,
controls=app)
def mainRoutine(argv=None):
"""
Main routine that calls fmask
If argv is given, it should be a list of pairs of parameter and arguments like in command line.
See mainRoutine(['-h']) for details on available parameters.
Example: mainRoutine(['--safedir', '<.SAFE directory>', '-o', '<output file>'])
If argv is None or not given, command line sys.args are used, see argparse.parse_args.
"""
cmdargs = getCmdargs(argv)
tempStack = False
if cmdargs.safedir is not None or cmdargs.granuledir is not None:
tempStack = True
resampledBands = makeStackAndAngles(cmdargs)
anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa)
anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2,
anglesfile, 1, anglesfile, 0)
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(cmdargs.toa)
fmaskFilenames.setOutputCloudMaskFile(cmdargs.output)
fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates)
fmaskConfig.setVerbose(cmdargs.verbose)
fmaskConfig.setTempDir(cmdargs.tempdir)
fmaskConfig.setTOARefScaling(10000.0)
fmaskConfig.setMinCloudSize(cmdargs.mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold /
100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold)
fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest)
# Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image
toaImgInfo = fileinfo.ImageInfo(cmdargs.toa)
fmaskConfig.setCloudBufferSize(
int(cmdargs.cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(
int(cmdargs.shadowbufferdistance / toaImgInfo.xRes))
fmask.doFmask(fmaskFilenames, fmaskConfig)
if (anglesfile != cmdargs.anglesfile):
# Must have been a temporary, so remove it
os.remove(anglesfile)
if tempStack and not cmdargs.keepintermediates:
for fn in [cmdargs.toa, cmdargs.anglesfile]:
if os.path.exists(fn):
os.remove(fn)
def chooseResampleMethod(outpixsize, inBandImg):
"""
Choose the right resample method, given the image and the desired output pixel size
"""
imginfo = fileinfo.ImageInfo(inBandImg)
inPixsize = imginfo.xRes
if outpixsize == inPixsize:
resample = "near"
elif outpixsize > inPixsize:
resample = "average"
else:
resample = "cubic"
return resample
def makeAngles(mtlfile, templateimg, outfile):
"""
Callable main routine
"""
mtlInfo = config.readMTLFile(mtlfile)
imgInfo = fileinfo.ImageInfo(templateimg)
corners = landsatangles.findImgCorners(templateimg, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
landsatangles.makeAnglesImage(templateimg, outfile,
nadirLine, extentSunAngles, satAzimuth, imgInfo)
def mainRoutine():
"""
Main routine that calls fmask
"""
cmdargs = getCmdargs()
tempStack = False
if cmdargs.safedir is not None or cmdargs.granuledir is not None:
tempStack = True
resampledBands = makeStackAndAngles(cmdargs)
anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa)
anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2,
anglesfile, 1, anglesfile, 0)
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(cmdargs.toa)
fmaskFilenames.setOutputCloudMaskFile(cmdargs.output)
fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates)
fmaskConfig.setVerbose(cmdargs.verbose)
fmaskConfig.setTempDir(cmdargs.tempdir)
fmaskConfig.setTOARefScaling(10000.0)
fmaskConfig.setMinCloudSize(cmdargs.mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold /
100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold)
fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest)
# Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image
toaImgInfo = fileinfo.ImageInfo(cmdargs.toa)
fmaskConfig.setCloudBufferSize(
int(cmdargs.cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(
int(cmdargs.shadowbufferdistance / toaImgInfo.xRes))
fmask.doFmask(fmaskFilenames, fmaskConfig)
if (anglesfile != cmdargs.anglesfile):
# Must have been a temporary, so remove it
os.remove(anglesfile)
if tempStack and not cmdargs.keepintermediates:
for fn in [cmdargs.toa, cmdargs.anglesfile]:
if os.path.exists(fn):
os.remove(fn)
def mainRoutine(cmdargs):
"""
Main routine
"""
mtlInfo = config.readMTLFile(cmdargs.mtl)
imgInfo = fileinfo.ImageInfo(cmdargs.templateimg)
corners = landsatangles.findImgCorners(cmdargs.templateimg, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
landsatangles.makeAnglesImage(cmdargs.templateimg, cmdargs.outfile,
nadirLine, extentSunAngles, satAzimuth,
imgInfo)
def autofmask(dirname):
os.chdir(dirname)
MTLfile = glob(os.path.join(dirname, '*MTL.TXT'))
refname=os.path.join(dirname,'ref.img')
themalname=os.path.join(dirname,'thermal.img')
srcReflist=os.path.join(dirname,'L*_B[1,2,3,4,5,7].TIF')
srcReflist=glob(srcReflist)
srcThemal=os.path.join(dirname,'L*_B6.TIF')
srcThemal=glob(srcThemal)
anglesname=os.path.join(dirname,'angles.img')
toaname=os.path.join(dirname,'toa.img')
# 合并文件
refMergeArgv = ['', '-separate', '-of', 'HFA', '-co', 'COMPRESSED=YES', '-o', refname]
refMergeArgv.extend(srcReflist)
themalMergeArgv = ['', '-separate', '-of', 'HFA', '-co', 'COMPRESSED=YES', '-o',themalname]
themalMergeArgv.extend(srcThemal)
if not os.path.exists(refname):
gdal_merge.main(refMergeArgv)
else:
print('跳过组合多光谱')
if not os.path.exists(themalname):
gdal_merge.main(themalMergeArgv)
else:
print('跳过组合热红外')
# 生成角度文件
# 读取文件信息
MTLfile = MTLfile[0]
mtlInfo = fmask.config.readMTLFile(MTLfile)
if not os.path.exists(anglesname):
imgInfo = fileinfo.ImageInfo(refname)
corners = landsatangles.findImgCorners(refname, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
landsatangles.makeAnglesImage(refname, anglesname, nadirLine, extentSunAngles, satAzimuth, imgInfo)
# 生成辅助临时文件：反射率
if not os.path.exists(toaname):
fmask.landsatTOA.makeTOAReflectance(refname, MTLfile, anglesname, toaname)
print("begin this")
LandsatFmaskRoutine(MTLfile)
def mainRoutine():
"""
Main routine that calls fmask
"""
cmdargs = getCmdargs()
anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa)
anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2,
anglesfile, 1, anglesfile, 0)
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(cmdargs.toa)
fmaskFilenames.setOutputCloudMaskFile(cmdargs.output)
fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates)
fmaskConfig.setVerbose(cmdargs.verbose)
fmaskConfig.setTempDir(cmdargs.tempdir)
fmaskConfig.setTOARefScaling(10000.0)
fmaskConfig.setMinCloudSize(cmdargs.mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold /
100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold)
# Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image
toaImgInfo = fileinfo.ImageInfo(cmdargs.toa)
fmaskConfig.setCloudBufferSize(
int(cmdargs.cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(
int(cmdargs.shadowbufferdistance / toaImgInfo.xRes))
fmask.doFmask(fmaskFilenames, fmaskConfig)
if anglesfile != cmdargs.anglesfile:
# Must have been a temporary vrt, so remove it
os.remove(anglesfile)
def makeAnglesImage(self, template_img, outfile, nadirLine,
extentSunAngles, satAzimuth, imgInfo):
"""
Make a single output image file of the sun and satellite angles for every
pixel in the template image.
"""
imgInfo = fileinfo.ImageInfo(template_img)
infiles = landsatangles.applier.FilenameAssociations()
outfiles = landsatangles.applier.FilenameAssociations()
otherargs = landsatangles.applier.OtherInputs()
controls = landsatangles.applier.ApplierControls()
infiles.img = template_img
outfiles.angles = outfile
(ctrLat, ctrLong) = landsatangles.getCtrLatLong(imgInfo)
otherargs.R = landsatangles.localRadius(ctrLat)
otherargs.nadirLine = nadirLine
otherargs.xMin = imgInfo.xMin
otherargs.xMax = imgInfo.xMax
otherargs.yMin = imgInfo.yMin
otherargs.yMax = imgInfo.yMax
otherargs.extentSunAngles = extentSunAngles
otherargs.satAltitude = 705000 # Landsat nominal altitude in metres
otherargs.satAzimuth = satAzimuth
otherargs.radianScale = 100 * 180 / np.pi # Store pixel values in degrees and scale factor of 100
controls.setStatsIgnore(500)
controls.setCalcStats(False)
controls.setOutputDriverName('GTiff')
landsatangles.applier.apply(landsatangles.makeAngles,
infiles,
outfiles,
otherargs,
controls=controls)
def do_fmask(mtl_file,
filters_enabled,
tmp_dir,
min_cloud_size=0,
cloud_prob_thresh=0.225,
cloud_buffer_size=4,
shadow_buffer_size=6,
cirrus_prob_ratio=0.04,
nir_fill_thresh=0.02,
swir2_thresh=0.03,
whiteness_thresh=0.7,
swir2_water_test=0.03,
nir_snow_thresh=0.11,
green_snow_thresh=0.1):
print("Fmask:")
input_dir = os.path.dirname(mtl_file)
# parser
mtl_file_parse = mtl2dict(mtl_file)
# get the landsat version
landsat_version = int(mtl_file_parse['SPACECRAFT_ID'][-1])
# set bands for reflective and thermal
if landsat_version in [4, 5]:
# get the reflective file names bands
reflective_bands = [
os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)])
for N in [1, 2, 3, 4, 5, 7]
]
# get the thermal file names bands
thermal_bands = [
os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)])
for N in [6]
]
# set bands for reflective and thermal
if landsat_version == 7:
# get the reflective file names bands
reflective_bands = [
os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)])
for N in [1, 2, 3, 4, 5, 7]
]
# get the thermal file names bands
thermal_bands = [
os.path.join(input_dir,
mtl_file_parse['FILE_NAME_BAND_6_VCID_' + str(N)])
for N in [1, 2]
]
# set bands for reflective and thermal
if landsat_version == 8:
# get the reflective file names bands
reflective_bands = [
os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)])
for N in [1, 2, 3, 4, 5, 6, 7, 9]
]
# get the thermal file names bands
thermal_bands = [
os.path.join(input_dir, mtl_file_parse['FILE_NAME_BAND_' + str(N)])
for N in [10, 11]
]
# set the prefer file name band for process
reflective_bands = [
get_prefer_name(file_path) for file_path in reflective_bands
]
thermal_bands = [get_prefer_name(file_path) for file_path in thermal_bands]
########################################
# reflective bands stack
# tmp file for reflective bands stack
reflective_stack_file = os.path.join(tmp_dir, "reflective_stack.tif")
if not os.path.isfile(reflective_stack_file):
gdal_merge.main(
["", "-separate", "-of", "GTiff", "-o", reflective_stack_file] +
reflective_bands)
########################################
# thermal bands stack
# tmp file for reflective bands stack
thermal_stack_file = os.path.join(tmp_dir, "thermal_stack.tif")
if not os.path.isfile(thermal_stack_file):
gdal_merge.main(
["", "-separate", "-of", "GTiff", "-o", thermal_stack_file] +
thermal_bands)
########################################
# estimates of per-pixel angles for sun
# and satellite azimuth and zenith
#
# fmask_usgsLandsatMakeAnglesImage.py
# tmp file for angles
angles_file = os.path.join(tmp_dir, "angles.tif")
mtlInfo = config.readMTLFile(mtl_file)
imgInfo = fileinfo.ImageInfo(reflective_stack_file)
corners = landsatangles.findImgCorners(reflective_stack_file, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
landsatangles.makeAnglesImage(reflective_stack_file, angles_file,
nadirLine, extentSunAngles, satAzimuth,
imgInfo)
########################################
# saturation mask
#
# fmask_usgsLandsatSaturationMask.py
# tmp file for angles
saturationmask_file = os.path.join(tmp_dir, "saturationmask.tif")
if landsat_version == 4:
sensor = config.FMASK_LANDSAT47
elif landsat_version == 5:
sensor = config.FMASK_LANDSAT47
elif landsat_version == 7:
sensor = config.FMASK_LANDSAT47
elif landsat_version == 8:
sensor = config.FMASK_LANDSAT8
# needed so the saturation function knows which
# bands are visible etc.
fmaskConfig = config.FmaskConfig(sensor)
saturationcheck.makeSaturationMask(fmaskConfig, reflective_stack_file,
saturationmask_file)
########################################
# top of Atmosphere reflectance
#
# fmask_usgsLandsatTOA.py
# tmp file for toa
toa_file = os.path.join(tmp_dir, "toa.tif")
landsatTOA.makeTOAReflectance(reflective_stack_file, mtl_file, angles_file,
toa_file)
########################################
# cloud mask
#
# fmask_usgsLandsatStacked.py
# tmp file for cloud
cloud_fmask_file = os.path.join(tmp_dir, "fmask.tif")
# 1040nm thermal band should always be the first (or only) band in a
# stack of Landsat thermal bands
thermalInfo = config.readThermalInfoFromLandsatMTL(mtl_file)
anglesInfo = config.AnglesFileInfo(angles_file, 3, angles_file, 2,
angles_file, 1, angles_file, 0)
if landsat_version == 4:
sensor = config.FMASK_LANDSAT47
elif landsat_version == 5:
sensor = config.FMASK_LANDSAT47
elif landsat_version == 7:
sensor = config.FMASK_LANDSAT47
elif landsat_version == 8:
sensor = config.FMASK_LANDSAT8
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(toa_file)
fmaskFilenames.setThermalFile(thermal_stack_file)
fmaskFilenames.setOutputCloudMaskFile(cloud_fmask_file)
fmaskFilenames.setSaturationMask(saturationmask_file) # TODO: optional
fmaskConfig = config.FmaskConfig(sensor)
fmaskConfig.setThermalInfo(thermalInfo)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(False)
fmaskConfig.setVerbose(False)
fmaskConfig.setTempDir(tmp_dir)
# Set the settings fmask filters from widget to FmaskConfig
fmaskConfig.setMinCloudSize(min_cloud_size)
fmaskConfig.setEqn17CloudProbThresh(cloud_prob_thresh)
fmaskConfig.setCloudBufferSize(int(cloud_buffer_size))
fmaskConfig.setShadowBufferSize(int(shadow_buffer_size))
fmaskConfig.setCirrusProbRatio(cirrus_prob_ratio)
fmaskConfig.setEqn19NIRFillThresh(nir_fill_thresh)
fmaskConfig.setEqn1Swir2Thresh(swir2_thresh)
fmaskConfig.setEqn2WhitenessThresh(whiteness_thresh)
fmaskConfig.setEqn7Swir2Thresh(swir2_water_test)
fmaskConfig.setEqn20NirSnowThresh(nir_snow_thresh)
fmaskConfig.setEqn20GreenSnowThresh(green_snow_thresh)
# set to 1 for all Fmask filters disabled
if filters_enabled["Fmask Cloud"]:
fmask.OUTCODE_CLOUD = 2
else:
fmask.OUTCODE_CLOUD = 1
if filters_enabled["Fmask Shadow"]:
fmask.OUTCODE_SHADOW = 3
else:
fmask.OUTCODE_SHADOW = 1
if filters_enabled["Fmask Snow"]:
fmask.OUTCODE_SNOW = 4
else:
fmask.OUTCODE_SNOW = 1
if filters_enabled["Fmask Water"]:
fmask.OUTCODE_WATER = 5
else:
fmask.OUTCODE_WATER = 1
# process Fmask
fmask.doFmask(fmaskFilenames, fmaskConfig)
return cloud_fmask_file
def do_fmask(self, filters_enabled, min_cloud_size=0, cloud_prob_thresh=0.225, cloud_buffer_size=4,
shadow_buffer_size=6, cirrus_prob_ratio=0.04, nir_fill_thresh=0.02, swir2_thresh=0.03,
whiteness_thresh=0.7, swir2_water_test=0.03, nir_snow_thresh=0.11, green_snow_thresh=0.1):
########################################
# reflective bands stack
# tmp file for reflective bands stack
self.reflective_stack_file = os.path.join(self.tmp_dir, "reflective_stack.tif")
if not os.path.isfile(self.reflective_stack_file):
update_process_bar(self.process_bar, 10, self.process_status,
self.tr("Making reflective bands stack..."))
gdal_merge.main(["", "-separate", "-of", "GTiff", "-o",
self.reflective_stack_file] + self.reflective_bands)
########################################
# thermal bands stack
# tmp file for reflective bands stack
self.thermal_stack_file = os.path.join(self.tmp_dir, "thermal_stack.tif")
if not os.path.isfile(self.thermal_stack_file):
update_process_bar(self.process_bar, 20, self.process_status,
self.tr("Making thermal bands stack..."))
gdal_merge.main(["", "-separate", "-of", "GTiff", "-o",
self.thermal_stack_file] + self.thermal_bands)
########################################
# clipping the reflective bands stack (only if is activated selected area or shape area)
self.reflective_stack_clip_file = os.path.join(self.tmp_dir, "reflective_stack_clip.tif")
self.reflective_stack_for_process = self.clip(self.reflective_stack_file, self.reflective_stack_clip_file)
########################################
# clipping the thermal bands stack (only if is activated selected area or shape area)
self.thermal_stack_clip_file = os.path.join(self.tmp_dir, "thermal_stack_clip.tif")
self.thermal_stack_for_process = self.clip(self.thermal_stack_file, self.thermal_stack_clip_file)
########################################
# estimates of per-pixel angles for sun
# and satellite azimuth and zenith
#
# fmask_usgsLandsatMakeAnglesImage.py
# tmp file for angles
self.angles_file = os.path.join(self.tmp_dir, "angles.tif")
update_process_bar(self.process_bar, 30, self.process_status,
self.tr("Making fmask angles file..."))
mtlInfo = config.readMTLFile(self.mtl_path)
imgInfo = fileinfo.ImageInfo(self.reflective_stack_for_process)
corners = landsatangles.findImgCorners(self.reflective_stack_for_process, imgInfo)
nadirLine = landsatangles.findNadirLine(corners)
extentSunAngles = landsatangles.sunAnglesForExtent(imgInfo, mtlInfo)
satAzimuth = landsatangles.satAzLeftRight(nadirLine)
landsatangles.makeAnglesImage(self.reflective_stack_for_process, self.angles_file,
nadirLine, extentSunAngles, satAzimuth, imgInfo)
########################################
# saturation mask
#
# fmask_usgsLandsatSaturationMask.py
# tmp file for angles
self.saturationmask_file = os.path.join(self.tmp_dir, "saturationmask.tif")
update_process_bar(self.process_bar, 40, self.process_status,
self.tr("Making saturation mask file..."))
if self.landsat_version == 4:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 5:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 7:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 8:
sensor = config.FMASK_LANDSAT8
# needed so the saturation function knows which
# bands are visible etc.
fmaskConfig = config.FmaskConfig(sensor)
saturationcheck.makeSaturationMask(fmaskConfig, self.reflective_stack_for_process,
self.saturationmask_file)
########################################
# top of Atmosphere reflectance
#
# fmask_usgsLandsatTOA.py
# tmp file for toa
self.toa_file = os.path.join(self.tmp_dir, "toa.tif")
update_process_bar(self.process_bar, 50, self.process_status,
self.tr("Making top of Atmosphere ref..."))
landsatTOA.makeTOAReflectance(self.reflective_stack_for_process, self.mtl_path,
self.angles_file, self.toa_file)
########################################
# cloud mask
#
# fmask_usgsLandsatStacked.py
# tmp file for cloud
self.cloud_fmask_file = os.path.join(self.tmp_dir, "cloud_fmask_{}.tif".format(datetime.now().strftime('%H%M%S')))
update_process_bar(self.process_bar, 70, self.process_status,
self.tr("Making cloud mask with fmask..."))
# 1040nm thermal band should always be the first (or only) band in a
# stack of Landsat thermal bands
thermalInfo = config.readThermalInfoFromLandsatMTL(self.mtl_path)
anglesInfo = config.AnglesFileInfo(self.angles_file, 3, self.angles_file,
2, self.angles_file, 1, self.angles_file, 0)
if self.landsat_version == 4:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 5:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 7:
sensor = config.FMASK_LANDSAT47
elif self.landsat_version == 8:
sensor = config.FMASK_LANDSAT8
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(self.toa_file)
fmaskFilenames.setThermalFile(self.thermal_stack_for_process)
fmaskFilenames.setOutputCloudMaskFile(self.cloud_fmask_file)
fmaskFilenames.setSaturationMask(self.saturationmask_file) # TODO: optional
fmaskConfig = config.FmaskConfig(sensor)
fmaskConfig.setThermalInfo(thermalInfo)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(False)
fmaskConfig.setVerbose(True)
fmaskConfig.setTempDir(self.tmp_dir)
# Set the settings fmask filters from widget to FmaskConfig
fmaskConfig.setMinCloudSize(min_cloud_size)
fmaskConfig.setEqn17CloudProbThresh(cloud_prob_thresh)
fmaskConfig.setCloudBufferSize(int(cloud_buffer_size))
fmaskConfig.setShadowBufferSize(int(shadow_buffer_size))
fmaskConfig.setCirrusProbRatio(cirrus_prob_ratio)
fmaskConfig.setEqn19NIRFillThresh(nir_fill_thresh)
fmaskConfig.setEqn1Swir2Thresh(swir2_thresh)
fmaskConfig.setEqn2WhitenessThresh(whiteness_thresh)
fmaskConfig.setEqn7Swir2Thresh(swir2_water_test)
fmaskConfig.setEqn20NirSnowThresh(nir_snow_thresh)
fmaskConfig.setEqn20GreenSnowThresh(green_snow_thresh)
# set to 1 for all Fmask filters disabled
if filters_enabled["Fmask Cloud"]:
fmask.OUTCODE_CLOUD = 2
else:
fmask.OUTCODE_CLOUD = 1
if filters_enabled["Fmask Shadow"]:
fmask.OUTCODE_SHADOW = 3
else:
fmask.OUTCODE_SHADOW = 1
if filters_enabled["Fmask Snow"]:
fmask.OUTCODE_SNOW = 4
else:
fmask.OUTCODE_SNOW = 1
if filters_enabled["Fmask Water"]:
fmask.OUTCODE_WATER = 5
else:
fmask.OUTCODE_WATER = 1
# process Fmask
fmask.doFmask(fmaskFilenames, fmaskConfig)
# save final result of masking
self.cloud_masking_files.append(self.cloud_fmask_file)
### ending fmask process
update_process_bar(self.process_bar, 100, self.process_status,
self.tr("DONE"))
def get_ST_masks(json_fp, bands=None, roi_img=None, gdal_format='KEA', num_processes=1, threshold=3):
"""Main function to run to generate the output masks. Given an input JSON file,
generates a mask for each date, for each band where 0=Inlier, 1=High outlier,
-1=Low outlier. Opening/closing of files, generation of blocks and use of
multiprocessing is all handled by RIOS.
A minimum of 12 observations is required to create the masks.
json_fp: Path to JSON file which provides a dictionary where for each
date, an input file name and an output file name are provided.
gdal_format: Short driver name for GDAL, e.g. KEA, GTiff.
num_processes: Number of concurrent processes to use.
bands: List of GDAL band numbers to use, e.g. [1, 3, 5]. Defaults to all.
threshold: Threshold for screening. Defaults to 3, meaning that observations
outside 3*RMSE of the fitted model will be counted as outliers.
Lower values will result in more outliers being detected.
"""
ip_paths = []
op_paths = []
dates = []
try:
# Open and read JSON file containing date:filepath pairs
with open(json_fp) as json_file:
image_list = json.load(json_file)
for date in image_list.items():
dates.append([datetime.strptime(date[0], '%Y-%m-%d').toordinal()])
ip_paths.append(date[1]['input'])
op_paths.append(date[1]['output'])
except FileNotFoundError:
print('Could not find the provided JSON file.')
sys.exit()
except json.decoder.JSONDecodeError as e:
print('There is an error in the provided JSON file: {}'.format(e))
sys.exit()
# Create object to hold input files
infiles = applier.FilenameAssociations()
infiles.images = ip_paths
# Create object to hold output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = op_paths
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set window size to 1 because we are working per-pixel
app.setWindowXsize(1)
app.setWindowYsize(1)
# Set progress
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
app.progress = progress_bar
# Set output file type
app.setOutputDriverName(gdal_format)
if roi_img is not None:
app.setReferenceImage(roi_img)
app.setFootprintType(applier.BOUNDS_FROM_REFERENCE)
app.setResampleMethod('near')
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Open first image in list to use as a template
template_image = fileinfo.ImageInfo(infiles.images[0])
# Get no data value
nodata = template_image.nodataval[0]
if not bands: # No bands specified - default to all
num_bands = template_image.rasterCount
bands = [i for i in range(1, num_bands + 1)]
else: # If a list of bands is provided
# Number of bands determines things like the size of the output array
num_bands = len(bands)
# Need to tell the applier to only use the specified bands
app.selectInputImageLayers(bands)
full_names = [template_image.layerNameFromNumber(i) for i in bands]
# Set up output layer name
app.setLayerNames(full_names)
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.dates = dates
other_args.threshold = threshold
other_args.nodata = nodata
other_args.num_bands = num_bands
template_image = None
try:
applier.apply(_gen_band_masks, infiles, outfiles, otherArgs=other_args, controls=app)
except RuntimeError as e:
print('There was an error processing the images: {}'.format(e))
print('Do all images in the JSON file exist?')
se = [180, -90]
if shapefile:
# Get a Layer's Extent; shapefile has to use lat/long
inShapefile = shapefile
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(inShapefile, 0)
inLayer = inDataSource.GetLayer()
ext = inLayer.GetExtent()
nw = [ext[0], ext[3]]
se = [ext[1], ext[2]]
if debug: print(nw)
if debug: print(se)
# get nodata value of input dataset; has to be set explicitely for gdalwarp
info = fileinfo.ImageInfo(infile)
dstnodata = info.nodataval
print(dstnodata[0])
if dstnodata[0] is None:
print("nodata value for input dataset not defined. Using 0")
dstnodata = [255]
# create 'standard' DGGS based on WGS84 and center meridian at 52 deg and corresponding WKT string
n_square = 1
s_square = 3
a = 6378137
central_meridian = 52
E = Ellipsoid(lon_0=central_meridian)
rddgs = RHEALPixDGGS(ellipsoid=E,
north_square=n_square,
south_square=s_square,
def main():
# create directory for cloudmasks
if _input.endswith(".SAFE/"):
paths = _input.split(os.sep)
cloudmaskdir = os.path.join(os.sep, *paths[0:-2], "cloudmasks")
os.makedirs(cloudmaskdir, exist_ok=True)
elif _input.endswith(".SAFE"):
paths = _input.split(os.sep)
cloudmaskdir = os.path.join(os.sep, *paths[0:-1], "cloudmasks")
os.makedirs(cloudmaskdir, exist_ok=True)
else:
cloudmaskdir = os.path.join(_input, "cloudmasks")
os.makedirs(cloudmaskdir, exist_ok=True)
# a dictionary containing all the default arguments from the argument parser
args = {
'safedir': None,
'granuledir': None,
'toa': None,
'anglesfile': None,
'output': None,
'verbose': True,
'pixsize': 20,
'keepintermediates': False,
'tempdir': '.',
'mincloudsize': 0,
'cloudbufferdistance': 150.0,
'shadowbufferdistance': 300.0,
'cloudprobthreshold': 100 * config.FmaskConfig.Eqn17CloudProbThresh,
'nirsnowthreshold': config.FmaskConfig.Eqn20NirSnowThresh,
'greensnowthreshold': config.FmaskConfig.Eqn20GreenSnowThresh,
'parallaxtest': False
}
# a helper class so that the arguments can be called with args.argument instead of args['argument']
class Cmdargs(object):
def __init__(self, args):
for key, value in args.items():
setattr(self, key, value)
for i in safe_dir:
cmdargs = Cmdargs(args)
if not i.endswith(os.sep):
safe = i.split(os.sep)[-1]
else:
safe = i.split(os.sep)[-2]
names = safe.split("_")
out_name = "cloudmask_" + names[0] + "_" + names[2] + "_" + names[
5] + "_.img"
print(os.path.join(cloudmaskdir, out_name))
cmdargs.output = os.path.join(cloudmaskdir, out_name)
cmdargs.safedir = i
tempStack = False
if cmdargs.safedir is not None or cmdargs.granuledir is not None:
tempStack = True
resampledBands = makeStackAndAngles(cmdargs)
anglesfile = checkAnglesFile(cmdargs.anglesfile, cmdargs.toa)
anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2,
anglesfile, 1, anglesfile, 0)
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(cmdargs.toa)
fmaskFilenames.setOutputCloudMaskFile(cmdargs.output)
fmaskConfig = config.FmaskConfig(config.FMASK_SENTINEL2)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates)
fmaskConfig.setVerbose(cmdargs.verbose)
fmaskConfig.setTempDir(cmdargs.tempdir)
fmaskConfig.setTOARefScaling(10000.0)
fmaskConfig.setMinCloudSize(cmdargs.mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(
cmdargs.cloudprobthreshold /
100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold)
fmaskConfig.setSen2displacementTest(cmdargs.parallaxtest)
# Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image
toaImgInfo = fileinfo.ImageInfo(cmdargs.toa)
fmaskConfig.setCloudBufferSize(
int(cmdargs.cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(
int(cmdargs.shadowbufferdistance / toaImgInfo.xRes))
fmask.doFmask(fmaskFilenames, fmaskConfig)
if anglesfile != cmdargs.anglesfile:
# Must have been a temporary, so remove it
os.remove(anglesfile)
if tempStack and not cmdargs.keepintermediates:
for fn in [cmdargs.toa, cmdargs.anglesfile]:
if os.path.exists(fn):
os.remove(fn)
def get_ST_model_coeffs(json_fp, output_fp, gdalformat='KEA', bands=None, num_processes=1, model_type='Lasso',
alpha=20, cv=False):
"""
Main function to run to generate the output image. Given an input JSON file
and an output file path, generates a multi-band output image where each pixel
contains the model details for that pixel. Opening/closing of files, generation
of blocks and use of multiprocessing is all handled by RIOS.
:param json_fp: Path to JSON file of date/filepath pairs.
:param output_fp: Path for output file.
:param gdalformat: Short driver name for GDAL, e.g. KEA, GTiff.
:param bands: List of GDAL band numbers to use in the analysis, e.g. [2, 5, 7].
:param num_processes: Number of concurrent processes to use.
:param model_type: Either 'Lasso' or 'OLS'. The type of model fitting to use. OLS will
be faster, but more likely to overfit. Both types will adjust the number of model
coefficients depending on the number of observations.
:param alpha: If using Lasso fitting, the alpha value controls the degree of
penalization of the coefficients. The lower the value, the closer
the model will fit the data. For surface reflectance, a value of
around 20 (the default) is usually OK.
:param cv: If using Lasso fitting, you can use cross validation to choose
the value of alpha by setting cv=True. However, this is not recommended and will
substantially increase run time.
"""
paths = []
dates = []
try:
# Open and read JSON file containing date:filepath pairs
with open(json_fp) as json_file:
image_list = json.load(json_file)
for date, img_path in image_list.items():
dates.append(datetime.strptime(date, '%Y-%m-%d').toordinal())
paths.append(img_path)
except FileNotFoundError:
print('Could not find the provided JSON file.')
sys.exit()
except json.decoder.JSONDecodeError as e:
print('There is an error in the provided JSON file: {}'.format(e))
sys.exit()
# Create object to hold input files
infiles = applier.FilenameAssociations()
infiles.images = paths
# Create object to hold output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = output_fp
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set window size to 1 because we are working per-pixel
app.setWindowXsize(1)
app.setWindowYsize(1)
# Set output file type
app.setOutputDriverName(gdalformat)
# Set progress
try:
import tqdm
progress_bar = rsgislib.TQDMProgressBar()
except:
progress_bar = cuiprogress.GDALProgressBar()
app.progress = progress_bar
# Set that pyramids and statistics are not calculated.
app.omitPyramids = True
app.calcStats = False
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Open first image in list to use as a template
template_image = fileinfo.ImageInfo(infiles.images[0])
# Get no data value
nodata_val = template_image.nodataval[0]
if not bands: # No bands specified - default to all
num_bands = template_image.rasterCount
bands = [i for i in range(1, num_bands + 1)]
else: # If a list of bands is provided
# Number of bands determines things like the size of the output array
num_bands = len(bands)
# Need to tell the applier to only use the specified bands
app.selectInputImageLayers(bands)
# Create list of actual names
full_names = [template_image.layerNameFromNumber(i) for i in bands]
template_image = None
# Set up output layer names based on band numbers
layer_names = gen_layer_names(full_names)
app.setLayerNames(layer_names)
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.dates = dates
other_args.num_bands = num_bands
other_args.nodata_val = nodata_val
other_args.model_type = model_type
other_args.alpha = alpha
other_args.cv = cv
try:
applier.apply(gen_per_band_models, infiles, outfiles, otherArgs=other_args, controls=app)
except RuntimeError as e:
print('There was an error processing the images: {}'.format(e))
print('Do all images in the JSON file exist?')
def run_tmask(json_fp,
gdal_format='KEA',
num_processes=1,
green_band=2,
nir_band=4,
swir_band=5,
threshold=40):
"""
Main function to run to generate the output masks. Given an input JSON file,
generates a mask for each date where 1=cloud/cloud shadow/snow and 0=clear.
Opening/closing of files, generation of blocks and use of multiprocessing is
all handled by RIOS.
A minimum of 12 observations is required to create the masks.
:param json_fp: Path to JSON file which provides a dictionary where for each
date, an input file name and an output file name are provided.
:param gdal_format: The file format of the output image (e.g., KEA, GTIFF). (Default: KEA)
:param num_processes: Number of concurrent processes to use. (Default: 1)
:param green_band: GDAL band number for green spectral band. Defaults to 2.
:param nir_band: GDAL band number for NIR spectral band. Defaults to 4.
:param swir_band: GDAL band number for SWIR spectral band. Defaults to 5.
:param threshold: Numerical threshold for screening out cloud, cloud shadow, and snow.
Defaults to 40. See Zhu & Woodcock (2014) for details.
"""
ip_paths = []
op_paths = []
dates = []
try:
# Open and read JSON file containing date:filepath pairs
with open(json_fp) as json_file:
image_list = json.load(json_file)
for date in image_list.items():
dates.append(
[datetime.strptime(date[0], '%Y-%m-%d').toordinal()])
ip_paths.append(date[1]['input'])
op_paths.append(date[1]['output'])
except FileNotFoundError:
print('Could not find the provided JSON file.')
sys.exit()
except json.decoder.JSONDecodeError as e:
print('There is an error in the provided JSON file: {}'.format(e))
sys.exit()
# Create object to hold input files
infiles = applier.FilenameAssociations()
infiles.images = ip_paths
# Create object to hold output file
outfiles = applier.FilenameAssociations()
outfiles.outimage = op_paths
# ApplierControls object holds details on how processing should be done
app = applier.ApplierControls()
# Set window size to 1 because we are working per-pixel
app.setWindowXsize(1)
app.setWindowYsize(1)
# Set output file type
app.setOutputDriverName(gdal_format)
# Use Python's multiprocessing module
app.setJobManagerType('multiprocessing')
app.setNumThreads(num_processes)
# Open first image in list to use as a template
template_image = fileinfo.ImageInfo(infiles.images[0])
# Get no data value
nodata = template_image.nodataval[0]
# Need to tell the applier to only use the specified bands
app.selectInputImageLayers([green_band, nir_band, swir_band])
# Set up output layer name
app.setLayerNames(['tmask'])
# Additional arguments - have to be passed as a single object
other_args = applier.OtherInputs()
other_args.dates = dates
other_args.threshold = threshold
other_args.nodata = nodata
try:
applier.apply(_gen_tmask,
infiles,
outfiles,
otherArgs=other_args,
controls=app)
except RuntimeError as e:
print('There was an error processing the images: {}'.format(e))
print('Do all images in the JSON file exist?')
def doMinimum(info, inputs, outputs, otherargs):
"Called from RIOS. Average the input files"
minimum = numpy.zeros(inputs.imgs[0].shape, dtype=numpy.float32)
for img in inputs.imgs:
img[numpy.isnan(img)] = otherargs.noDataVal
imgNonNull = (img != otherargs.noDataVal)
minNull = (minimum == otherargs.noDataVal)
minimum[minNull] = img[minNull]
newMin = (imgNonNull & ~minNull & (img < minimum))
minimum[newMin] = img[newMin]
outputs.min = minimum.astype(img.dtype)
infiles = applier.FilenameAssociations()
# names of imput images
infiles.imgs = sys.argv[1:]
otherargs = applier.OtherInputs()
otherargs.noDataVal = float(fileinfo.ImageInfo(infiles.imgs[0]).nodataval[0])
print(otherargs.noDataVal)
# Last name given is the output
outfiles = applier.FilenameAssociations()
outfiles.min = "outfile18.img"
controls = applier.ApplierControls()
controls.setFootprintType(applier.UNION)
applier.apply(doMinimum, infiles, outfiles, otherargs, controls=controls)
def mainRoutine():
"""
Main routine that calls fmask
"""
cmdargs = getCmdargs()
tempStack = False
if cmdargs.scenedir is not None:
tempStack = True
makeStacksAndAngles(cmdargs)
if (cmdargs.thermal is None or cmdargs.anglesfile is None or
cmdargs.mtl is None is None or cmdargs.output is None
or cmdargs.toa is None):
raise SystemExit('Not all required input parameters supplied')
# 1040nm thermal band should always be the first (or only) band in a
# stack of Landsat thermal bands
thermalInfo = config.readThermalInfoFromLandsatMTL(cmdargs.mtl)
anglesfile = cmdargs.anglesfile
anglesInfo = config.AnglesFileInfo(anglesfile, 3, anglesfile, 2, anglesfile, 1, anglesfile, 0)
mtlInfo = config.readMTLFile(cmdargs.mtl)
landsat = mtlInfo['SPACECRAFT_ID'][-1]
if landsat == '4':
sensor = config.FMASK_LANDSAT47
elif landsat == '5':
sensor = config.FMASK_LANDSAT47
elif landsat == '7':
sensor = config.FMASK_LANDSAT47
elif landsat == '8':
sensor = config.FMASK_LANDSAT8
else:
raise SystemExit('Unsupported Landsat sensor')
fmaskFilenames = config.FmaskFilenames()
fmaskFilenames.setTOAReflectanceFile(cmdargs.toa)
fmaskFilenames.setThermalFile(cmdargs.thermal)
fmaskFilenames.setOutputCloudMaskFile(cmdargs.output)
if cmdargs.saturation is not None:
fmaskFilenames.setSaturationMask(cmdargs.saturation)
else:
print('saturation mask not supplied - see fmask_usgsLandsatSaturationMask.py')
fmaskConfig = config.FmaskConfig(sensor)
fmaskConfig.setThermalInfo(thermalInfo)
fmaskConfig.setAnglesInfo(anglesInfo)
fmaskConfig.setKeepIntermediates(cmdargs.keepintermediates)
fmaskConfig.setVerbose(cmdargs.verbose)
fmaskConfig.setTempDir(cmdargs.tempdir)
fmaskConfig.setMinCloudSize(cmdargs.mincloudsize)
fmaskConfig.setEqn17CloudProbThresh(cmdargs.cloudprobthreshold / 100) # Note conversion from percentage
fmaskConfig.setEqn20NirSnowThresh(cmdargs.nirsnowthreshold)
fmaskConfig.setEqn20GreenSnowThresh(cmdargs.greensnowthreshold)
# Work out a suitable buffer size, in pixels, dependent on the resolution of the input TOA image
toaImgInfo = fileinfo.ImageInfo(cmdargs.toa)
fmaskConfig.setCloudBufferSize(int(cmdargs.cloudbufferdistance / toaImgInfo.xRes))
fmaskConfig.setShadowBufferSize(int(cmdargs.shadowbufferdistance / toaImgInfo.xRes))
fmask.doFmask(fmaskFilenames, fmaskConfig)
if tempStack and not cmdargs.keepintermediates:
for fn in [cmdargs.refstack, cmdargs.thermal, cmdargs.anglesfile,
cmdargs.saturation, cmdargs.toa]:
if os.path.exists(fn):
os.remove(fn)
