NEWDATA_DIR = 'newdata'
"""
subdirectory that will be created where the tar file is extracted
that will contain the 'new' files which will be compared with those
in TESTDATA_DIR.
"""
VERSION_FILE = 'version.txt'
"name of the file containing the version information in the tar file"
# maybe these should be in lidarprocessor also?
ARRAY_TYPE_TRANSMITTED = 100
ARRAY_TYPE_RECEIVED = 101
# path to gdalchksum.py so we can run it on Windows
GDALCHKSUM = find_executable('gdalchksum.py')
if GDALCHKSUM is None:
raise IOError('Cannot find gdalchksum.py in $PATH')
class TestingError(Exception):
"Base class for testing Exceptions"
class TestingVersionError(TestingError):
"Was a mismatch in versions"
class TestingDataMismatch(TestingError):
"Data does not match between expected and newly calculated"
class Checksum(object):
"""
Holds the separate checksum dictionaries for points, pulses etc.
import h5py
from numba import jit
import ctypes
from rios.parallel.jobmanager import find_executable
# Need to give ourselves access to H5Sselect_hyperslab()
# within the HDF5 library so we can call it from numba
# Sadly, we also need to cope with not accessing this, in order that ReadTheDocs will still
# be able to build to documentation. Hence the elaborate try/except madness.
try:
if sys.platform == 'win32':
# Under Python 3.8 and later we must need to know the path to the DLL
# For conda installs we could use $CONDA_PREFIX, but we want this
# to work for all cases and user may have hdf5.dll somewhere weird.
# This breaks the security of the change, but keeps things working....
dllPath = find_executable('hdf5.dll')
HDF5_DLL = ctypes.CDLL(dllPath)
elif sys.platform == 'darwin':
HDF5_DLL = ctypes.CDLL('libhdf5.dylib')
else:
HDF5_DLL = ctypes.CDLL('libhdf5.so')
H5Sselect_hyperslab = HDF5_DLL.H5Sselect_hyperslab
# checked on 64 and 32 bits
H5Sselect_hyperslab.argtypes = [
ctypes.c_int32, ctypes.c_int32, ctypes.c_void_p, ctypes.c_void_p,
ctypes.c_void_p, ctypes.c_void_p
]
H5Sselect_hyperslab.restype = ctypes.c_int32
except Exception:
H5Sselect_hyperslab = None
def makeStackAndAngles(cmdargs):
"""
Make an intermediate stack of all the TOA reflectance bands. Also make an image
of the angles. Fill in the names of these in the cmdargs object.
"""
if cmdargs.granuledir is None and cmdargs.safedir is not None:
cmdargs.granuledir = findGranuleDir(cmdargs.safedir)
# Find the other commands we need, even under Windoze
gdalmergeCmd = find_executable("gdal_merge.py")
if gdalmergeCmd is None:
msg = "Unable to find gdal_merge.py command. Check installation of GDAL package. "
raise fmaskerrors.FmaskInstallationError(msg)
# Make the angles file
(fd, anglesfile) = tempfile.mkstemp(dir=cmdargs.tempdir,
prefix="angles_tmp_",
suffix=".img")
os.close(fd)
xmlfile = findGranuleXml(cmdargs.granuledir)
if cmdargs.verbose:
print("Making angles image")
sentinel2makeAnglesImage.makeAngles(xmlfile, anglesfile)
cmdargs.anglesfile = anglesfile
# Make a stack of the reflectance bands. Not that we do an explicit resample to the
# output pixel size, to avoid picking up the overview layers with the ESA jpg files.
# According to @vincentschut, these are shifted slightly, and should be avoided.
bandList = [
'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A', 'B09',
'B10', 'B11', 'B12'
]
imgDir = "{}/IMG_DATA".format(cmdargs.granuledir)
resampledBands = []
for band in bandList:
(fd, tmpBand) = tempfile.mkstemp(dir=cmdargs.tempdir,
prefix="tmp_{}_".format(band),
suffix=".vrt")
os.close(fd)
inBandImgList = glob.glob("{}/*_{}.jp2".format(imgDir, band))
if len(inBandImgList) != 1:
raise fmaskerrors.FmaskFileError(
"Cannot find input band {}".format(band))
inBandImg = inBandImgList[0]
# Now make a resampled copy to the desired pixel size, using the right resample method
resampleMethod = chooseResampleMethod(cmdargs.pixsize, inBandImg)
subprocess.check_call([
GDALWARPCMD, '-q', '-tr',
str(cmdargs.pixsize),
str(cmdargs.pixsize), '-co', 'TILED=YES', '-of', 'VRT', '-r',
resampleMethod, inBandImg, tmpBand
])
resampledBands.append(tmpBand)
# Now make a stack of these
if cmdargs.verbose:
print("Making stack of all bands, at {}m pixel size".format(
cmdargs.pixsize))
(fd, tmpStack) = tempfile.mkstemp(dir=cmdargs.tempdir,
prefix="tmp_allbands_",
suffix=".img")
os.close(fd)
cmdargs.toa = tmpStack
# ensure we are launching python rather than relying on the OS to do the right thing (ie Windows)
subprocess.check_call(
[sys.executable, gdalmergeCmd, '-q', '-of', DEFAULTDRIVERNAME] +
CMDLINECREATIONOPTIONS + ['-separate', '-o', cmdargs.toa] +
resampledBands)
for fn in resampledBands:
os.remove(fn)
return resampledBands
from fmask import fmaskerrors
from fmask.cmdline import sentinel2makeAnglesImage
from fmask import fmask
# for GDAL command line utilities
CMDLINECREATIONOPTIONS = []
if DEFAULTDRIVERNAME in dfltDriverOptions:
for opt in dfltDriverOptions[DEFAULTDRIVERNAME]:
CMDLINECREATIONOPTIONS.append('-co')
CMDLINECREATIONOPTIONS.append(opt)
if sys.platform.startswith('win'):
GDALWARPCMDNAME = "gdalwarp.exe"
else:
GDALWARPCMDNAME = "gdalwarp"
GDALWARPCMD = find_executable(GDALWARPCMDNAME)
if GDALWARPCMD is None:
msg = "Unable to find {} command. Check installation of GDAL package".format(
GDALWARPCMDNAME)
raise fmaskerrors.FmaskInstallationError(msg)
def getCmdargs():
"""
Get command line arguments
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"--safedir",
help=
("Name of .SAFE directory, as unzipped from " +
from rios.parallel.jobmanager import find_executable
from fmask import config
from fmask import fmaskerrors
from fmask.cmdline import sentinel2makeAnglesImage
from fmask import fmask
# for GDAL command line utilities
CMDLINECREATIONOPTIONS = []
if DEFAULTDRIVERNAME in dfltDriverOptions:
for opt in dfltDriverOptions[DEFAULTDRIVERNAME]:
CMDLINECREATIONOPTIONS.append('-co')
CMDLINECREATIONOPTIONS.append(opt)
if sys.platform.startswith('win'):
GDALWARPCMD = find_executable("gdalwarp.exe")
else:
GDALWARPCMD = find_executable("gdalwarp")
def getCmdargs():
"""
Get command line arguments
"""
parser = argparse.ArgumentParser()
parser.add_argument("--safedir", help=("Name of .SAFE directory, as unzipped from " +
"a standard ESA L1C zip file. Using this option will automatically create intermediate " +
"stacks of the input bands, and so does NOT require --toa or --anglesfile. "))
parser.add_argument("--granuledir", help=("Name of granule sub-directory within the " +
".SAFE directory, as unzipped from a standard ESA L1C zip file. This option is an " +
"alternative to --safedir, for use with ESA's old format zipfiles which had multiple " +
"granules in each zipfile. Specify the subdirectory of the single tile, under the " +
from __future__ import print_function, division
import os
import sys
import glob
from rios.parallel.jobmanager import find_executable
if len(sys.argv) < 2:
raise SystemExit("missing arguments to expand")
outList = [] # will have expanded wildcards
exeName = sys.argv[1]
if exeName.endswith('.py'):
# look up the path of the script and execute with python
outList.append(sys.executable)
scriptPath = find_executable(exeName)
if scriptPath is None:
raise SystemExit("Can't find %s" % exeName)
outList.append(scriptPath)
else:
# normal exe - just append it
outList.append(exeName)
# go through the remaining args and expand them if we can
for arg in sys.argv[2:]:
expanded = glob.glob(arg)
if len(expanded) == 0:
# not a file, just add the original string
outList.append(arg)
else:
def makeStacksAndAngles(cmdargs):
"""
Find the name of the MTL file.
Make an intermediate stacks of all the TOA reflectance and thermal bands.
Also make an image of the angles, saturation and TOA reflectance.
Fill in the names of these in the cmdargs object.
"""
# find MTL file
wldpath = os.path.join(cmdargs.scenedir, '*_MTL.txt')
mtlList = glob.glob(wldpath)
if len(mtlList) != 1:
raise fmaskerrors.FmaskFileError("Cannot find a *_MTL.txt file in specified dir")
cmdargs.mtl = mtlList[0]
gdalmergeCmd = find_executable("gdal_merge.py")
if gdalmergeCmd is None:
msg = "Unable to find gdal_merge.py command. Check installation of GDAL package. "
raise fmaskerrors.FmaskInstallationError(msg)
# we need to find the 'SPACECRAFT_ID' to work out the wildcards to use
mtlInfo = config.readMTLFile(cmdargs.mtl)
landsat = mtlInfo['SPACECRAFT_ID'][-1]
if landsat == '4' or landsat == '5':
refWildcard = 'L*_B[1,2,3,4,5,7].TIF'
thermalWildcard = 'L*_B6.TIF'
elif landsat == '7':
refWildcard = 'L*_B[1,2,3,4,5,7].TIF'
thermalWildcard = 'L*_B6_VCID_?.TIF'
elif landsat == '8':
refWildcard = 'LC*_B[1-7,9].TIF'
thermalWildcard = 'LC*_B1[0,1].TIF'
else:
raise SystemExit('Unsupported Landsat sensor')
wldpath = os.path.join(cmdargs.scenedir, refWildcard)
refFiles = sorted(glob.glob(wldpath))
if len(refFiles) == 0:
raise fmaskerrors.FmaskFileError("Cannot find expected reflectance files for sensor")
wldpath = os.path.join(cmdargs.scenedir, thermalWildcard)
thermalFiles = sorted(glob.glob(wldpath))
if len(thermalFiles) == 0:
raise fmaskerrors.FmaskFileError("Cannot find expected thermal files for sensor")
if cmdargs.verbose:
print("Making stack of all reflectance bands")
(fd, tmpRefStack) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="tmp_allrefbands_",
suffix=".img")
os.close(fd)
# use sys.executable so Windows works
subprocess.check_call([sys.executable, gdalmergeCmd, '-q', '-of', DEFAULTDRIVERNAME] +
CMDLINECREATIONOPTIONS + ['-separate', '-o', tmpRefStack] + refFiles)
# stash so we can delete later
cmdargs.refstack = tmpRefStack
if cmdargs.verbose:
print("Making stack of all thermal bands")
(fd, tmpThermStack) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="tmp_allthermalbands_",
suffix=".img")
os.close(fd)
subprocess.check_call([sys.executable, gdalmergeCmd, '-q', '-of', DEFAULTDRIVERNAME] +
CMDLINECREATIONOPTIONS + ['-separate', '-o', tmpThermStack] + thermalFiles)
cmdargs.thermal = tmpThermStack
# now the angles
if cmdargs.verbose:
print("Creating angles file")
(fd, anglesfile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="angles_tmp_",
suffix=".img")
os.close(fd)
usgsLandsatMakeAnglesImage.makeAngles(cmdargs.mtl, tmpRefStack, anglesfile)
cmdargs.anglesfile = anglesfile
# saturation
if cmdargs.verbose:
print("Creating saturation file")
(fd, saturationfile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="saturation_tmp_",
suffix=".img")
os.close(fd)
usgsLandsatSaturationMask.makeSaturationMask(cmdargs.mtl, tmpRefStack, saturationfile)
cmdargs.saturation = saturationfile
# TOA
if cmdargs.verbose:
print("Creating TOA file")
(fs, toafile) = tempfile.mkstemp(dir=cmdargs.tempdir, prefix="toa_tmp_",
suffix=".img")
os.close(fd)
landsatTOA.makeTOAReflectance(tmpRefStack, cmdargs.mtl, anglesfile, toafile)
cmdargs.toa = toafile