def uzh_prepare(reference, outdir, source):
"""
create an UZH incident angle subset resampled to a reference image.
Parameters
----------
reference: str
the reference file with the target extent
outdir: str
the directory to write the new file to;
new files are named uzh_{epsg}_{index}.tif, e.g. uzh_4326_1.tif.
source: str
the original product to be subsetted
Returns
-------
numpy.ndarray
the content of the file written to `outdir`
"""
with Raster(reference) as ras:
xRes, yRes = ras.res
epsg = ras.epsg
ext = ras.extent
warp_opts = {'options': ['-q'], 'format': 'GTiff', 'multithread': True,
'dstNodata': -99, 'resampleAlg': 'bilinear'}
if not os.path.isdir(outdir):
os.makedirs(outdir)
# find existing files
uzh_subs = finder(outdir, ['uzh_[0-9]{4,5}_[0-9].tif'], regex=True)
# check if any of the existing files matches the extent of the reference
match = False
if len(uzh_subs) > 0:
for j, sub in enumerate(uzh_subs):
with Raster(sub) as ras:
if ras.extent == ext:
uzh_sub = sub
match = True
if not match:
with Raster(source) as ras:
if ras.epsg != epsg:
raise RuntimeError('CRS mismatch')
basename = 'uzh_{}_{}.tif'.format(epsg, len(uzh_subs))
uzh_sub = os.path.join(outdir, basename)
print('creating', uzh_sub)
warp_opts['dstSRS'] = 'EPSG:{}'.format(epsg)
warp_opts['xRes'] = xRes
warp_opts['yRes'] = yRes
warp_opts['outputBounds'] = (ext['xmin'], ext['ymin'],
ext['xmax'], ext['ymax'])
gdalwarp(src=source, dst=uzh_sub, options=warp_opts)
return uzh_sub
def clc_prepare(reference, outdir, source):
"""
create a CLC subset resampled to a reference image.
Parameters
----------
reference: str
the reference file with the target CRS and extent
outdir: str
the directory to write the new file to;
new files are named clc{index}.tif, e.g. clc1.tif.
source: str
the original product to be subsetted
Returns
-------
str
the name of the file written to `outdir`
"""
with Raster(reference) as ras:
xRes, yRes = ras.res
epsg = ras.epsg
ext = ras.extent
#########################################################################
warp_opts = {
'options': ['-q'],
'format': 'GTiff',
'multithread': True,
'dstNodata': -99,
'resampleAlg': 'mode'
}
if not os.path.isdir(outdir):
os.makedirs(outdir)
clc_subs = finder(outdir, ['clc[0-9].tif'], regex=True)
match = False
if len(clc_subs) > 0:
for j, sub in enumerate(clc_subs):
with Raster(sub) as ras:
if ras.extent == ext:
clc_sub = sub
match = True
if not match:
clc_sub = os.path.join(outdir, 'clc{}.tif'.format(len(clc_subs)))
print('creating', clc_sub)
warp_opts['dstSRS'] = 'EPSG:{}'.format(epsg)
warp_opts['xRes'] = xRes
warp_opts['yRes'] = yRes
warp_opts['outputBounds'] = (ext['xmin'], ext['ymin'], ext['xmax'],
ext['ymax'])
gdalwarp(src=source, dst=clc_sub, options=warp_opts)
return clc_sub
def clc_prep(clc, reference, outname):
"""
resample and crop the corine product to the resolution and extent of a reference image.
Parameters
----------
clc: str
the name of the CLC input file
reference: str
the name of the reference file
outname: str
the named of the output image
Returns
-------
"""
with Raster(reference).bbox() as box_ras:
with Raster(clc).bbox() as box_clc:
if intersect(box_ras, box_clc) is None:
print('no intersect')
return
if not os.path.isfile(outname):
with Raster(reference) as ras:
ref_crs = ras.projection
xres, yres = ras.res
with ras.bbox() as box:
ref_ext = box.extent
outputBounds = (ref_ext['xmin'], ref_ext['ymin'], ref_ext['xmax'],
ref_ext['ymax'])
gdalwarp_opt = {
'format': 'GTiff',
'outputBounds': outputBounds,
'multithread': True,
'xRes': xres,
'yRes': yres,
'dstSRS': ref_crs,
'resampleAlg': 'mode'
}
gdalwarp(src=clc, dst=outname, options=gdalwarp_opt)
else:
print('outfile already exists')
def mosaic(demlist, outname, byteorder=1, gammapar=True):
"""
mosaicing of multiple DEMs
Parameters
----------
demlist: list
a list of DEM names to be mosaiced
outname: str
the name of the final mosaic file
byteorder: {0, 1}
the byte order of the mosaic
- 0: small endian
- 1: big endian
gammapar: bool
create a Gamma parameter file for the mosaic?
Returns
-------
"""
if len(demlist) < 2:
raise IOError('length of demlist < 2')
with raster.Raster(demlist[0]) as ras:
nodata = ras.nodata
par = {
'format': 'ENVI',
'srcNodata': nodata,
' dstNodata': nodata,
'options': ['-q']
}
gdalwarp(demlist, outname, par)
if byteorder == 1:
swap(outname, outname + '_swap')
for item in [outname, outname + '.hdr', outname + '.aux.xml']:
os.remove(item)
os.rename(outname + '_swap', outname)
os.rename(outname + '_swap.hdr', outname + '.hdr')
if gammapar:
dempar(outname)
def test_stack(tmpdir, testdata):
name = testdata['tif']
outname = os.path.join(str(tmpdir), 'test')
tr = (30, 30)
# no input files provided
with pytest.raises(RuntimeError):
stack(srcfiles=[],
resampling='near',
targetres=tr,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# two files, but only one layer name
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
srcnodata=-99,
dstnodata=-99,
dstfile=outname,
layernames=['a'])
# targetres must be a two-entry tuple/list
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name],
resampling='near',
targetres=30,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# only one file specified
with pytest.raises(RuntimeError):
stack(srcfiles=[name],
resampling='near',
targetres=tr,
overwrite=True,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# targetres must contain two values
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name],
resampling='near',
targetres=(30, 30, 30),
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# unknown resampling method
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name],
resampling='foobar',
targetres=tr,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# non-existing files
with pytest.raises(RuntimeError):
stack(srcfiles=['foo', 'bar'],
resampling='near',
targetres=tr,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# create a multi-band stack
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
overwrite=True,
srcnodata=-99,
dstnodata=-99,
dstfile=outname,
layernames=['test1', 'test2'])
with Raster(outname) as ras:
assert ras.bands == 2
# Raster.rescale currently only supports one band
with pytest.raises(ValueError):
ras.rescale(lambda x: x * 10)
# outname exists and overwrite is False
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
overwrite=False,
srcnodata=-99,
dstnodata=-99,
dstfile=outname,
layernames=['test1', 'test2'])
# pass shapefile
outname = os.path.join(str(tmpdir), 'test2')
with Raster(name).bbox() as box:
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
overwrite=True,
srcnodata=-99,
dstnodata=-99,
dstfile=outname,
shapefile=box,
layernames=['test1', 'test2'])
with Raster(outname) as ras:
assert ras.bands == 2
# pass shapefile and do mosaicing
outname = os.path.join(str(tmpdir), 'test3')
with Raster(name).bbox() as box:
stack(srcfiles=[[name, name]],
resampling='near',
targetres=tr,
overwrite=True,
srcnodata=-99,
dstnodata=-99,
dstfile=outname,
shapefile=box)
with Raster(outname + '.tif') as ras:
assert ras.bands == 1
assert ras.format == 'GTiff'
# projection mismatch
name2 = os.path.join(str(tmpdir), os.path.basename(name))
outname = os.path.join(str(tmpdir), 'test4')
gdalwarp(name, name2, options={'dstSRS': crsConvert(4326, 'wkt')})
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name2],
resampling='near',
targetres=tr,
overwrite=True,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# no projection found
outname = os.path.join(str(tmpdir), 'test5')
gdal_translate(name, name2, {'options': ['-co', 'PROFILE=BASELINE']})
with Raster(name2) as ras:
print(ras.projection)
with pytest.raises(RuntimeError):
stack(srcfiles=[name2, name2],
resampling='near',
targetres=tr,
overwrite=True,
srcnodata=-99,
dstnodata=-99,
dstfile=outname)
# create separate GeoTiffs
outdir = os.path.join(str(tmpdir), 'subdir')
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
overwrite=True,
layernames=['test1', 'test2'],
srcnodata=-99,
dstnodata=-99,
dstfile=outdir,
separate=True,
compress=True)
# repeat with overwrite disabled (no error raised, just a print message)
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
overwrite=False,
layernames=['test1', 'test2'],
srcnodata=-99,
dstnodata=-99,
dstfile=outdir,
separate=True,
compress=True)
# repeat without layernames but sortfun
# bandnames not unique
outdir = os.path.join(str(tmpdir), 'subdir2')
with pytest.raises(RuntimeError):
stack(srcfiles=[name, name],
resampling='near',
targetres=tr,
overwrite=True,
sortfun=os.path.basename,
srcnodata=-99,
dstnodata=-99,
dstfile=outdir,
separate=True,
compress=True)
# repeat without layernames but sortfun
name2 = os.path.join(str(tmpdir),
os.path.basename(name).replace('VV', 'XX'))
shutil.copyfile(name, name2)
outdir = os.path.join(str(tmpdir), 'subdir2')
stack(srcfiles=[name, name2],
resampling='near',
targetres=tr,
overwrite=True,
sortfun=os.path.basename,
srcnodata=-99,
dstnodata=-99,
dstfile=outdir,
separate=True,
compress=True)
# shapefile filtering
outdir = os.path.join(str(tmpdir), 'subdir3')
files = [testdata['tif'], testdata['tif2'], testdata['tif3']]
with Raster(files[0]).bbox() as box:
stack(srcfiles=files,
resampling='near',
targetres=(30, 30),
overwrite=False,
layernames=['test1', 'test2', 'test3'],
srcnodata=-99,
dstnodata=-99,
dstfile=outdir,
separate=True,
compress=True,
shapefile=box)
# repeated run with different scene selection and only one scene after spatial filtering
stack(srcfiles=files[1:],
resampling='near',
targetres=(30, 30),
overwrite=True,
layernames=['test2', 'test3'],
srcnodata=-99,
dstnodata=-99,
dstfile=outdir,
separate=True,
compress=True,
shapefile=box)
def dem_autocreate(geometry,
demType,
outfile,
buffer=0.01,
logpath=None,
username=None,
password=None):
"""
| automatically create a DEM in Gamma format for a defined spatial geometry
| the following steps will be performed:
- collect all tiles overlapping with the geometry
* if they don't yet exist locally they will automatically be downloaded
* the tiles will be downloaded into the SNAP auxdata directory structure, e.g. $HOME/.snap/auxdata/dem/SRTM 3Sec
- create a mosaic GeoTiff of the same spatial extent as the input geometry plus a defined buffer using gdalwarp
- subtract the EGM96-WGS84 Geoid-Ellipsoid difference and convert the result to Gamma format using Gamma command srtm2dem
* this correction is not done for TanDEM-X data, which contains ellipsoid heights; see `here <https://geoservice.dlr.de/web/dataguide/tdm90>`_
Parameters
----------
geometry: spatialist.vector.Vector
a vector geometry delimiting the output DEM size; CRS must be WGS84 LatLon (EPSG 4326)
demType: str
the type of DEM to be used; see :func:`~pyroSAR.auxdata.dem_autoload` for options
outfile: str
the name of the final DEM file
buffer: float
a buffer in degrees to create around the geometry
logpath: str
a directory to write Gamma logfiles to
username: str or None
(optional) the user name for services requiring registration; see :func:`~pyroSAR.auxdata.dem_autoload`
password: str or None
(optional) the password for the registration account
Returns
-------
"""
if os.path.isfile(outfile):
print('outfile already exists')
return
tmpdir = outfile + '__tmp'
os.makedirs(tmpdir)
try:
if logpath is not None and not os.path.isdir(logpath):
os.makedirs(logpath)
vrt = os.path.join(tmpdir, 'dem.vrt')
dem = os.path.join(tmpdir, 'dem.tif')
print('collecting DEM tiles')
vrt = dem_autoload([geometry],
demType,
vrt=vrt,
username=username,
password=password,
buffer=buffer)
print('creating mosaic')
gdalwarp(vrt, dem, {'format': 'GTiff'})
outfile_tmp = os.path.join(tmpdir, os.path.basename(outfile))
# The heights of the TanDEM-X DEM products are ellipsoidal heights, all others are EGM96 Geoid heights
# Gamma works only with Ellipsoid heights and the offset needs to be corrected
if demType == 'TDX90m':
gflg = 0
message = 'conversion to Gamma format'
else:
gflg = 2
message = 'geoid correction and conversion to Gamma format'
print(message)
diff.srtm2dem(SRTM_DEM=dem,
DEM=outfile_tmp,
DEM_par=outfile_tmp + '.par',
gflg=gflg,
geoid='-',
logpath=logpath,
outdir=tmpdir)
par2hdr(outfile_tmp + '.par', outfile_tmp + '.hdr')
for suffix in ['', '.par', '.hdr']:
shutil.copyfile(outfile_tmp + suffix, outfile + suffix)
except RuntimeError as e:
raise e
finally:
shutil.rmtree(tmpdir)
def inc_stack(small, gamma, snap, outdir, prefix=''):
outnames_base = ['small', 'gamma', 'snap']
outnames = [
os.path.join(outdir, prefix + x) + '.tif' for x in outnames_base
]
if all([os.path.isfile(x) for x in outnames]):
return outnames
# set SMALL product nodata GeoTiff value
with Raster(small)[0:100, 0:100] as ras:
if ras.nodata is None:
print('setting nodata value for SMALL product')
mat = ras.matrix()
nodata = float(mat[0, 0])
ras2 = gdal.Open(small, GA_Update)
ras2.GetRasterBand(1).SetNoDataValue(nodata)
ras2 = None
tmpdir = os.path.join(outdir, 'tmp')
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
small_edit = os.path.join(
tmpdir,
os.path.basename(small).replace('.tif', '_edit.tif'))
if not os.path.isfile(small_edit):
print('reducing resolution of SMALL product')
gdal_translate(small,
small_edit,
options={
'xRes': 90,
'yRes': 90,
'resampleAlg': 'average',
'format': 'GTiff'
})
# subtract 90 degrees from SMALL product
small_out = outnames[0]
if not os.path.isfile(small_out):
print('subtracting 90 degrees from SMALL product')
with Raster(small_edit) as ras:
mat = ras.matrix() - 90
ras.assign(mat, 0)
print('creating {}'.format(small_out))
ras.write(small_out, format='GTiff', nodata=-99)
# set GAMMA product nodata value
with Raster(gamma) as ras:
if ras.nodata != 0:
print('setting nodata value of GAMMA product')
ras2 = gdal.Open(gamma, GA_Update)
ras2.GetRasterBand(1).SetNoDataValue(0)
ras2 = None
# convert GAMMA product from radians to degrees
gamma_deg = os.path.join(
tmpdir,
os.path.basename(gamma).replace('.tif', '_deg.tif'))
if not os.path.isfile(gamma_deg):
print('converting GAMMA product from radians to degrees')
with Raster(gamma) as ras:
mat = np.rad2deg(ras.matrix())
ras.assign(mat, 0)
ras.write(gamma_deg, format='GTiff')
gamma = gamma_deg
# use extent of SMALL product as reference
ext = Raster(small_out).bbox().extent
# create new directory for the stacked files
if not os.path.isdir(outdir):
os.makedirs(outdir)
# warp the products to their common extent
warp_opts = {
'options': ['-q'],
'format': 'GTiff',
'multithread': True,
'outputBounds': (ext['xmin'], ext['ymin'], ext['xmax'], ext['ymax']),
'dstNodata': -99,
'xRes': 90,
'yRes': 90,
'resampleAlg': 'bilinear',
'dstSRS': 'EPSG:32632'
}
for i, item in enumerate([gamma, snap]):
outfile = outnames[i + 1]
if not os.path.isfile(outfile):
print('creating {}'.format(outfile))
gdalwarp(src=item, dst=outfile, options=warp_opts)
shutil.rmtree(tmpdir)
return outnames
