def main():
# define input directory containing file sto be stacked
dir_in = '/...'
# define output file name
dstfile = '/.../x'
# shapefile (for stack boundaries)
shp = '/../x.shp'
# store results in separate files or one single stack file? If separate then dstfile is used as a directory.
sep = True
# list files to be resampled; those not overlapping with the shapefile geometry will excluded by function stack
srcfiles = finder(dir_in, ['S1*_VV_*norm_db.tif'])
# check whether dstfile is already a file
if os.path.isfile(dstfile):
raise IOError('dstfile already exists')
# create groups of similar time stamps for mosaicking.
# All images with a time stamp of less than 30s difference will be grouped
groups = groupbyTime(srcfiles, seconds, 30)
# final function call
# groups will be mosaicked first
# the resulting images will all have the same extent
stack(srcfiles=groups, dstfile=dstfile, resampling='bilinear',
targetres=[20, 20], srcnodata=-99, dstnodata=-99,
shapefile=shp, sortfun=seconds, separate=sep, overwrite=False)
def __identify_snap(self):
"""
do a comprehensive search for an ESA SNAP installation
Returns
-------
bool
has the SNAP properties file been changed?
"""
# create a list of possible SNAP executables
defaults = ['snap64.exe', 'snap32.exe', 'snap.exe', 'snap']
paths = os.environ['PATH'].split(os.path.pathsep)
options = [
os.path.join(path, option) for path in paths for option in defaults
]
options = [x for x in options if os.path.isfile(x)]
if not hasattr(self, 'path') or not os.path.isfile(self.path):
executables = options
else:
executables = [self.path] + options
# for each possible SNAP executable, check whether additional files and directories exist relative to it
# to confirm whether it actually is a ESA SNAP installation or something else like e.g. the Ubuntu App Manager
for path in executables:
if os.path.islink(path):
path = os.path.realpath(path)
# check whether a directory etc exists relative to the SNAP executable
etc = os.path.join(os.path.dirname(os.path.dirname(path)), 'etc')
if not os.path.isdir(etc):
continue
# check the content of the etc directory
auxdata = os.listdir(etc)
if 'snap.auxdata.properties' not in auxdata:
continue
else:
auxdata_properties = os.path.join(etc,
'snap.auxdata.properties')
# identify the gpt executable
gpt_candidates = finder(os.path.dirname(path), ['gpt', 'gpt.exe'])
if len(gpt_candidates) == 0:
continue
else:
gpt = gpt_candidates[0]
self.path = path
self.etc = etc
self.gpt = gpt
self.auxdata = auxdata
self.properties = auxdata_properties
return
warnings.warn(
'SNAP could not be identified. If you have installed it please add the path to the SNAP '
'executables (bin subdirectory) to the PATH environment. '
'E.g. in the Linux .bashrc file add the following line:\nexport PATH=$PATH:path/to/snap/bin"'
)
def _reorganize(self):
"""
compress and move EOF files into subdirectories
Returns
-------
"""
message = True
for subdir in [self.outdir_poe, self.outdir_res]:
if not os.path.isdir(subdir):
continue
files = finder(subdir, [self.pattern], recursive=False, regex=True)
for eof in files:
base = os.path.basename(eof)
target = os.path.join(self._subdir(eof), base + '.zip')
os.makedirs(os.path.dirname(target), exist_ok=True)
if not os.path.isfile(target):
if message:
print('compressing and reorganizing EOF files')
message = False
with zf.ZipFile(file=target,
mode='w',
compression=zf.ZIP_DEFLATED) as zip:
zip.write(filename=eof, arcname=base)
os.remove(eof)
def makeSRTM(scenes, srtmdir, outname):
"""
Create a DEM in Gamma format from SRTM tiles
- coordinates are read to determine the required DEM extent and select the necessary hgt tiles
- mosaics SRTM DEM tiles, converts them to Gamma format and subtracts offset to WGS84 ellipsoid
intended for SRTM products downloaded from:
- USGS: https://gdex.cr.usgs.gov/gdex/
- CGIAR: http://srtm.csi.cgiar.org
Parameters
----------
scenes: list of str or pyroSAR.ID
a list of Gamma parameter files or pyroSAR ID objects to read the DEM extent from
srtmdir: str
a directory containing the SRTM hgt tiles
outname: str
the name of the final DEM file
Returns
-------
"""
tempdir = outname + '___temp'
os.makedirs(tempdir)
hgt_options = hgt(scenes)
hgt_files = finder(srtmdir, hgt_options)
nodatas = list(set([raster.Raster(x).nodata for x in hgt_files]))
if len(nodatas) == 1:
nodata = nodatas[0]
else:
raise RuntimeError('different nodata values are not permitted')
srtm_vrt = os.path.join(tempdir, 'srtm.vrt')
srtm_temp = srtm_vrt.replace('.vrt', '_tmp')
srtm_final = srtm_vrt.replace('.vrt', '')
gdalbuildvrt(hgt_files, srtm_vrt, {'srcNodata': nodata, 'options': ['-overwrite']})
gdal_translate(srtm_vrt, srtm_temp, {'format': 'ENVI', 'noData': nodata})
diff.srtm2dem(SRTM_DEM=srtm_temp,
DEM=srtm_final,
DEM_par=srtm_final + '.par',
gflg=2,
geoid='-',
outdir=tempdir)
shutil.move(srtm_final, outname)
shutil.move(srtm_final + '.par', outname + '.par')
par2hdr(outname + '.par', outname + '.hdr')
shutil.rmtree(tempdir)
def test_geocode(tmpdir, testdata):
scene = testdata['s1']
geocode(scene, str(tmpdir), test=True)
xmlfile = finder(str(tmpdir), ['*.xml'])[0]
tree = Workflow(xmlfile)
nodes = tree.nodes()
assert is_consistent(nodes) is True
groups = groupbyWorkers(xmlfile, 2)
split(xmlfile, groups)
id = identify(scene)
basename = '{}_{}'.format(id.outname_base(), tree.suffix)
procdir = os.path.join(str(tmpdir), basename)
assert os.path.isdir(procdir)
tempdir = os.path.join(procdir, 'temp')
assert os.path.isdir(tempdir)
parts = finder(tempdir, ['*.xml'])
assert len(parts) == 4
def main():
# some arbitrary directory for the source code
workdir = os.path.join(os.path.expanduser('~'), '.pyrosar', 'snap_code')
os.makedirs(workdir, exist_ok=True)
# the name of the Java properties file containing the operator-suffix lookup
outfile = 'snap.suffices.properties'
# clone all relevant toolboxes
for tbx in ['snap-engine', 'snap-desktop', 's1tbx']:
print(tbx)
target = os.path.join(workdir, tbx)
if not os.path.isdir(target):
url = 'https://github.com/senbox-org/{}'.format(tbx)
sp.check_call(['git', 'clone', url], cwd=workdir)
else:
sp.check_call(['git', 'pull'], cwd=target)
# search patterns for relevant files
# Usually files containing operator classes are named <operator>Op.java but with out dashes
# e.g. TerrainFlatteningOp.java for the Terrain-Flattening operator
# One exception is Calibration for which there is a sub-class for each SAR sensor
operators = finder(workdir, ['*Op.java', 'BaseCalibrator.java'])
# a list for collection the suffices
collect = []
for op in operators:
with open(op) as infile:
content = infile.read()
# the suffix is defined as a class attribute PRODUCT_SUFFIX
pattern = 'String PRODUCT_SUFFIX = \"_([a-zA-Z]*)\"'
match = re.search(pattern, content)
if match:
suffix = match.groups()[0]
else:
suffix = ''
# the name of the operator as available in the UI
pattern = 'alias = \"([a-zA-Z-]*)\"'
match = re.search(pattern, content)
if match:
alias = match.groups()[0]
else:
alias = None
# only collect operators for which an alias exists, i.e. which are exposed in the UI,
# and for which a suffix is defined. In the UI, all operators for which no suffix exists
# will just get no suffix in any written file.
if alias is not None and suffix != '':
print(alias, suffix)
collect.append('{0}={1}'.format(alias, suffix))
print('found {} matching operators'.format(len(collect)))
with open(outfile, 'w') as out:
out.write('\n'.join(sorted(collect, key=str.lower)))
def parse_module(bindir, outfile):
"""
parse all Gamma commands of a module to functions and save them to a Python script.
Parameters
----------
bindir: str
the `bin` directory of a module containing the commands
outfile: str
the name of the Python file to write
Returns
-------
Examples
--------
>>> import os
>>> from pyroSAR.gamma.parser import parse_module
>>> outname = os.path.join(os.environ['HOME'], 'isp.py')
>>> parse_module('/cluster/GAMMA_SOFTWARE-20161207/ISP/bin', outname)
"""
if not os.path.isdir(bindir):
raise OSError('directory does not exist: {}'.format(bindir))
excludes = ['coord_trans', # doesn't take any parameters and is interactive
'RSAT2_SLC_preproc', # takes option flags
'mk_ASF_CEOS_list', # "cannot create : Directory nonexistent"
'2PASS_UNW', # parameter name inconsistencies
'mk_diff_2d', # takes option flags
'gamma_doc' # opens the Gamma documentation
]
failed = []
outstring = ''
for cmd in sorted(finder(bindir, [r'^\w+$'], regex=True), key=lambda s: s.lower()):
basename = os.path.basename(cmd)
if basename not in excludes:
# print(basename)
try:
fun = parse_command(cmd)
except RuntimeError as e:
failed.append('{0}: {1}'.format(basename, str(e)))
continue
except DeprecationWarning:
continue
except:
failed.append('{0}: {1}'.format(basename, 'error yet to be assessed'))
continue
outstring += fun + '\n\n'
if len(outstring) > 0:
if not os.path.isfile(outfile):
with open(outfile, 'w') as out:
out.write('from pyroSAR.gamma.auxil import process\n\n\n')
with open(outfile, 'a') as out:
out.write(outstring)
if len(failed) > 0:
print('the following functions could not be parsed:\n{0}\n({1} total)'.format('\n'.join(failed), len(failed)))
def uzh_prepare(reference, outdir, source):
"""
create an UZH incident angle subset resampled to a reference image.
Parameters
----------
reference: str
the reference file with the target extent
outdir: str
the directory to write the new file to;
new files are named uzh_{epsg}_{index}.tif, e.g. uzh_4326_1.tif.
source: str
the original product to be subsetted
Returns
-------
numpy.ndarray
the content of the file written to `outdir`
"""
with Raster(reference) as ras:
xRes, yRes = ras.res
epsg = ras.epsg
ext = ras.extent
warp_opts = {'options': ['-q'], 'format': 'GTiff', 'multithread': True,
'dstNodata': -99, 'resampleAlg': 'bilinear'}
if not os.path.isdir(outdir):
os.makedirs(outdir)
# find existing files
uzh_subs = finder(outdir, ['uzh_[0-9]{4,5}_[0-9].tif'], regex=True)
# check if any of the existing files matches the extent of the reference
match = False
if len(uzh_subs) > 0:
for j, sub in enumerate(uzh_subs):
with Raster(sub) as ras:
if ras.extent == ext:
uzh_sub = sub
match = True
if not match:
with Raster(source) as ras:
if ras.epsg != epsg:
raise RuntimeError('CRS mismatch')
basename = 'uzh_{}_{}.tif'.format(epsg, len(uzh_subs))
uzh_sub = os.path.join(outdir, basename)
print('creating', uzh_sub)
warp_opts['dstSRS'] = 'EPSG:{}'.format(epsg)
warp_opts['xRes'] = xRes
warp_opts['yRes'] = yRes
warp_opts['outputBounds'] = (ext['xmin'], ext['ymin'],
ext['xmax'], ext['ymax'])
gdalwarp(src=source, dst=uzh_sub, options=warp_opts)
return uzh_sub
def clc_prepare(reference, outdir, source):
"""
create a CLC subset resampled to a reference image.
Parameters
----------
reference: str
the reference file with the target CRS and extent
outdir: str
the directory to write the new file to;
new files are named clc{index}.tif, e.g. clc1.tif.
source: str
the original product to be subsetted
Returns
-------
str
the name of the file written to `outdir`
"""
with Raster(reference) as ras:
xRes, yRes = ras.res
epsg = ras.epsg
ext = ras.extent
#########################################################################
warp_opts = {
'options': ['-q'],
'format': 'GTiff',
'multithread': True,
'dstNodata': -99,
'resampleAlg': 'mode'
}
if not os.path.isdir(outdir):
os.makedirs(outdir)
clc_subs = finder(outdir, ['clc[0-9].tif'], regex=True)
match = False
if len(clc_subs) > 0:
for j, sub in enumerate(clc_subs):
with Raster(sub) as ras:
if ras.extent == ext:
clc_sub = sub
match = True
if not match:
clc_sub = os.path.join(outdir, 'clc{}.tif'.format(len(clc_subs)))
print('creating', clc_sub)
warp_opts['dstSRS'] = 'EPSG:{}'.format(epsg)
warp_opts['xRes'] = xRes
warp_opts['yRes'] = yRes
warp_opts['outputBounds'] = (ext['xmin'], ext['ymin'], ext['xmax'],
ext['ymax'])
gdalwarp(src=source, dst=clc_sub, options=warp_opts)
return clc_sub
def autoparse():
"""
automatic parsing of Gamma commands.
This function will detect the Gamma installation via environment variable `GAMMA_HOME`, detect all available
modules (e.g. ISP, DIFF) and parse all of the module's commands via function :func:`parse_module`.
A new Python module will be created called `gammaparse`, which is stored under `$HOME/.pyrosar`.
Upon importing the `pyroSAR.gamma` submodule, this function is run automatically and module `gammaparse`
is imported as `api`.
Returns
-------
Examples
--------
>>> from pyroSAR.gamma.api import diff
>>> print('create_dem_par' in dir(diff))
True
"""
home = ExamineGamma().home
target = os.path.join(os.path.expanduser('~'), '.pyrosar', 'gammaparse')
if not os.path.isdir(target):
os.makedirs(target)
for module in finder(home, ['[A-Z]*'], foldermode=2):
outfile = os.path.join(target,
os.path.basename(module).lower() + '.py')
if not os.path.isfile(outfile):
print('parsing module {} to {}'.format(os.path.basename(module),
outfile))
for submodule in ['bin', 'scripts']:
print('-' * 10 + '\n{}'.format(submodule))
try:
parse_module(os.path.join(module, submodule), outfile)
except OSError:
print('..does not exist')
print('=' * 20)
modules = [
re.sub(r'\.py', '', os.path.basename(x))
for x in finder(target, [r'[a-z]+\.py$'], regex=True)
]
if len(modules) > 0:
with open(os.path.join(target, '__init__.py'), 'w') as init:
init.write('from . import {}'.format(', '.join(modules)))
def __buildvrt(archives, vrtfile, pattern, vsi, extent, nodata):
locals = [
vsi + x for x in dissolve([finder(x, [pattern]) for x in archives])
]
gdalbuildvrt(src=locals,
dst=vrtfile,
options={
'outputBounds': (extent['xmin'], extent['ymin'],
extent['xmax'], extent['ymax']),
'srcNodata':
nodata
})
def __buildvrt(archives, vrtfile, pattern, vsi, extent, nodata=None, srs=None):
locals = [vsi + x for x in dissolve([finder(x, [pattern]) for x in archives])]
if nodata is None:
with Raster(locals[0]) as ras:
nodata = ras.nodata
opts = {'outputBounds': (extent['xmin'], extent['ymin'],
extent['xmax'], extent['ymax']),
'srcNodata': nodata}
if srs is not None:
opts['outputSRS'] = crsConvert(srs, 'wkt')
gdalbuildvrt(src=locals, dst=vrtfile,
options=opts)
def main():
dirs = [
"stack_savi"
] #, "stack_ndvi", "stack_msavi", "stack_reip", "stack_rvi", "stack_dvi"]
resolution = [30, 30]
# shapefile (for stack boundaries)
shp = 'F:/geodata/geo402/02_features/LADYBRAND_final_enlarged_study_area.shp'
# store results in separate files or one single stack file? If separate then dstfile is used as a directory.
sep = True
for dir in dirs:
# define input directory containing files to be stacked
dir_in = 'F:/geodata/geo402/S2/xx_S2_indices/' + dir
print(dir_in)
os.makedirs(dir_in, exist_ok=True)
# define output file name
dstfile = 'F:/geodata/geo402/S2/xx_S2_indices/mosaics/' + dir
print(dstfile)
# list files to be resampled; those not overlapping with the shapefile geometry will excluded by function stack
srcfiles = finder(dir_in, ['*'])
# check whether dstfile is already a file
if os.path.isfile(dstfile):
raise IOError('dstfile already exists')
# create groups of similar time stamps for mosaicking.
# All images with a time stamp of less than 30s difference will be grouped
groups = groupbyTime(srcfiles, seconds, 30)
# final function call
# groups will be mosaicked first
# the resulting images will all have the same extent
stack(srcfiles=groups,
dstfile=dstfile,
resampling='bilinear',
targetres=resolution,
srcnodata=-9999,
dstnodata=-9999,
shapefile=shp,
sortfun=seconds,
separate=sep,
overwrite=True)
def find_datasets(directory, recursive=False, **kwargs):
"""
find pyroSAR datasets in a directory based on their metadata
Parameters
----------
directory: str
the name of the directory to be searched
recursive: bool
search the directory recursively into subdirectories?
kwargs:
Metadata attributes for filtering the scene list supplied as `key=value`. e.g. `sensor='S1A'`.
Multiple allowed options can be provided in tuples, e.g. `sensor=('S1A', 'S1B')`.
Any types other than tuples require an exact match, e.g. `proc_steps=['grd', 'mli', 'geo', 'norm', 'db']`
will be matched only if these processing steps are contained in the product name in this exact order.
The special attributes `start` and `stop` can be used for time filtering where `start<=value<=stop`.
See function :func:`parse_datasetname` for further options.
Returns
-------
list of str
the file names found in the directory and filtered by metadata attributes
Examples
--------
>>> selection = find_datasets('path/to/files', sensor=('S1A', 'S1B'), polarization='VV')
"""
files = finder(directory, [product_pattern],
regex=True,
recursive=recursive)
selection = []
for file in files:
meta = parse_datasetname(file)
matches = []
for key, val in kwargs.items():
if key == 'start':
match = val <= meta['start']
elif key == 'stop':
match = val >= meta[
'start'] # only the start time stamp is contained in the filename
elif isinstance(val, tuple):
match = meta[key] in val
else:
match = meta[key] == val
matches.append(match)
if all(matches):
selection.append(file)
return selection
def getLocals(self, osvtype='POE'):
"""
get a list of local files of specific type
Parameters
----------
osvtype: {'POE', 'RES'}
the type of orbit files required
Returns
-------
list
a selection of local OSV files
"""
directory = self._typeEvaluate(osvtype)
return finder(directory, [self.pattern], regex=True)
def main():
'''
both polarisations are included in the script but are stored in different folders
'''
resolution = [30, 30]
# shapefile (for stack boundaries)
shp = 'F:/geodata/geo402/##study_area/LADYBRAND_final_enlarged_study_area.shp'
# store results in separate files or one single stack file? If separate then dstfile is used as a directory.
sep = False
# define input directory containing files to be stacked
dir_in = 'F:/geodata/geo402/S1_GRD/xx_new/GRD_VH_vrts/'
print(dir_in)
# os.makedirs(dir_in, exist_ok=True)
# define output file name
dstfile = 'F:/geodata/geo402/S1_GRD/xx_new/S1A_IW_GRD_VH_stack'
print(dstfile)
# list files to be resampled; those not overlapping with the shapefile geometry will excluded by function stack
srcfiles = finder(dir_in, ['*'])
# check whether dstfile is already a file
if os.path.isfile(dstfile):
raise IOError('dstfile already exists')
# create groups of similar time stamps for mosaicking.
# All images with a time stamp of less than 30s difference will be grouped
groups = groupbyTime(srcfiles, seconds, 30)
# final function call
# groups will be mosaicked first
# the resulting images will all have the same extent
stack(srcfiles=groups,
dstfile=dstfile,
resampling='bilinear',
targetres=resolution,
srcnodata=-99,
dstnodata=-99,
shapefile=shp,
sortfun=seconds,
separate=sep,
overwrite=True,
cores=7)
def test_finder(tmpdir):
dir = str(tmpdir)
dir_sub1 = os.path.join(dir, 'testdir1')
dir_sub2 = os.path.join(dir, 'testdir2')
os.makedirs(dir_sub1)
os.makedirs(dir_sub2)
with open(os.path.join(dir_sub1, 'testfile1.txt'), 'w') as t1:
t1.write('test')
with open(os.path.join(dir_sub2, 'testfile2.txt'), 'w') as t2:
t2.write('test')
assert len(anc.finder(dir, ['test*'], foldermode=0)) == 2
assert len(anc.finder(dir, ['test*'], foldermode=0, recursive=False)) == 0
assert len(anc.finder(dir, ['test*'], foldermode=1)) == 4
assert len(anc.finder(dir, ['test*'], foldermode=2)) == 2
assert len(anc.finder([dir_sub1, dir_sub2], ['test*'])) == 2
with pytest.raises(TypeError):
anc.finder(1, [])
def mindate(self, osvtype='POE', datetype='start'):
"""
return the earliest date of locally existing POE/RES files
Parameters
----------
osvtype: {'POE', 'RES'}
the type of orbit files required
datetype: {'publish', 'start', 'stop'}
one of three possible date types contained in the OSV filename
Returns
-------
str
a timestamp in format YYYYmmddTHHMMSS
"""
address, directory = self._typeEvaluate(osvtype)
files = finder(directory, [self.pattern], regex=True)
return min([self.date(x, datetype) for x in files]) if len(files) > 0 else None
def parse_module(bindir, outfile):
"""
parse all Gamma commands of a module to functions and save them to a Python script.
Parameters
----------
bindir: str
the `bin` directory of a module containing the commands
outfile: str
the name of the Python file to write
Returns
-------
Examples
--------
>>> import os
>>> from pyroSAR.gamma.parser import parse_module
>>> outname = os.path.join(os.environ['HOME'], 'isp.py')
>>> parse_module('/cluster/GAMMA_SOFTWARE-20161207/ISP/bin', outname)
"""
excludes = [
'coord_trans', 'mosaic', 'lin_comb', 'lin_comb_cpx', 'validate'
]
failed = []
outstring = 'from pyroSAR.gamma.auxil import process\n\n\n'
for cmd in sorted(finder(bindir, ['*']), key=lambda s: s.lower()):
basename = os.path.basename(cmd)
if basename not in excludes:
# print(basename)
try:
fun = parse_command(cmd)
except RuntimeError as e:
failed.append('{0}: {1}'.format(basename, str(e)))
continue
outstring += fun + '\n\n'
with open(outfile, 'w') as out:
out.write(outstring)
if len(failed) > 0:
print('the following functions could not be parsed:\n{0}\n({1} total)'.
format('\n'.join(failed), len(failed)))
product='GRD',
acquisition_mode='IW',
vv=1)
print('{0}: {1} scenes found for site {2}'.format(socket.gethostname(),
len(selection_proc),
sitename))
#######################################################################################
# call to processing utility
if len(selection_proc) > 1:
print('start processing')
for scene in selection_proc:
geocode(infile=scene, outdir=outdir, tr=resolution, scaling='db')
return len(selection_proc)
if __name__ == '__main__':
#######################################################################################
# update Sentinel-1 GRD scene archive database
# define a directory containing zipped scene archives and list all files starting with 'S1A' or 'S1B'
archive_s1 = '/.../sentinel1/GRD'
scenes_s1 = finder(archive_s1, ['^S1[AB]'], regex=True, recursive=False)
with Archive(dbfile) as archive:
archive.insert(scenes_s1)
#######################################################################################
# start the processing
results = list(futures.map(worker, sites))
def gpt(xmlfile):
"""
wrapper for ESA SNAP Graph Processing Tool GPT
input is a readily formatted workflow xml file as created by function geocode in module snap.util
"""
try:
gpt_exec = ExamineSnap().gpt
except AttributeError:
raise RuntimeError('could not find SNAP GPT executable')
with open(xmlfile, 'r') as infile:
workflow = ET.fromstring(infile.read())
write = workflow.find('.//node[@id="Write"]')
outname = write.find('.//parameters/file').text
outdir = os.path.dirname(outname)
format = write.find('.//parameters/formatName').text
infile = workflow.find('.//node[@id="Read"]/parameters/file').text
if format == 'GeoTiff-BigTIFF':
cmd = [
gpt_exec,
# '-Dsnap.dataio.reader.tileWidth=*',
# '-Dsnap.dataio.reader.tileHeight=1',
'-Dsnap.dataio.bigtiff.tiling.width=256',
'-Dsnap.dataio.bigtiff.tiling.height=256',
# '-Dsnap.dataio.bigtiff.compression.type=LZW',
# '-Dsnap.dataio.bigtiff.compression.quality=0.75',
xmlfile
]
else:
cmd = [gpt_exec, xmlfile]
# print('- processing workflow {}'.format(os.path.basename(xmlfile)))
proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8') if isinstance(out, bytes) else out
err = err.decode('utf-8') if isinstance(err, bytes) else err
if proc.returncode != 0:
if os.path.isfile(outname + '.tif'):
os.remove(outname + '.tif')
elif os.path.isdir(outname):
shutil.rmtree(outname)
print(out + err)
print('failed: {}'.format(os.path.basename(infile)))
err_match = re.search('Error: (.*)\n', out + err)
errmessage = err_match.group(1) if err_match else err
raise RuntimeError(errmessage)
if format == 'ENVI':
# print('- converting to GTiff')
suffix = parse_suffix(workflow)
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff',
'noData': 0
}
for item in finder(outname, ['*.img']):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
gdal_translate(item, name_new, translateoptions)
shutil.rmtree(outname)
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
def convert2gamma(id,
directory,
S1_noiseremoval=True,
logpath=None,
outdir=None,
shellscript=None):
"""
general function for converting SAR images to GAMMA format
Parameters
----------
id: ~pyroSAR.drivers.ID
an SAR scene object of type pyroSAR.ID or any subclass
directory: str
the output directory for the converted images
S1_noiseremoval: bool
only Sentinel-1: should noise removal be applied to the image?
logpath: str or None
a directory to write command logfiles to
outdir: str or None
the directory to execute the command in
shellscript: str or None
a file to write the Gamma commands to in shell format
Returns
-------
"""
if not isinstance(id, ID):
raise IOError('id must be of type pyroSAR.ID')
if id.compression is not None:
raise RuntimeError('scene is not yet unpacked')
if not os.path.isdir(directory):
os.makedirs(directory)
if isinstance(id, CEOS_ERS):
if id.sensor in ['ERS1', 'ERS2']:
if id.product == 'SLC' and id.meta['proc_system'] in [
'PGS-ERS', 'VMP-ERS', 'SPF-ERS'
]:
basename = '{}_{}_{}'.format(id.outname_base(),
id.polarizations[0],
id.product.lower())
outname = os.path.join(directory, basename)
if not os.path.isfile(outname):
lea = id.findfiles('LEA_01.001')[0]
dat = id.findfiles('DAT_01.001')[0]
title = re.sub(r'\.PS$', '', os.path.basename(id.file))
isp.par_ESA_ERS(CEOS_SAR_leader=lea,
SLC_par=outname + '.par',
CEOS_DAT=dat,
SLC=outname,
inlist=[title],
logpath=logpath,
outdir=outdir,
shellscript=shellscript)
else:
print('scene already converted')
else:
raise NotImplementedError(
'ERS {} product of {} processor in CEOS format not implemented yet'
.format(id.product, id.meta['proc_system']))
else:
raise NotImplementedError(
'sensor {} in CEOS format not implemented yet'.format(
id.sensor))
elif isinstance(id, CEOS_PSR):
images = id.findfiles('^IMG-')
if id.product == '1.0':
raise RuntimeError('PALSAR level 1.0 products are not supported')
for image in images:
polarization = re.search('[HV]{2}',
os.path.basename(image)).group(0)
if id.product == '1.1':
outname_base = '{}_{}_slc'.format(id.outname_base(),
polarization)
outname = os.path.join(directory, outname_base)
isp.par_EORC_PALSAR(CEOS_leader=id.file,
SLC_par=outname + '.par',
CEOS_data=image,
SLC=outname,
logpath=logpath,
outdir=outdir,
shellscript=shellscript)
else:
outname_base = '{}_{}_mli_geo'.format(id.outname_base(),
polarization)
outname = os.path.join(directory, outname_base)
diff.par_EORC_PALSAR_geo(CEOS_leader=id.file,
MLI_par=outname + '.par',
DEM_par=outname + '_dem.par',
CEOS_data=image,
MLI=outname,
logpath=logpath,
outdir=outdir,
shellscript=shellscript)
par2hdr(outname + '.par', outname + '.hdr')
elif isinstance(id, ESA):
"""
the command par_ASAR also accepts a K_dB argument for calibration in which case the resulting image names will carry the suffix GRD;
this is not implemented here but instead in function calibrate
"""
outname = os.path.join(directory, id.outname_base())
if not id.is_processed(directory):
isp.par_ASAR(ASAR_ERS_file=os.path.basename(id.file),
output_name=outname,
outdir=os.path.dirname(id.file),
logpath=logpath,
shellscript=shellscript)
os.remove(outname + '.hdr')
for item in finder(directory, [os.path.basename(outname)],
regex=True):
ext = '.par' if item.endswith('.par') else ''
base = os.path.basename(item).strip(ext)
base = base.replace('.', '_')
base = base.replace('PRI', 'pri')
base = base.replace('SLC', 'slc')
newname = os.path.join(directory, base + ext)
os.rename(item, newname)
if newname.endswith('.par'):
par2hdr(newname, newname.replace('.par', '.hdr'))
else:
raise IOError('scene already processed')
elif isinstance(id, SAFE):
if id.product == 'OCN':
raise IOError('Sentinel-1 OCN products are not supported')
if id.meta['category'] == 'A':
raise IOError(
'Sentinel-1 annotation-only products are not supported')
for xml_ann in finder(os.path.join(id.scene, 'annotation'),
[id.pattern_ds],
regex=True):
base = os.path.basename(xml_ann)
match = re.compile(id.pattern_ds).match(base)
tiff = os.path.join(id.scene, 'measurement',
base.replace('.xml', '.tiff'))
xml_cal = os.path.join(id.scene, 'annotation', 'calibration',
'calibration-' + base)
product = match.group('product')
# specify noise calibration file
# L1 GRD product: thermal noise already subtracted, specify xml_noise to add back thermal noise
# SLC products: specify noise file to remove noise
# xml_noise = '-': noise file not specified
if (S1_noiseremoval
and product == 'slc') or (not S1_noiseremoval
and product == 'grd'):
xml_noise = os.path.join(id.scene, 'annotation', 'calibration',
'noise-' + base)
else:
xml_noise = '-'
fields = (id.outname_base(), match.group('pol').upper(), product)
name = os.path.join(directory, '_'.join(fields))
pars = {
'GeoTIFF': tiff,
'annotation_XML': xml_ann,
'calibration_XML': xml_cal,
'noise_XML': xml_noise,
'logpath': logpath,
'shellscript': shellscript,
'outdir': outdir
}
if product == 'slc':
swath = match.group('swath').upper()
name = name.replace(
'{:_<{length}}'.format(id.acquisition_mode,
length=len(swath)), swath)
pars['SLC'] = name
pars['SLC_par'] = name + '.par'
pars['TOPS_par'] = name + '.tops_par'
isp.par_S1_SLC(**pars)
else:
pars['MLI'] = name
pars['MLI_par'] = name + '.par'
isp.par_S1_GRD(**pars)
par2hdr(name + '.par', name + '.hdr')
elif isinstance(id, TSX):
images = id.findfiles(id.pattern_ds)
pattern = re.compile(id.pattern_ds)
for image in images:
pol = pattern.match(os.path.basename(image)).group('pol')
outname = os.path.join(directory, id.outname_base() + '_' + pol)
pars = {
'annotation_XML': id.file,
'pol': pol,
'logpath': logpath,
'shellscript': shellscript,
'outdir': outdir
}
if id.product == 'SSC':
outname += '_slc'
pars['COSAR'] = image
pars['SLC_par'] = outname + '.par'
pars['SLC'] = outname
isp.par_TX_SLC(**pars)
elif id.product == 'MGD':
outname += '_mli'
pars['GeoTIFF'] = image
pars['GRD_par'] = outname + '.par'
pars['GRD'] = outname
isp.par_TX_GRD(**pars)
elif id.product in ['GEC', 'EEC']:
outname += '_mli_geo'
pars['GeoTIFF'] = image
pars['MLI_par'] = outname + '.par'
pars['DEM_par'] = outname + '_dem.par'
pars['GEO'] = outname
diff.par_TX_geo(**pars)
else:
raise RuntimeError('unknown product: {}'.format(id.product))
par2hdr(outname + '.par', outname + '.hdr')
else:
raise NotImplementedError(
'conversion for class {} is not implemented yet'.format(
type(id).__name__))
def gpt(xmlfile, groups=None, cleanup=True):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
Returns
-------
"""
workflow = Workflow(xmlfile)
write = workflow['Write']
outname = write.parameters['file']
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName').text
if dem_name == 'External DEM':
dem_nodata = float(workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
print('executing node sequence{}..'.format('s' if groups is not None else ''))
if groups is not None:
subs = split(xmlfile, groups)
for sub in subs:
execute(sub, cleanup=cleanup)
else:
execute(xmlfile, cleanup=cleanup)
if format == 'ENVI':
print('converting to GTiff')
suffix = workflow.suffix
translateoptions = {'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'}
for item in finder(outname, ['*.img'], recursive=False):
if re.search('[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix, '{0}_{1}.tif'.format(pol, suffix))
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
if cleanup:
shutil.rmtree(outname)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
print('done')
def gpt(xmlfile,
groups=None,
cleanup=True,
gpt_exceptions=None,
removeS1BorderNoiseMethod='pyroSAR'):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
gpt_exceptions: dict
a dictionary to override the configured GPT executable for certain operators;
each (sub-)workflow containing this operator will be executed with the define executable;
- e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
removeS1BorderNoiseMethod: str
the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following:
- 'ESA': the pure implementation as described by ESA
- 'pyroSAR': the ESA method plus the custom pyroSAR refinement
Returns
-------
"""
workflow = Workflow(xmlfile)
write = workflow['Write']
outname = write.parameters['file']
suffix = workflow.suffix
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName').text
if dem_name == 'External DEM':
dem_nodata = float(
workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
if 'Remove-GRD-Border-Noise' in workflow.ids and removeS1BorderNoiseMethod == 'pyroSAR':
if 'SliceAssembly' in workflow.operators:
raise RuntimeError(
"pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs"
)
xmlfile = os.path.join(outname,
os.path.basename(xmlfile.replace('_bnr', '')))
if not os.path.isdir(outname):
os.makedirs(outname)
# border noise removal is done outside of SNAP and the node is thus removed from the workflow
del workflow['Remove-GRD-Border-Noise']
# remove the node name from the groups
i = 0
while i < len(groups) - 1:
if 'Remove-GRD-Border-Noise' in groups[i]:
del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
if len(groups[i]) == 0:
del groups[i]
else:
i += 1
# identify the input scene, unpack it and perform the custom border noise removal
read = workflow['Read']
scene = identify(read.parameters['file'])
print('unpacking scene')
scene.unpack(outname)
print('removing border noise..')
scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod)
# change the name of the input file to that of the unpacked archive
read.parameters['file'] = scene.scene
# write a new workflow file
workflow.write(xmlfile)
print('executing node sequence{}..'.format(
's' if groups is not None else ''))
if groups is not None:
subs = split(xmlfile, groups)
for sub in subs:
execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
else:
execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
if format == 'ENVI':
print('converting to GTiff')
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'
}
for item in finder(outname, ['*.img'], recursive=False):
if re.search('[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
if cleanup:
shutil.rmtree(outname)
###########################################################################
# write the Sentinel-1 manifest.safe file as addition to the actual product
attrs = parse_datasetname(outname)
ext = '' if attrs['extensions'] is None else attrs['extensions']
readers = workflow['operator=Read']
for reader in readers:
infile = reader.parameters['file']
try:
id = identify(infile)
if id.sensor in ['S1A', 'S1B']:
manifest = id.getFileObj(id.findfiles('manifest.safe')[0])
basename = '_'.join([id.outname_base() + ext, 'manifest.safe'])
outdir = os.path.dirname(outname)
outname_manifest = os.path.join(outdir, basename)
with open(outname_manifest, 'wb') as out:
out.write(manifest.read())
except RuntimeError:
continue
###########################################################################
print('done')
def test_finder(tmpdir, testdata):
dir = str(tmpdir)
dir_sub1 = os.path.join(dir, 'testdir1')
dir_sub2 = os.path.join(dir, 'testdir2')
os.makedirs(dir_sub1)
os.makedirs(dir_sub2)
with open(os.path.join(dir_sub1, 'testfile1.txt'), 'w') as t1:
t1.write('test')
with open(os.path.join(dir_sub2, 'testfile2.txt'), 'w') as t2:
t2.write('test')
assert len(anc.finder(dir, ['test*'], foldermode=0)) == 2
assert len(anc.finder(dir, ['test*'], foldermode=0, recursive=False)) == 0
assert len(anc.finder(dir, ['test*'], foldermode=1)) == 4
assert len(anc.finder(dir, ['test*'], foldermode=2)) == 2
assert len(anc.finder([dir_sub1, dir_sub2], ['test*'])) == 2
assert len(anc.finder(testdata['zip'], ['file*'])) == 3
assert len(anc.finder(testdata['zip'], ['*'], foldermode=1)) == 5
assert len(anc.finder(testdata['zip'], ['[a-z]{1}'], foldermode=2, regex=True)) == 2
assert len(anc.finder(testdata['tar'], ['file*'])) == 3
assert len(anc.finder(testdata['tar'], ['*'], foldermode=1)) == 5
assert len(anc.finder(testdata['tar'], ['[a-z]{1}'], foldermode=2, regex=True)) == 2
with pytest.raises(TypeError):
anc.finder(1, [])
with pytest.raises(ValueError):
anc.finder(dir, ['test*'], foldermode=3)
with pytest.raises(RuntimeError):
anc.finder('foobar', ['test*'], foldermode=2)
with pytest.raises(RuntimeError):
anc.finder(testdata['tif'], ['test*'], foldermode=2)
def gpt(xmlfile, groups=None, cleanup=True, gpt_exceptions=None):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
gpt_exceptions: dict
a dictionary to override the configured GPT executable for certain operators;
each (sub-)workflow containing this operator will be executed with the define executable;
- e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
Returns
-------
"""
workflow = Workflow(xmlfile)
write = workflow['Write']
outname = write.parameters['file']
suffix = workflow.suffix
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName').text
if dem_name == 'External DEM':
dem_nodata = float(
workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
if 'Remove-GRD-Border-Noise' in workflow.ids:
xmlfile = os.path.join(outname,
os.path.basename(xmlfile.replace('_bnr', '')))
if not os.path.isdir(outname):
os.makedirs(outname)
# border noise removal is done outside of SNAP and the node is thus removed from the workflow
del workflow['Remove-GRD-Border-Noise']
# remove the node name from the groups
i = 0
while i < len(groups) - 1:
if 'Remove-GRD-Border-Noise' in groups[i]:
del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
if len(groups[i]) == 0:
del groups[i]
else:
i += 1
# identify the input scene, unpack it and perform the custom border noise removal
read = workflow['Read']
scene = identify(read.parameters['file'])
print('unpacking scene')
scene.unpack(outname)
print('removing border noise..')
scene.removeGRDBorderNoise()
# change the name of the input file to that of the unpacked archive
read.parameters['file'] = scene.scene
# write a new workflow file
workflow.write(xmlfile)
print('executing node sequence{}..'.format(
's' if groups is not None else ''))
if groups is not None:
subs = split(xmlfile, groups)
for sub in subs:
execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
else:
execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
if format == 'ENVI':
print('converting to GTiff')
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'
}
for item in finder(outname, ['*.img'], recursive=False):
if re.search('[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
if cleanup:
shutil.rmtree(outname)
print('done')
def hgt_collect(parfiles, outdir, demdir=None, arcsec=3):
"""
automatic downloading and unpacking of srtm tiles
Parameters
----------
parfiles: list of str or pyroSAR.ID
a list of Gamma parameter files or pyroSAR ID objects
outdir: str
a target directory to download the tiles to
demdir: str or None
an additional directory already containing hgt tiles
arcsec: {1, 3}
the spatial resolution to be used
Returns
-------
list
the names of all local hgt tiles overlapping with the parfiles
"""
# concatenate required hgt tile names
target_ids = hgt(parfiles)
targets = []
pattern = '[NS][0-9]{2}[EW][0-9]{3}'
# if an additional dem directory has been defined, check this directory for required hgt tiles
if demdir is not None:
targets.extend(finder(demdir, target_ids))
# check for additional potentially existing hgt tiles in the defined output directory
extras = [os.path.join(outdir, x) for x in target_ids if
os.path.isfile(os.path.join(outdir, x)) and not re.search(x, '\n'.join(targets))]
targets.extend(extras)
print('found {} relevant SRTM tiles...'.format(len(targets)))
# search server for all required tiles, which were not found in the local directories
if len(targets) < len(target_ids):
print('searching for additional SRTM tiles on the server...')
onlines = []
if arcsec == 1:
remotes = ['http://e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/']
remotepattern = pattern + '.SRTMGL1.hgt.zip'
elif arcsec == 3:
server = 'https://dds.cr.usgs.gov/srtm/version2_1/SRTM3/'
remotes = [os.path.join(server, x) for x in
['Africa', 'Australia', 'Eurasia', 'Islands', 'North_America', 'South_America']]
remotepattern = pattern + '[.]hgt.zip'
else:
raise ValueError('argument arcsec must be of value 1 or 3')
for remote in remotes:
response = urlopen(remote).read()
items = sorted(set(re.findall(remotepattern, response)))
for item in items:
outname = re.findall(pattern, item)[0] + '.hgt'
if outname in target_ids and outname not in [os.path.basename(x) for x in targets]:
onlines.append(os.path.join(remote, item))
# if additional tiles have been found online, download and unzip them to the local directory
if len(onlines) > 0:
print('downloading {} SRTM tiles...'.format(len(onlines)))
for candidate in onlines:
localname = os.path.join(outdir, re.findall(pattern, candidate)[0] + '.hgt')
infile = urlopen(candidate)
with open(localname + '.zip', 'wb') as outfile:
outfile.write(infile.read())
infile.close()
with zf.ZipFile(localname + '.zip', 'r') as z:
z.extractall(outdir)
os.remove(localname + '.zip')
targets.append(localname)
return targets
def gpt(xmlfile,
outdir,
groups=None,
cleanup=True,
gpt_exceptions=None,
gpt_args=None,
removeS1BorderNoiseMethod='pyroSAR',
basename_extensions=None,
multisource=False):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
outdir: str
the directory into which to write the final files
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
gpt_exceptions: dict
a dictionary to override the configured GPT executable for certain operators;
each (sub-)workflow containing this operator will be executed with the define executable;
- e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
gpt_args: list or None
a list of additional arguments to be passed to the gpt call
- e.g. ``['-x', '-c', '2048M']`` for increased tile cache size and intermediate clearing
removeS1BorderNoiseMethod: str
the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following:
- 'ESA': the pure implementation as described by ESA
- 'pyroSAR': the ESA method plus the custom pyroSAR refinement
basename_extensions: list of str
names of additional parameters to append to the basename, e.g. ['orbitNumber_rel']
Returns
-------
Raises
------
RuntimeError
"""
workflow = Workflow(xmlfile)
if multisource:
read = workflow['ProductSet-Reader']
# scene = identify_many(read.parameters['fileList'].split(",")) # not working
elif not multisource:
read = workflow['Read']
scene = identify(read.parameters['file'])
write = workflow['Write']
tmpname = write.parameters['file']
suffix = workflow.suffix()
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName')
if dem_name is not None:
if dem_name.text == 'External DEM':
dem_nodata = float(
workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
if 'Remove-GRD-Border-Noise' in workflow.ids \
and removeS1BorderNoiseMethod == 'pyroSAR' \
and scene.meta['IPF_version'] < 2.9:
if 'SliceAssembly' in workflow.operators:
raise RuntimeError(
"pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs"
)
xmlfile = os.path.join(tmpname,
os.path.basename(xmlfile.replace('_bnr', '')))
os.makedirs(tmpname, exist_ok=True)
# border noise removal is done outside of SNAP and the node is thus removed from the workflow
del workflow['Remove-GRD-Border-Noise']
# remove the node name from the groups
i = 0
while i < len(groups) - 1:
if 'Remove-GRD-Border-Noise' in groups[i]:
del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
if len(groups[i]) == 0:
del groups[i]
else:
i += 1
# unpack the scene if necessary and perform the custom border noise removal
print('unpacking scene')
if scene.compression is not None:
scene.unpack(tmpname)
print('removing border noise..')
scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod)
# change the name of the input file to that of the unpacked archive
read.parameters['file'] = scene.scene
# write a new workflow file
workflow.write(xmlfile)
print('executing node sequence{}..'.format(
's' if groups is not None else ''))
try:
if groups is not None:
tmpdir = os.path.join(tmpname, 'tmp')
subs = split(xmlfile, groups, tmpdir)
for sub in subs:
execute(sub,
cleanup=cleanup,
gpt_exceptions=gpt_exceptions,
gpt_args=gpt_args)
else:
execute(xmlfile,
cleanup=cleanup,
gpt_exceptions=gpt_exceptions,
gpt_args=gpt_args)
except RuntimeError as e:
if cleanup and os.path.exists(tmpname):
shutil.rmtree(tmpname, onerror=windows_fileprefix)
raise RuntimeError(str(e) + '\nfailed: {}'.format(xmlfile))
outname = os.path.join(outdir, os.path.basename(tmpname))
if format == 'ENVI':
print('converting to GTiff')
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'
}
for item in finder(tmpname, ['*.img'], recursive=False):
if re.search('ma0_[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
if 'Sigma0' in item:
name_new = name_new.replace('TF_', '')
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
if re.search('layover_shadow_mask', base):
base = re.sub('layover_shadow_mask_[HV]{2}',
'layoverShadowMask', base)
if re.search('scatteringArea', base):
base = re.sub('scatteringArea_[HV]{2}', 'scatteringArea',
base)
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
###########################################################################
# write the Sentinel-1 manifest.safe file as addition to the actual product
if not multisource:
readers = workflow['operator=Read']
for reader in readers:
infile = reader.parameters['file']
try:
id = identify(infile)
if id.sensor in ['S1A', 'S1B']:
manifest = id.getFileObj(id.findfiles('manifest.safe')[0])
basename = id.outname_base(basename_extensions)
basename = '{0}_manifest.safe'.format(basename)
outname_manifest = os.path.join(outdir, basename)
with open(outname_manifest, 'wb') as out:
out.write(manifest.read())
except RuntimeError:
continue
###########################################################################
if cleanup and os.path.exists(tmpname):
shutil.rmtree(tmpname, onerror=windows_fileprefix)
print('done')