def gpt(xmlfile,
outdir,
groups=None,
cleanup=True,
gpt_exceptions=None,
gpt_args=None,
removeS1BorderNoiseMethod='pyroSAR',
basename_extensions=None,
multisource=False):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
outdir: str
the directory into which to write the final files
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
gpt_exceptions: dict
a dictionary to override the configured GPT executable for certain operators;
each (sub-)workflow containing this operator will be executed with the define executable;
- e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
gpt_args: list or None
a list of additional arguments to be passed to the gpt call
- e.g. ``['-x', '-c', '2048M']`` for increased tile cache size and intermediate clearing
removeS1BorderNoiseMethod: str
the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following:
- 'ESA': the pure implementation as described by ESA
- 'pyroSAR': the ESA method plus the custom pyroSAR refinement
basename_extensions: list of str
names of additional parameters to append to the basename, e.g. ['orbitNumber_rel']
Returns
-------
Raises
------
RuntimeError
"""
workflow = Workflow(xmlfile)
if multisource:
read = workflow['ProductSet-Reader']
# scene = identify_many(read.parameters['fileList'].split(",")) # not working
elif not multisource:
read = workflow['Read']
scene = identify(read.parameters['file'])
write = workflow['Write']
tmpname = write.parameters['file']
suffix = workflow.suffix()
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName')
if dem_name is not None:
if dem_name.text == 'External DEM':
dem_nodata = float(
workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
if 'Remove-GRD-Border-Noise' in workflow.ids \
and removeS1BorderNoiseMethod == 'pyroSAR' \
and scene.meta['IPF_version'] < 2.9:
if 'SliceAssembly' in workflow.operators:
raise RuntimeError(
"pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs"
)
xmlfile = os.path.join(tmpname,
os.path.basename(xmlfile.replace('_bnr', '')))
os.makedirs(tmpname, exist_ok=True)
# border noise removal is done outside of SNAP and the node is thus removed from the workflow
del workflow['Remove-GRD-Border-Noise']
# remove the node name from the groups
i = 0
while i < len(groups) - 1:
if 'Remove-GRD-Border-Noise' in groups[i]:
del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
if len(groups[i]) == 0:
del groups[i]
else:
i += 1
# unpack the scene if necessary and perform the custom border noise removal
print('unpacking scene')
if scene.compression is not None:
scene.unpack(tmpname)
print('removing border noise..')
scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod)
# change the name of the input file to that of the unpacked archive
read.parameters['file'] = scene.scene
# write a new workflow file
workflow.write(xmlfile)
print('executing node sequence{}..'.format(
's' if groups is not None else ''))
try:
if groups is not None:
tmpdir = os.path.join(tmpname, 'tmp')
subs = split(xmlfile, groups, tmpdir)
for sub in subs:
execute(sub,
cleanup=cleanup,
gpt_exceptions=gpt_exceptions,
gpt_args=gpt_args)
else:
execute(xmlfile,
cleanup=cleanup,
gpt_exceptions=gpt_exceptions,
gpt_args=gpt_args)
except RuntimeError as e:
if cleanup and os.path.exists(tmpname):
shutil.rmtree(tmpname, onerror=windows_fileprefix)
raise RuntimeError(str(e) + '\nfailed: {}'.format(xmlfile))
outname = os.path.join(outdir, os.path.basename(tmpname))
if format == 'ENVI':
print('converting to GTiff')
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'
}
for item in finder(tmpname, ['*.img'], recursive=False):
if re.search('ma0_[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
if 'Sigma0' in item:
name_new = name_new.replace('TF_', '')
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
if re.search('layover_shadow_mask', base):
base = re.sub('layover_shadow_mask_[HV]{2}',
'layoverShadowMask', base)
if re.search('scatteringArea', base):
base = re.sub('scatteringArea_[HV]{2}', 'scatteringArea',
base)
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
###########################################################################
# write the Sentinel-1 manifest.safe file as addition to the actual product
if not multisource:
readers = workflow['operator=Read']
for reader in readers:
infile = reader.parameters['file']
try:
id = identify(infile)
if id.sensor in ['S1A', 'S1B']:
manifest = id.getFileObj(id.findfiles('manifest.safe')[0])
basename = id.outname_base(basename_extensions)
basename = '{0}_manifest.safe'.format(basename)
outname_manifest = os.path.join(outdir, basename)
with open(outname_manifest, 'wb') as out:
out.write(manifest.read())
except RuntimeError:
continue
###########################################################################
if cleanup and os.path.exists(tmpname):
shutil.rmtree(tmpname, onerror=windows_fileprefix)
print('done')
def gpt(xmlfile, groups=None, cleanup=True, gpt_exceptions=None):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
gpt_exceptions: dict
a dictionary to override the configured GPT executable for certain operators;
each (sub-)workflow containing this operator will be executed with the define executable;
- e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
Returns
-------
"""
workflow = Workflow(xmlfile)
write = workflow['Write']
outname = write.parameters['file']
suffix = workflow.suffix
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName').text
if dem_name == 'External DEM':
dem_nodata = float(
workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
if 'Remove-GRD-Border-Noise' in workflow.ids:
xmlfile = os.path.join(outname,
os.path.basename(xmlfile.replace('_bnr', '')))
if not os.path.isdir(outname):
os.makedirs(outname)
# border noise removal is done outside of SNAP and the node is thus removed from the workflow
del workflow['Remove-GRD-Border-Noise']
# remove the node name from the groups
i = 0
while i < len(groups) - 1:
if 'Remove-GRD-Border-Noise' in groups[i]:
del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
if len(groups[i]) == 0:
del groups[i]
else:
i += 1
# identify the input scene, unpack it and perform the custom border noise removal
read = workflow['Read']
scene = identify(read.parameters['file'])
print('unpacking scene')
scene.unpack(outname)
print('removing border noise..')
scene.removeGRDBorderNoise()
# change the name of the input file to that of the unpacked archive
read.parameters['file'] = scene.scene
# write a new workflow file
workflow.write(xmlfile)
print('executing node sequence{}..'.format(
's' if groups is not None else ''))
if groups is not None:
subs = split(xmlfile, groups)
for sub in subs:
execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
else:
execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
if format == 'ENVI':
print('converting to GTiff')
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'
}
for item in finder(outname, ['*.img'], recursive=False):
if re.search('[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
if cleanup:
shutil.rmtree(outname)
print('done')
def gpt(xmlfile, groups=None, cleanup=True):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
Returns
-------
"""
workflow = Workflow(xmlfile)
write = workflow['Write']
outname = write.parameters['file']
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName').text
if dem_name == 'External DEM':
dem_nodata = float(workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
print('executing node sequence{}..'.format('s' if groups is not None else ''))
if groups is not None:
subs = split(xmlfile, groups)
for sub in subs:
execute(sub, cleanup=cleanup)
else:
execute(xmlfile, cleanup=cleanup)
if format == 'ENVI':
print('converting to GTiff')
suffix = workflow.suffix
translateoptions = {'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'}
for item in finder(outname, ['*.img'], recursive=False):
if re.search('[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix, '{0}_{1}.tif'.format(pol, suffix))
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
if cleanup:
shutil.rmtree(outname)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
print('done')
def gpt(xmlfile,
groups=None,
cleanup=True,
gpt_exceptions=None,
removeS1BorderNoiseMethod='pyroSAR'):
"""
wrapper for ESA SNAP's Graph Processing Tool GPT.
Input is a readily formatted workflow XML file as
created by function :func:`~pyroSAR.snap.util.geocode`.
Additional to calling GPT, this function will
* execute the workflow in groups as defined by `groups`
* encode a nodata value into the output file if the format is GeoTiff-BigTIFF
* convert output files to GeoTiff if the output format is ENVI
Parameters
----------
xmlfile: str
the name of the workflow XML file
groups: list
a list of lists each containing IDs for individual nodes
cleanup: bool
should all files written to the temporary directory during function execution be deleted after processing?
gpt_exceptions: dict
a dictionary to override the configured GPT executable for certain operators;
each (sub-)workflow containing this operator will be executed with the define executable;
- e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
removeS1BorderNoiseMethod: str
the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following:
- 'ESA': the pure implementation as described by ESA
- 'pyroSAR': the ESA method plus the custom pyroSAR refinement
Returns
-------
"""
workflow = Workflow(xmlfile)
write = workflow['Write']
outname = write.parameters['file']
suffix = workflow.suffix
format = write.parameters['formatName']
dem_name = workflow.tree.find('.//demName').text
if dem_name == 'External DEM':
dem_nodata = float(
workflow.tree.find('.//externalDEMNoDataValue').text)
else:
dem_nodata = 0
if 'Remove-GRD-Border-Noise' in workflow.ids and removeS1BorderNoiseMethod == 'pyroSAR':
if 'SliceAssembly' in workflow.operators:
raise RuntimeError(
"pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs"
)
xmlfile = os.path.join(outname,
os.path.basename(xmlfile.replace('_bnr', '')))
if not os.path.isdir(outname):
os.makedirs(outname)
# border noise removal is done outside of SNAP and the node is thus removed from the workflow
del workflow['Remove-GRD-Border-Noise']
# remove the node name from the groups
i = 0
while i < len(groups) - 1:
if 'Remove-GRD-Border-Noise' in groups[i]:
del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
if len(groups[i]) == 0:
del groups[i]
else:
i += 1
# identify the input scene, unpack it and perform the custom border noise removal
read = workflow['Read']
scene = identify(read.parameters['file'])
print('unpacking scene')
scene.unpack(outname)
print('removing border noise..')
scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod)
# change the name of the input file to that of the unpacked archive
read.parameters['file'] = scene.scene
# write a new workflow file
workflow.write(xmlfile)
print('executing node sequence{}..'.format(
's' if groups is not None else ''))
if groups is not None:
subs = split(xmlfile, groups)
for sub in subs:
execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
else:
execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
if format == 'ENVI':
print('converting to GTiff')
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff'
}
for item in finder(outname, ['*.img'], recursive=False):
if re.search('[HV]{2}', item):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
else:
base = os.path.splitext(os.path.basename(item))[0] \
.replace('elevation', 'DEM')
name_new = outname.replace(suffix, '{0}.tif'.format(base))
nodata = dem_nodata if re.search('elevation', item) else 0
translateoptions['noData'] = nodata
gdal_translate(item, name_new, translateoptions)
# by default the nodata value is not registered in the GTiff metadata
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None
if cleanup:
shutil.rmtree(outname)
###########################################################################
# write the Sentinel-1 manifest.safe file as addition to the actual product
attrs = parse_datasetname(outname)
ext = '' if attrs['extensions'] is None else attrs['extensions']
readers = workflow['operator=Read']
for reader in readers:
infile = reader.parameters['file']
try:
id = identify(infile)
if id.sensor in ['S1A', 'S1B']:
manifest = id.getFileObj(id.findfiles('manifest.safe')[0])
basename = '_'.join([id.outname_base() + ext, 'manifest.safe'])
outdir = os.path.dirname(outname)
outname_manifest = os.path.join(outdir, basename)
with open(outname_manifest, 'wb') as out:
out.write(manifest.read())
except RuntimeError:
continue
###########################################################################
print('done')
def gpt(xmlfile):
"""
wrapper for ESA SNAP Graph Processing Tool GPT
input is a readily formatted workflow xml file as created by function geocode in module snap.util
"""
try:
gpt_exec = ExamineSnap().gpt
except AttributeError:
raise RuntimeError('could not find SNAP GPT executable')
with open(xmlfile, 'r') as infile:
workflow = ET.fromstring(infile.read())
write = workflow.find('.//node[@id="Write"]')
outname = write.find('.//parameters/file').text
outdir = os.path.dirname(outname)
format = write.find('.//parameters/formatName').text
infile = workflow.find('.//node[@id="Read"]/parameters/file').text
if format == 'GeoTiff-BigTIFF':
cmd = [
gpt_exec,
# '-Dsnap.dataio.reader.tileWidth=*',
# '-Dsnap.dataio.reader.tileHeight=1',
'-Dsnap.dataio.bigtiff.tiling.width=256',
'-Dsnap.dataio.bigtiff.tiling.height=256',
# '-Dsnap.dataio.bigtiff.compression.type=LZW',
# '-Dsnap.dataio.bigtiff.compression.quality=0.75',
xmlfile
]
else:
cmd = [gpt_exec, xmlfile]
# print('- processing workflow {}'.format(os.path.basename(xmlfile)))
proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8') if isinstance(out, bytes) else out
err = err.decode('utf-8') if isinstance(err, bytes) else err
if proc.returncode != 0:
if os.path.isfile(outname + '.tif'):
os.remove(outname + '.tif')
elif os.path.isdir(outname):
shutil.rmtree(outname)
print(out + err)
print('failed: {}'.format(os.path.basename(infile)))
err_match = re.search('Error: (.*)\n', out + err)
errmessage = err_match.group(1) if err_match else err
raise RuntimeError(errmessage)
if format == 'ENVI':
# print('- converting to GTiff')
suffix = parse_suffix(workflow)
translateoptions = {
'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
'format': 'GTiff',
'noData': 0
}
for item in finder(outname, ['*.img']):
pol = re.search('[HV]{2}', item).group()
name_new = outname.replace(suffix,
'{0}_{1}.tif'.format(pol, suffix))
gdal_translate(item, name_new, translateoptions)
shutil.rmtree(outname)
elif format == 'GeoTiff-BigTIFF':
ras = gdal.Open(outname + '.tif', GA_Update)
for i in range(1, ras.RasterCount + 1):
ras.GetRasterBand(i).SetNoDataValue(0)
ras = None