def gpt(xmlfile,
        outdir,
        groups=None,
        cleanup=True,
        gpt_exceptions=None,
        gpt_args=None,
        removeS1BorderNoiseMethod='pyroSAR',
        basename_extensions=None,
        multisource=False):
    """
    wrapper for ESA SNAP's Graph Processing Tool GPT.
    Input is a readily formatted workflow XML file as
    created by function :func:`~pyroSAR.snap.util.geocode`.
    Additional to calling GPT, this function will
    
     * execute the workflow in groups as defined by `groups`
     * encode a nodata value into the output file if the format is GeoTiff-BigTIFF
     * convert output files to GeoTiff if the output format is ENVI
    
    Parameters
    ----------
    xmlfile: str
        the name of the workflow XML file
    outdir: str
        the directory into which to write the final files
    groups: list
        a list of lists each containing IDs for individual nodes
    cleanup: bool
        should all files written to the temporary directory during function execution be deleted after processing?
    gpt_exceptions: dict
        a dictionary to override the configured GPT executable for certain operators;
        each (sub-)workflow containing this operator will be executed with the define executable;
        
         - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
    gpt_args: list or None
        a list of additional arguments to be passed to the gpt call
        
        - e.g. ``['-x', '-c', '2048M']`` for increased tile cache size and intermediate clearing
    removeS1BorderNoiseMethod: str
        the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following:
         - 'ESA': the pure implementation as described by ESA
         - 'pyroSAR': the ESA method plus the custom pyroSAR refinement
    basename_extensions: list of str
        names of additional parameters to append to the basename, e.g. ['orbitNumber_rel']
    
    Returns
    -------
    
    Raises
    ------
    RuntimeError
    """

    workflow = Workflow(xmlfile)

    if multisource:
        read = workflow['ProductSet-Reader']
        # scene = identify_many(read.parameters['fileList'].split(",")) # not working
    elif not multisource:
        read = workflow['Read']
        scene = identify(read.parameters['file'])

    write = workflow['Write']
    tmpname = write.parameters['file']
    suffix = workflow.suffix()
    format = write.parameters['formatName']
    dem_name = workflow.tree.find('.//demName')
    if dem_name is not None:
        if dem_name.text == 'External DEM':
            dem_nodata = float(
                workflow.tree.find('.//externalDEMNoDataValue').text)
        else:
            dem_nodata = 0

    if 'Remove-GRD-Border-Noise' in workflow.ids \
            and removeS1BorderNoiseMethod == 'pyroSAR' \
            and scene.meta['IPF_version'] < 2.9:
        if 'SliceAssembly' in workflow.operators:
            raise RuntimeError(
                "pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs"
            )
        xmlfile = os.path.join(tmpname,
                               os.path.basename(xmlfile.replace('_bnr', '')))
        os.makedirs(tmpname, exist_ok=True)
        # border noise removal is done outside of SNAP and the node is thus removed from the workflow
        del workflow['Remove-GRD-Border-Noise']
        # remove the node name from the groups
        i = 0
        while i < len(groups) - 1:
            if 'Remove-GRD-Border-Noise' in groups[i]:
                del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
            if len(groups[i]) == 0:
                del groups[i]
            else:
                i += 1
        # unpack the scene if necessary and perform the custom border noise removal
        print('unpacking scene')
        if scene.compression is not None:
            scene.unpack(tmpname)
        print('removing border noise..')
        scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod)
        # change the name of the input file to that of the unpacked archive
        read.parameters['file'] = scene.scene
        # write a new workflow file
        workflow.write(xmlfile)

    print('executing node sequence{}..'.format(
        's' if groups is not None else ''))
    try:
        if groups is not None:
            tmpdir = os.path.join(tmpname, 'tmp')
            subs = split(xmlfile, groups, tmpdir)
            for sub in subs:
                execute(sub,
                        cleanup=cleanup,
                        gpt_exceptions=gpt_exceptions,
                        gpt_args=gpt_args)
        else:
            execute(xmlfile,
                    cleanup=cleanup,
                    gpt_exceptions=gpt_exceptions,
                    gpt_args=gpt_args)
    except RuntimeError as e:
        if cleanup and os.path.exists(tmpname):
            shutil.rmtree(tmpname, onerror=windows_fileprefix)
        raise RuntimeError(str(e) + '\nfailed: {}'.format(xmlfile))

    outname = os.path.join(outdir, os.path.basename(tmpname))

    if format == 'ENVI':
        print('converting to GTiff')
        translateoptions = {
            'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
            'format': 'GTiff'
        }
        for item in finder(tmpname, ['*.img'], recursive=False):
            if re.search('ma0_[HV]{2}', item):
                pol = re.search('[HV]{2}', item).group()
                name_new = outname.replace(suffix,
                                           '{0}_{1}.tif'.format(pol, suffix))
                if 'Sigma0' in item:
                    name_new = name_new.replace('TF_', '')
            else:
                base = os.path.splitext(os.path.basename(item))[0] \
                    .replace('elevation', 'DEM')
                if re.search('layover_shadow_mask', base):
                    base = re.sub('layover_shadow_mask_[HV]{2}',
                                  'layoverShadowMask', base)
                if re.search('scatteringArea', base):
                    base = re.sub('scatteringArea_[HV]{2}', 'scatteringArea',
                                  base)
                name_new = outname.replace(suffix, '{0}.tif'.format(base))
            nodata = dem_nodata if re.search('elevation', item) else 0
            translateoptions['noData'] = nodata
            gdal_translate(item, name_new, translateoptions)
    # by default the nodata value is not registered in the GTiff metadata
    elif format == 'GeoTiff-BigTIFF':
        ras = gdal.Open(outname + '.tif', GA_Update)
        for i in range(1, ras.RasterCount + 1):
            ras.GetRasterBand(i).SetNoDataValue(0)
        ras = None
    ###########################################################################
    # write the Sentinel-1 manifest.safe file as addition to the actual product
    if not multisource:
        readers = workflow['operator=Read']
        for reader in readers:
            infile = reader.parameters['file']
            try:
                id = identify(infile)
                if id.sensor in ['S1A', 'S1B']:
                    manifest = id.getFileObj(id.findfiles('manifest.safe')[0])
                    basename = id.outname_base(basename_extensions)
                    basename = '{0}_manifest.safe'.format(basename)
                    outname_manifest = os.path.join(outdir, basename)
                    with open(outname_manifest, 'wb') as out:
                        out.write(manifest.read())
            except RuntimeError:
                continue
    ###########################################################################
    if cleanup and os.path.exists(tmpname):
        shutil.rmtree(tmpname, onerror=windows_fileprefix)
    print('done')
Beispiel #2
0
def gpt(xmlfile, groups=None, cleanup=True, gpt_exceptions=None):
    """
    wrapper for ESA SNAP's Graph Processing Tool GPT.
    Input is a readily formatted workflow XML file as
    created by function :func:`~pyroSAR.snap.util.geocode`.
    Additional to calling GPT, this function will
    
     * execute the workflow in groups as defined by `groups`
     * encode a nodata value into the output file if the format is GeoTiff-BigTIFF
     * convert output files to GeoTiff if the output format is ENVI
    
    Parameters
    ----------
    xmlfile: str
        the name of the workflow XML file
    groups: list
        a list of lists each containing IDs for individual nodes
    cleanup: bool
        should all files written to the temporary directory during function execution be deleted after processing?
    gpt_exceptions: dict
        a dictionary to override the configured GPT executable for certain operators;
        each (sub-)workflow containing this operator will be executed with the define executable;
        
         - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
    
    Returns
    -------

    """

    workflow = Workflow(xmlfile)
    write = workflow['Write']
    outname = write.parameters['file']
    suffix = workflow.suffix
    format = write.parameters['formatName']
    dem_name = workflow.tree.find('.//demName').text
    if dem_name == 'External DEM':
        dem_nodata = float(
            workflow.tree.find('.//externalDEMNoDataValue').text)
    else:
        dem_nodata = 0

    if 'Remove-GRD-Border-Noise' in workflow.ids:
        xmlfile = os.path.join(outname,
                               os.path.basename(xmlfile.replace('_bnr', '')))
        if not os.path.isdir(outname):
            os.makedirs(outname)
        # border noise removal is done outside of SNAP and the node is thus removed from the workflow
        del workflow['Remove-GRD-Border-Noise']
        # remove the node name from the groups
        i = 0
        while i < len(groups) - 1:
            if 'Remove-GRD-Border-Noise' in groups[i]:
                del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
            if len(groups[i]) == 0:
                del groups[i]
            else:
                i += 1
        # identify the input scene, unpack it and perform the custom border noise removal
        read = workflow['Read']
        scene = identify(read.parameters['file'])
        print('unpacking scene')
        scene.unpack(outname)
        print('removing border noise..')
        scene.removeGRDBorderNoise()
        # change the name of the input file to that of the unpacked archive
        read.parameters['file'] = scene.scene
        # write a new workflow file
        workflow.write(xmlfile)

    print('executing node sequence{}..'.format(
        's' if groups is not None else ''))
    if groups is not None:
        subs = split(xmlfile, groups)
        for sub in subs:
            execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
    else:
        execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions)

    if format == 'ENVI':
        print('converting to GTiff')
        translateoptions = {
            'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
            'format': 'GTiff'
        }
        for item in finder(outname, ['*.img'], recursive=False):
            if re.search('[HV]{2}', item):
                pol = re.search('[HV]{2}', item).group()
                name_new = outname.replace(suffix,
                                           '{0}_{1}.tif'.format(pol, suffix))
            else:
                base = os.path.splitext(os.path.basename(item))[0] \
                    .replace('elevation', 'DEM')
                name_new = outname.replace(suffix, '{0}.tif'.format(base))
            nodata = dem_nodata if re.search('elevation', item) else 0
            translateoptions['noData'] = nodata
            gdal_translate(item, name_new, translateoptions)
    # by default the nodata value is not registered in the GTiff metadata
    elif format == 'GeoTiff-BigTIFF':
        ras = gdal.Open(outname + '.tif', GA_Update)
        for i in range(1, ras.RasterCount + 1):
            ras.GetRasterBand(i).SetNoDataValue(0)
        ras = None
    if cleanup:
        shutil.rmtree(outname)
    print('done')
def gpt(xmlfile, groups=None, cleanup=True):
    """
    wrapper for ESA SNAP's Graph Processing Tool GPT.
    Input is a readily formatted workflow XML file as
    created by function :func:`~pyroSAR.snap.util.geocode`.
    Additional to calling GPT, this function will
    
     * execute the workflow in groups as defined by `groups`
     * encode a nodata value into the output file if the format is GeoTiff-BigTIFF
     * convert output files to GeoTiff if the output format is ENVI
    
    Parameters
    ----------
    xmlfile: str
        the name of the workflow XML file
    groups: list
        a list of lists each containing IDs for individual nodes
    cleanup: bool
        should all files written to the temporary directory during function execution be deleted after processing?
    
    Returns
    -------

    """
    
    workflow = Workflow(xmlfile)
    write = workflow['Write']
    outname = write.parameters['file']
    format = write.parameters['formatName']
    dem_name = workflow.tree.find('.//demName').text
    if dem_name == 'External DEM':
        dem_nodata = float(workflow.tree.find('.//externalDEMNoDataValue').text)
    else:
        dem_nodata = 0
    print('executing node sequence{}..'.format('s' if groups is not None else ''))
    if groups is not None:
        subs = split(xmlfile, groups)
        for sub in subs:
            execute(sub, cleanup=cleanup)
    else:
        execute(xmlfile, cleanup=cleanup)
    
    if format == 'ENVI':
        print('converting to GTiff')
        suffix = workflow.suffix
        translateoptions = {'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
                            'format': 'GTiff'}
        for item in finder(outname, ['*.img'], recursive=False):
            if re.search('[HV]{2}', item):
                pol = re.search('[HV]{2}', item).group()
                name_new = outname.replace(suffix, '{0}_{1}.tif'.format(pol, suffix))
            else:
                base = os.path.splitext(os.path.basename(item))[0] \
                    .replace('elevation', 'DEM')
                name_new = outname.replace(suffix, '{0}.tif'.format(base))
            nodata = dem_nodata if re.search('elevation', item) else 0
            translateoptions['noData'] = nodata
            gdal_translate(item, name_new, translateoptions)
        if cleanup:
            shutil.rmtree(outname)
    # by default the nodata value is not registered in the GTiff metadata
    elif format == 'GeoTiff-BigTIFF':
        ras = gdal.Open(outname + '.tif', GA_Update)
        for i in range(1, ras.RasterCount + 1):
            ras.GetRasterBand(i).SetNoDataValue(0)
        ras = None
    print('done')
Beispiel #4
0
def gpt(xmlfile,
        groups=None,
        cleanup=True,
        gpt_exceptions=None,
        removeS1BorderNoiseMethod='pyroSAR'):
    """
    wrapper for ESA SNAP's Graph Processing Tool GPT.
    Input is a readily formatted workflow XML file as
    created by function :func:`~pyroSAR.snap.util.geocode`.
    Additional to calling GPT, this function will
    
     * execute the workflow in groups as defined by `groups`
     * encode a nodata value into the output file if the format is GeoTiff-BigTIFF
     * convert output files to GeoTiff if the output format is ENVI
    
    Parameters
    ----------
    xmlfile: str
        the name of the workflow XML file
    groups: list
        a list of lists each containing IDs for individual nodes
    cleanup: bool
        should all files written to the temporary directory during function execution be deleted after processing?
    gpt_exceptions: dict
        a dictionary to override the configured GPT executable for certain operators;
        each (sub-)workflow containing this operator will be executed with the define executable;
        
         - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}``
    removeS1BorderNoiseMethod: str
        the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following:
         - 'ESA': the pure implementation as described by ESA
         - 'pyroSAR': the ESA method plus the custom pyroSAR refinement
    
    Returns
    -------

    """

    workflow = Workflow(xmlfile)
    write = workflow['Write']
    outname = write.parameters['file']
    suffix = workflow.suffix
    format = write.parameters['formatName']
    dem_name = workflow.tree.find('.//demName').text
    if dem_name == 'External DEM':
        dem_nodata = float(
            workflow.tree.find('.//externalDEMNoDataValue').text)
    else:
        dem_nodata = 0

    if 'Remove-GRD-Border-Noise' in workflow.ids and removeS1BorderNoiseMethod == 'pyroSAR':
        if 'SliceAssembly' in workflow.operators:
            raise RuntimeError(
                "pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs"
            )
        xmlfile = os.path.join(outname,
                               os.path.basename(xmlfile.replace('_bnr', '')))
        if not os.path.isdir(outname):
            os.makedirs(outname)
        # border noise removal is done outside of SNAP and the node is thus removed from the workflow
        del workflow['Remove-GRD-Border-Noise']
        # remove the node name from the groups
        i = 0
        while i < len(groups) - 1:
            if 'Remove-GRD-Border-Noise' in groups[i]:
                del groups[i][groups[i].index('Remove-GRD-Border-Noise')]
            if len(groups[i]) == 0:
                del groups[i]
            else:
                i += 1
        # identify the input scene, unpack it and perform the custom border noise removal
        read = workflow['Read']
        scene = identify(read.parameters['file'])
        print('unpacking scene')
        scene.unpack(outname)
        print('removing border noise..')
        scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod)
        # change the name of the input file to that of the unpacked archive
        read.parameters['file'] = scene.scene
        # write a new workflow file
        workflow.write(xmlfile)

    print('executing node sequence{}..'.format(
        's' if groups is not None else ''))
    if groups is not None:
        subs = split(xmlfile, groups)
        for sub in subs:
            execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions)
    else:
        execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions)

    if format == 'ENVI':
        print('converting to GTiff')
        translateoptions = {
            'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
            'format': 'GTiff'
        }
        for item in finder(outname, ['*.img'], recursive=False):
            if re.search('[HV]{2}', item):
                pol = re.search('[HV]{2}', item).group()
                name_new = outname.replace(suffix,
                                           '{0}_{1}.tif'.format(pol, suffix))
            else:
                base = os.path.splitext(os.path.basename(item))[0] \
                    .replace('elevation', 'DEM')
                name_new = outname.replace(suffix, '{0}.tif'.format(base))
            nodata = dem_nodata if re.search('elevation', item) else 0
            translateoptions['noData'] = nodata
            gdal_translate(item, name_new, translateoptions)
    # by default the nodata value is not registered in the GTiff metadata
    elif format == 'GeoTiff-BigTIFF':
        ras = gdal.Open(outname + '.tif', GA_Update)
        for i in range(1, ras.RasterCount + 1):
            ras.GetRasterBand(i).SetNoDataValue(0)
        ras = None
    if cleanup:
        shutil.rmtree(outname)
    ###########################################################################
    # write the Sentinel-1 manifest.safe file as addition to the actual product
    attrs = parse_datasetname(outname)
    ext = '' if attrs['extensions'] is None else attrs['extensions']
    readers = workflow['operator=Read']
    for reader in readers:
        infile = reader.parameters['file']
        try:
            id = identify(infile)
            if id.sensor in ['S1A', 'S1B']:
                manifest = id.getFileObj(id.findfiles('manifest.safe')[0])
                basename = '_'.join([id.outname_base() + ext, 'manifest.safe'])
                outdir = os.path.dirname(outname)
                outname_manifest = os.path.join(outdir, basename)
                with open(outname_manifest, 'wb') as out:
                    out.write(manifest.read())
        except RuntimeError:
            continue
    ###########################################################################
    print('done')
Beispiel #5
0
def gpt(xmlfile):
    """
    wrapper for ESA SNAP Graph Processing Tool GPT
    input is a readily formatted workflow xml file as created by function geocode in module snap.util
    """
    try:
        gpt_exec = ExamineSnap().gpt
    except AttributeError:
        raise RuntimeError('could not find SNAP GPT executable')

    with open(xmlfile, 'r') as infile:
        workflow = ET.fromstring(infile.read())
    write = workflow.find('.//node[@id="Write"]')
    outname = write.find('.//parameters/file').text
    outdir = os.path.dirname(outname)
    format = write.find('.//parameters/formatName').text
    infile = workflow.find('.//node[@id="Read"]/parameters/file').text

    if format == 'GeoTiff-BigTIFF':
        cmd = [
            gpt_exec,
            # '-Dsnap.dataio.reader.tileWidth=*',
            # '-Dsnap.dataio.reader.tileHeight=1',
            '-Dsnap.dataio.bigtiff.tiling.width=256',
            '-Dsnap.dataio.bigtiff.tiling.height=256',
            # '-Dsnap.dataio.bigtiff.compression.type=LZW',
            # '-Dsnap.dataio.bigtiff.compression.quality=0.75',
            xmlfile
        ]
    else:
        cmd = [gpt_exec, xmlfile]
    # print('- processing workflow {}'.format(os.path.basename(xmlfile)))
    proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
    out, err = proc.communicate()
    out = out.decode('utf-8') if isinstance(out, bytes) else out
    err = err.decode('utf-8') if isinstance(err, bytes) else err
    if proc.returncode != 0:
        if os.path.isfile(outname + '.tif'):
            os.remove(outname + '.tif')
        elif os.path.isdir(outname):
            shutil.rmtree(outname)
        print(out + err)
        print('failed: {}'.format(os.path.basename(infile)))
        err_match = re.search('Error: (.*)\n', out + err)
        errmessage = err_match.group(1) if err_match else err
        raise RuntimeError(errmessage)
    if format == 'ENVI':
        # print('- converting to GTiff')
        suffix = parse_suffix(workflow)
        translateoptions = {
            'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'],
            'format': 'GTiff',
            'noData': 0
        }
        for item in finder(outname, ['*.img']):
            pol = re.search('[HV]{2}', item).group()
            name_new = outname.replace(suffix,
                                       '{0}_{1}.tif'.format(pol, suffix))
            gdal_translate(item, name_new, translateoptions)
        shutil.rmtree(outname)
    elif format == 'GeoTiff-BigTIFF':
        ras = gdal.Open(outname + '.tif', GA_Update)
        for i in range(1, ras.RasterCount + 1):
            ras.GetRasterBand(i).SetNoDataValue(0)
        ras = None