def suffix(self, stop=None): """ Get the SNAP operator suffix sequence Parameters ---------- stop: str the ID of the last workflow node Returns ------- str a file suffix created from the order of which the nodes will be executed """ nodes = self.tree.findall('node') names = [re.sub(r'[ ]*\([0-9]+\)', '', y.attrib['id']) for y in nodes] names_unique = [] for name in names: if name not in names_unique: names_unique.append(name) if name == stop: break config = ExamineSnap() suffix = '_'.join( filter(None, [config.get_suffix(x) for x in names_unique])) return suffix
def suffix(self): """ Returns ------- str a file suffix created from the order of which the nodes will be executed """ nodes = self.tree.findall('node') names = [re.sub(r'[ ]*\([0-9]+\)', '', y.attrib['id']) for y in nodes] names_unique = [] for name in names: if name not in names_unique: names_unique.append(name) config = ExamineSnap() suffix = '_'.join( filter(None, [config.get_suffix(x) for x in names_unique])) return suffix
def __init__(self, geometries): if not isinstance(geometries, list): raise RuntimeError('geometries must be of type list') for geometry in geometries: if geometry.getProjection('epsg') != 4326: raise RuntimeError('input geometry CRS must be WGS84 LatLon (EPSG 4326)') self.geometries = geometries try: self.auxdatapath = ExamineSnap().auxdatapath except AttributeError: self.auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata')
def __init__(self, osvdir=None, timeout=20): self.timeout = timeout if osvdir is None: try: auxdatapath = ExamineSnap().auxdatapath except AttributeError: auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata') osvdir = os.path.join(auxdatapath, 'Orbits', 'Sentinel-1') self.outdir_poe = os.path.join(osvdir, 'POEORB') self.outdir_res = os.path.join(osvdir, 'RESORB') self.pattern = r'S1[AB]_OPER_AUX_(?:POE|RES)ORB_OPOD_[0-9TV_]{48}\.EOF' self.pattern_fine = r'(?P<sensor>S1[AB])_OPER_AUX_' \ r'(?P<type>(?:POE|RES)ORB)_OPOD_' \ r'(?P<publish>[0-9]{8}T[0-9]{6})_V' \ r'(?P<start>[0-9]{8}T[0-9]{6})_' \ r'(?P<stop>[0-9]{8}T[0-9]{6})\.EOF' self._init_dir() self._reorganize()
def get_egm96_lookup(): """ If not found, download SNAP's lookup table for converting EGM96 geoid heights to WGS84 ellipsoid heights Returns ------- """ try: auxdatapath = ExamineSnap().auxdatapath except AttributeError: auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata') local = os.path.join(auxdatapath, 'dem', 'egm96', 'ww15mgh_b.zip') os.makedirs(os.path.dirname(local), exist_ok=True) if not os.path.isfile(local): remote = 'http://step.esa.int/auxdata/dem/egm96/ww15mgh_b.zip' print('{} <<-- {}'.format(local, remote)) r = requests.get(remote) with open(local, 'wb') as out: out.write(r.content)
def test_installation(): reg = ExamineSnap() assert os.path.isfile(reg.gpt)
def parse_node(name, use_existing=True): """ parse an XML node recipe. The XML representation and parameter default values are read from the docstring of an individual node by calling `gpt <node> -h`. The result is then written to an XML text file under `$HOME/.pyroSAR/snap/nodes` which is subsequently read for parsing instead of again calling `gpt`. Parameters ---------- name: str the name of the processing node, e.g. Terrain-Correction use_existing: bool use an existing XML text file or force re-parsing the gpt docstring and overwriting the XML file? Returns ------- Node the parsed node Examples -------- >>> tnr = parse_node('ThermalNoiseRemoval') >>> print(tnr.parameters) {'selectedPolarisations': None, 'removeThermalNoise': 'true', 'reIntroduceThermalNoise': 'false'} """ name = name if name.endswith('.xml') else name + '.xml' operator = os.path.splitext(name)[0] abspath = os.path.join(os.path.expanduser('~'), '.pyrosar', 'snap', 'nodes') os.makedirs(abspath, exist_ok=True) absname = os.path.join(abspath, name) if not os.path.isfile(absname) or not use_existing: gpt = ExamineSnap().gpt cmd = [gpt, operator, '-h'] out, err = run(cmd=cmd, void=False) graph = re.search('<graph id.*', out, flags=re.DOTALL).group() graph = re.sub(r'>\${.*', '/>', graph) # remove placeholder values like ${value} graph = re.sub(r'<\.\.\./>.*', '', graph) # remove <.../> placeholders if operator == 'BandMaths': graph = graph.replace('sourceProducts', 'sourceProduct') tree = ET.fromstring(graph) for elt in tree.iter(): if elt.text in ['string', 'double', 'integer', 'float']: elt.text = None node = tree.find('node') node.attrib['id'] = operator # add a second source product entry for multi-source nodes # multi-source nodes are those with an entry 'sourceProducts' instead of 'sourceProduct' # exceptions are registered in this list: multisource = ['Back-Geocoding'] if operator != 'Read' and operator != 'ProductSet-Reader': source = node.find('.//sources') child = source[0] if child.tag == 'sourceProducts' or operator in multisource: child2 = ET.SubElement(source, 'sourceProduct.1', {'refid': 'Read (2)'}) child.tag = 'sourceProduct' child.attrib['refid'] = 'Read' child.text = None if operator == 'BandMaths': tree.find('.//parameters').set( 'class', 'com.bc.ceres.binding.dom.XppDomElement') tband = tree.find('.//targetBand') for item in [ 'spectralWavelength', 'spectralBandwidth', 'scalingOffset', 'scalingFactor', 'validExpression', 'spectralBandIndex' ]: el = tband.find('.//{}'.format(item)) tband.remove(el) node = Node(node) # read the default values from the parameter documentation parameters = node.parameters.keys() out += '-P' for parameter in parameters: p1 = r'-P{}.*?-P'.format(parameter) p2 = r"Default\ value\ is '([a-zA-Z0-9 ._\(\)]+)'" r1 = re.search(p1, out, re.S) if r1: sub = r1.group() r2 = re.search(p2, sub) if r2: value = r2.groups()[0] node.parameters[parameter] = value continue node.parameters[parameter] = None # fill in some additional defaults if operator == 'BandMerge': node.parameters['geographicError'] = '1.0E-5' with open(absname, 'w') as xml: xml.write(str(node)) return node else: with open(absname, 'r') as workflow: element = ET.fromstring(workflow.read()) return Node(element)
def execute(xmlfile, cleanup=True, gpt_exceptions=None, gpt_args=None, verbose=True): """ execute SNAP workflows via the Graph Processing Tool gpt. This function merely calls gpt with some additional command line arguments and raises a RuntimeError on fail. This function is used internally by function :func:`gpt`, which should be used instead. Parameters ---------- xmlfile: str the name of the workflow XML file cleanup: bool should all files written to the temporary directory during function execution be deleted after processing? gpt_exceptions: dict a dictionary to override the configured GPT executable for certain operators; each (sub-)workflow containing this operator will be executed with the define executable; - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}`` gpt_args: list or None a list of additional arguments to be passed to the gpt call - e.g. ``['-x', '-c', '2048M']`` for increased tile cache size and intermediate clearing verbose: bool print out status messages? Returns ------- Raises ------ RuntimeError """ # read the file and extract some information workflow = Workflow(xmlfile) write = workflow['Write'] outname = write.parameters['file'] workers = [x.id for x in workflow if x.operator not in ['Read', 'Write']] message = ' -> '.join(workers) gpt_exec = None if gpt_exceptions is not None: for item, exec in gpt_exceptions.items(): if item in workers: gpt_exec = exec message += ' (using {})'.format(exec) break if verbose: print(message) # try to find the GPT executable if gpt_exec is None: try: gpt_exec = ExamineSnap().gpt except AttributeError: raise RuntimeError('could not find SNAP GPT executable') # create the list of arguments to be passed to the subprocess module calling GPT cmd = [gpt_exec, '-e'] if isinstance(gpt_args, list): cmd.extend(gpt_args) if format == 'GeoTiff-BigTIFF': cmd.extend([ # '-Dsnap.dataio.reader.tileWidth=*', # '-Dsnap.dataio.reader.tileHeight=1', '-Dsnap.dataio.bigtiff.tiling.width=256', '-Dsnap.dataio.bigtiff.tiling.height=256', # '-Dsnap.dataio.bigtiff.compression.type=LZW', # '-Dsnap.dataio.bigtiff.compression.quality=0.75' ]) cmd.append(xmlfile) # execute the workflow proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) out, err = proc.communicate() out = out.decode('utf-8') if isinstance(out, bytes) else out err = err.decode('utf-8') if isinstance(err, bytes) else err # check for a message indicating an unknown parameter, # which can easily be removed from the workflow pattern = r"Error: \[NodeId: (?P<id>[a-zA-Z0-9-_]*)\] " \ r"Operator \'[a-zA-Z0-9-_]*\': " \ r"Unknown element \'(?P<par>[a-zA-Z]*)\'" match = re.search(pattern, err) if proc.returncode == 0: pattern = r'(?P<level>WARNING: )([a-zA-Z.]*: )(?P<message>No intersection.*)' match = re.search(pattern, err) if match is not None: raise RuntimeError(re.search(pattern, err).group('message')) return # delete unknown parameters and run the modified workflow elif proc.returncode == 1 and match is not None: replace = match.groupdict() with Workflow(xmlfile) as flow: print( ' removing parameter {id}:{par} and executing modified workflow' .format(**replace)) node = flow[replace['id']] del node.parameters[replace['par']] flow.write(xmlfile) execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions, gpt_args=gpt_args, verbose=verbose) # append additional information to the error message and raise an error else: if proc.returncode == -9: submessage = '[{}] the process was killed by SNAP (process return code -9). ' \ 'One possible cause is a lack of memory.'.format(os.path.basename(xmlfile)) else: submessage = '{}{}\n[{}] failed with return code {}' if cleanup: if os.path.isfile(outname + '.tif'): os.remove(outname + '.tif') elif os.path.isdir(outname): shutil.rmtree(outname, onerror=windows_fileprefix) elif outname.endswith('.dim'): os.remove(outname) shutil.rmtree(outname.replace('.dim', '.data'), onerror=windows_fileprefix) raise RuntimeError( submessage.format(out, err, os.path.basename(xmlfile), proc.returncode))
def parse_node(name, use_existing=True): """ parse an XML node recipe. The XML representation and parameter default values are read from the docstring of an individual node by calling `gpt <node> -h`. The result is then written to an XML text file under `$HOME/.pyroSAR/snap/nodes` which is subsequently read for parsing instead of again calling `gpt`. Parameters ---------- name: str the name of the processing node, e.g. Terrain-Correction use_existing: bool use an existing XML text file or force re-parsing the gpt docstring and overwriting the XML file? Returns ------- Node the parsed node Examples -------- >>> tnr = parse_node('ThermalNoiseRemoval') >>> print(tnr.parameters) {'selectedPolarisations': None, 'removeThermalNoise': 'true', 'reIntroduceThermalNoise': 'false'} """ name = name if name.endswith('.xml') else name + '.xml' operator = os.path.splitext(name)[0] abspath = os.path.join(os.path.expanduser('~'), '.pyrosar', 'snap', 'nodes') os.makedirs(abspath, exist_ok=True) absname = os.path.join(abspath, name) if not os.path.isfile(absname) or not use_existing: gpt = ExamineSnap().gpt cmd = [gpt, operator, '-h'] out, err = run(cmd=cmd, void=False) graph = re.search('<graph id.*', out, flags=re.DOTALL).group() tree = ET.fromstring(graph) node = tree.find('node') node.attrib['id'] = operator # add a second source product entry for multi-source nodes # multi-source nodes are those with an entry 'sourceProducts' instead of 'sourceProduct' # exceptions are registered in this list: multisource = ['Back-Geocoding'] if operator != 'Read': source = node.find('.//sources') child = source[0] if child.tag == 'sourceProducts' or operator in multisource: child2 = ET.SubElement(source, 'sourceProduct.1', {'refid': 'Read (2)'}) child.tag = 'sourceProduct' child.attrib['refid'] = 'Read' child.text = None node = Node(node) # read the default values from the parameter documentation parameters = node.parameters.keys() out += '-P' for parameter in parameters: p1 = r'-P{}.*?-P'.format(parameter) p2 = r"Default\ value\ is '([a-zA-Z0-9 ._\(\)]+)'" r1 = re.search(p1, out, re.S) if r1: sub = r1.group() r2 = re.search(p2, sub) if r2: value = r2.groups()[0] node.parameters[parameter] = value continue node.parameters[parameter] = None with open(absname, 'w') as xml: xml.write(str(node)) return node else: with open(absname, 'r') as workflow: element = ET.fromstring(workflow.read()) return Node(element)
def execute(xmlfile, cleanup=True, gpt_exceptions=None, verbose=True): """ execute SNAP workflows via the Graph Processing Tool gpt. This function merely calls gpt with some additional command line arguments and raises a RuntimeError on fail. This function is used internally by function :func:`gpt`, which should be used instead. Parameters ---------- xmlfile: str the name of the workflow XML file cleanup: bool should all files written to the temporary directory during function execution be deleted after processing? gpt_exceptions: dict a dictionary to override the configured GPT executable for certain operators; each (sub-)workflow containing this operator will be executed with the define executable; - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}`` verbose: bool print out status messages? Returns ------- Raises ------ RuntimeError """ # read the file and extract some information workflow = Workflow(xmlfile) write = workflow['Write'] outname = write.parameters['file'] infile = workflow['Read'].parameters['file'] nodes = workflow.nodes() workers = [x.id for x in nodes if x.operator not in ['Read', 'Write']] message = ' -> '.join(workers) gpt_exec = None if gpt_exceptions is not None: for item, exec in gpt_exceptions.items(): if item in workers: gpt_exec = exec message += ' (using {})'.format(exec) break if verbose: print(message) # try to find the GPT executable if gpt_exec is None: try: gpt_exec = ExamineSnap().gpt except AttributeError: raise RuntimeError('could not find SNAP GPT executable') # create the list of arguments to be passed to the subprocess module calling GPT if format == 'GeoTiff-BigTIFF': cmd = [ gpt_exec, # '-Dsnap.dataio.reader.tileWidth=*', # '-Dsnap.dataio.reader.tileHeight=1', '-Dsnap.dataio.bigtiff.tiling.width=256', '-Dsnap.dataio.bigtiff.tiling.height=256', # '-Dsnap.dataio.bigtiff.compression.type=LZW', # '-Dsnap.dataio.bigtiff.compression.quality=0.75', xmlfile ] else: cmd = [gpt_exec, xmlfile] # execute the workflow proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE) out, err = proc.communicate() out = out.decode('utf-8') if isinstance(out, bytes) else out err = err.decode('utf-8') if isinstance(err, bytes) else err # delete intermediate files if an error occurred if proc.returncode == 1: pattern = r"Error: \[NodeId: (?P<id>[a-zA-Z0-9-_]*)\] " \ r"Operator \'[a-zA-Z0-9-_]*\': " \ r"Unknown element \'(?P<par>[a-zA-Z]*)\'" match = re.search(pattern, err) if match is not None: replace = match.groupdict() with Workflow(xmlfile) as flow: print( ' removing parameter {id}:{par} and executing modified workflow' .format(**replace)) node = flow[replace['id']] del node.parameters[replace['par']] flow.write(xmlfile) execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions, verbose=verbose) else: if cleanup: if os.path.isfile(outname + '.tif'): os.remove(outname + '.tif') elif os.path.isdir(outname): shutil.rmtree(outname) print(out + err) print('failed: {}'.format(os.path.basename(infile))) err_match = re.search('Error: (.*)\n', out + err) errmessage = err_match.group(1) if err_match else err raise RuntimeError(errmessage) elif proc.returncode == -9: if cleanup: if os.path.isfile(outname + '.tif'): os.remove(outname + '.tif') elif os.path.isdir(outname): shutil.rmtree(outname) print( 'the process was killed by SNAP. One possible cause is a lack of memory.' ) else: print('process return code: {}'.format(proc.returncode))
def correctOSV(id, osvdir=None, osvType='POE', logpath=None, outdir=None, shellscript=None): """ correct GAMMA parameter files with orbit state vector information from dedicated OSV files; OSV files are downloaded automatically to either the defined `osvdir` or a sub-directory `osv` of the scene directory Parameters ---------- id: ~pyroSAR.drivers.ID the scene to be corrected osvdir: str the directory of OSV files; subdirectories POEORB and RESORB are created automatically osvType: {'POE', 'RES'} the OSV type to be used logpath: str or None a directory to write command logfiles to outdir: str or None the directory to execute the command in shellscript: str or None a file to write the Gamma commands to in shell format Returns ------- Examples -------- >>> from pyroSAR import identify >>> from pyroSAR.gamma import correctOSV, convert2gamma >>> filename = 'S1A_IW_GRDH_1SDV_20150222T170750_20150222T170815_004739_005DD8_3768.zip' # identify the SAR scene >>> scene = identify(filename) # unpack the zipped scene to an arbitrary directory >>> scene.unpack('/home/test') >>> print(scene.scene) /home/test/S1A_IW_GRDH_1SDV_20150222T170750_20150222T170815_004739_005DD8_3768.SAFE # convert the unpacked scene to GAMMA format >>> convert2gamma(id=scene, directory=scene.scene) # correct the OSV information of the converted GAMMA images >>> correctOSV(id=scene, osvdir='/home/test/osv') See Also -------- :meth:`pyroSAR.drivers.SAFE.getOSV` """ if not isinstance(id, ID): raise IOError('id must be of type pyroSAR.ID') if id.sensor not in ['S1A', 'S1B']: raise IOError( 'this method is currently only available for Sentinel-1. Please stay tuned...' ) if not os.path.isdir(logpath): os.makedirs(logpath) if osvdir is None: try: auxdatapath = ExamineSnap().auxdatapath except AttributeError: auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata') osvdir = os.path.join(auxdatapath, 'Orbits', 'Sentinel-1') try: id.getOSV(osvdir, osvType) except URLError: print('..no internet access') images = id.getGammaImages(id.scene) # read parameter file entries into object with ISPPar(images[0] + '.par') as par: # extract acquisition time stamp timestamp = datetime.strptime( par.date, '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y%m%dT%H%M%S') # find an OSV file matching the time stamp and defined OSV type(s) with OSV(osvdir) as osv: osvfile = osv.match(sensor=id.sensor, timestamp=timestamp, osvtype=osvType) if not osvfile: raise RuntimeError('no Orbit State Vector file found') if osvfile.endswith('.zip'): osvdir = os.path.join(id.scene, 'osv') with zf.ZipFile(osvfile) as zip: zip.extractall(path=osvdir) osvfile = os.path.join(osvdir, os.path.basename(osvfile).replace('.zip', '')) # update the GAMMA parameter file with the selected orbit state vectors print('correcting state vectors with file {}'.format(osvfile)) for image in images: isp.S1_OPOD_vec(SLC_par=image + '.par', OPOD=osvfile, logpath=logpath, outdir=outdir, shellscript=shellscript)