def test_run(tmpdir, testdata): log = os.path.join(str(tmpdir), 'test_run.log') out, err = anc.run(cmd=['gdalinfo', testdata['tif']], logfile=log, void=False) with pytest.raises(OSError): anc.run(['foobar']) with pytest.raises(sp.CalledProcessError): anc.run(['gdalinfo', 'foobar'])
def main(): # define a shapefile for the study area shp = '/.../testsite.shp' # define the output name of the DEM (no file extension like .tif etc.!) outname = '/path/to/demfile' # define a buffer around the study area boundaries (in degrees) buffer = 0.01 # load the defined shapefile with vector.Vector(shp) as site: # reproject the shapefile to latlon (in-memory, no file written or modified) site.reproject('+proj=longlat +datum=WGS84 +no_defs ') # extract the extent (bounding box) of the shapefile ext = site.extent # add the buffer to the bounding box ext['xmin'] -= buffer ext['ymin'] -= buffer ext['xmax'] += buffer ext['ymax'] += buffer # define a GDAL VRT file containing all SRTM tiles # this file has all hgt tiles in the same directory registered and is used for subsetting/mosaicing srtm_vrt = '/path/to/SRTM_1_HGT.vrt' # create a temporary directory for writing intermediate files (will be deleted at the end) tmpdir = outname + '__tmp' if not os.path.isdir(tmpdir): os.makedirs(tmpdir) # define a name for a temporary DEM file dem_tmp = os.path.join(tmpdir, 'srtm_tmp.tif') # create a DEM mosaic for the study site run([ 'gdalwarp', '-q', '-of', 'GTiff', '-te', ext['xmin'], ext['ymin'], ext['xmax'], ext['ymax'], srtm_vrt, dem_tmp ]) # transform the DEM to GAMMA format (including EGM96 geoid to WGS84 ellipsoid height reference correction) gamma.process(['srtm2dem', dem_tmp, outname, outname + '.par', 2, '-'], outdir=tmpdir) # create an ENVI header file par2hdr(outname + '.par', outname + '.hdr') # remove the temporary directory with all intermediate files shutil.rmtree(tmpdir) # optional: transform DEM to UTM projection # the UTM zone is automatically computed for the center of the DEM file dem.transform(outname, outname + '_utm', posting=20)
def process(cmd, outdir=None, logpath=None, inlist=None, void=True): """ wrapper function to execute GAMMA commands via module :mod:`subprocess` Parameters ---------- cmd: list the command line arguments outdir: str the directory to execute the command in logpath: str a directory to write logfiles to; the file will be named {GAMMA command}.log, e.g. gc_map.log inlist: list a list of values, which is passed as interactive inputs via stdin void: bool return the stdout and stderr messages? Returns ------- tuple of str or None the stdout and stderr messages if void is False, otherwise None """ log = os.path.join(logpath, cmd[0] + '.log') if logpath else None out, err = run(cmd, outdir=outdir, logfile=log, inlist=inlist, void=False, errorpass=True) gammaErrorHandler(out, err) if not void: return out, err
def parse_node(name, use_existing=True): """ parse an XML node recipe. The XML representation and parameter default values are read from the docstring of an individual node by calling `gpt <node> -h`. The result is then written to an XML text file under `$HOME/.pyroSAR/snap/nodes` which is subsequently read for parsing instead of again calling `gpt`. Parameters ---------- name: str the name of the processing node, e.g. Terrain-Correction use_existing: bool use an existing XML text file or force re-parsing the gpt docstring and overwriting the XML file? Returns ------- Node the parsed node Examples -------- >>> tnr = parse_node('ThermalNoiseRemoval') >>> print(tnr.parameters) {'selectedPolarisations': None, 'removeThermalNoise': 'true', 'reIntroduceThermalNoise': 'false'} """ name = name if name.endswith('.xml') else name + '.xml' operator = os.path.splitext(name)[0] abspath = os.path.join(os.path.expanduser('~'), '.pyrosar', 'snap', 'nodes') os.makedirs(abspath, exist_ok=True) absname = os.path.join(abspath, name) if not os.path.isfile(absname) or not use_existing: gpt = ExamineSnap().gpt cmd = [gpt, operator, '-h'] out, err = run(cmd=cmd, void=False) graph = re.search('<graph id.*', out, flags=re.DOTALL).group() graph = re.sub(r'>\${.*', '/>', graph) # remove placeholder values like ${value} graph = re.sub(r'<\.\.\./>.*', '', graph) # remove <.../> placeholders if operator == 'BandMaths': graph = graph.replace('sourceProducts', 'sourceProduct') tree = ET.fromstring(graph) for elt in tree.iter(): if elt.text in ['string', 'double', 'integer', 'float']: elt.text = None node = tree.find('node') node.attrib['id'] = operator # add a second source product entry for multi-source nodes # multi-source nodes are those with an entry 'sourceProducts' instead of 'sourceProduct' # exceptions are registered in this list: multisource = ['Back-Geocoding'] if operator != 'Read' and operator != 'ProductSet-Reader': source = node.find('.//sources') child = source[0] if child.tag == 'sourceProducts' or operator in multisource: child2 = ET.SubElement(source, 'sourceProduct.1', {'refid': 'Read (2)'}) child.tag = 'sourceProduct' child.attrib['refid'] = 'Read' child.text = None if operator == 'BandMaths': tree.find('.//parameters').set( 'class', 'com.bc.ceres.binding.dom.XppDomElement') tband = tree.find('.//targetBand') for item in [ 'spectralWavelength', 'spectralBandwidth', 'scalingOffset', 'scalingFactor', 'validExpression', 'spectralBandIndex' ]: el = tband.find('.//{}'.format(item)) tband.remove(el) node = Node(node) # read the default values from the parameter documentation parameters = node.parameters.keys() out += '-P' for parameter in parameters: p1 = r'-P{}.*?-P'.format(parameter) p2 = r"Default\ value\ is '([a-zA-Z0-9 ._\(\)]+)'" r1 = re.search(p1, out, re.S) if r1: sub = r1.group() r2 = re.search(p2, sub) if r2: value = r2.groups()[0] node.parameters[parameter] = value continue node.parameters[parameter] = None # fill in some additional defaults if operator == 'BandMerge': node.parameters['geographicError'] = '1.0E-5' with open(absname, 'w') as xml: xml.write(str(node)) return node else: with open(absname, 'r') as workflow: element = ET.fromstring(workflow.read()) return Node(element)
def parse_node(name, use_existing=True): """ parse an XML node recipe. The XML representation and parameter default values are read from the docstring of an individual node by calling `gpt <node> -h`. The result is then written to an XML text file under `$HOME/.pyroSAR/snap/nodes` which is subsequently read for parsing instead of again calling `gpt`. Parameters ---------- name: str the name of the processing node, e.g. Terrain-Correction use_existing: bool use an existing XML text file or force re-parsing the gpt docstring and overwriting the XML file? Returns ------- Node the parsed node Examples -------- >>> tnr = parse_node('ThermalNoiseRemoval') >>> print(tnr.parameters) {'selectedPolarisations': None, 'removeThermalNoise': 'true', 'reIntroduceThermalNoise': 'false'} """ name = name if name.endswith('.xml') else name + '.xml' operator = os.path.splitext(name)[0] abspath = os.path.join(os.path.expanduser('~'), '.pyrosar', 'snap', 'nodes') os.makedirs(abspath, exist_ok=True) absname = os.path.join(abspath, name) if not os.path.isfile(absname) or not use_existing: gpt = ExamineSnap().gpt cmd = [gpt, operator, '-h'] out, err = run(cmd=cmd, void=False) graph = re.search('<graph id.*', out, flags=re.DOTALL).group() tree = ET.fromstring(graph) node = tree.find('node') node.attrib['id'] = operator # add a second source product entry for multi-source nodes # multi-source nodes are those with an entry 'sourceProducts' instead of 'sourceProduct' # exceptions are registered in this list: multisource = ['Back-Geocoding'] if operator != 'Read': source = node.find('.//sources') child = source[0] if child.tag == 'sourceProducts' or operator in multisource: child2 = ET.SubElement(source, 'sourceProduct.1', {'refid': 'Read (2)'}) child.tag = 'sourceProduct' child.attrib['refid'] = 'Read' child.text = None node = Node(node) # read the default values from the parameter documentation parameters = node.parameters.keys() out += '-P' for parameter in parameters: p1 = r'-P{}.*?-P'.format(parameter) p2 = r"Default\ value\ is '([a-zA-Z0-9 ._\(\)]+)'" r1 = re.search(p1, out, re.S) if r1: sub = r1.group() r2 = re.search(p2, sub) if r2: value = r2.groups()[0] node.parameters[parameter] = value continue node.parameters[parameter] = None with open(absname, 'w') as xml: xml.write(str(node)) return node else: with open(absname, 'r') as workflow: element = ET.fromstring(workflow.read()) return Node(element)
def process(cmd, outdir=None, logfile=None, logpath=None, inlist=None, void=True, shellscript=None): """ wrapper function to execute GAMMA commands via module :mod:`subprocess` Parameters ---------- cmd: list the command line arguments outdir: str the directory to execute the command in logfile: str a file to write the command log to; overrides parameter logpath logpath: str a directory to write logfiles to; the file will be named {GAMMA command}.log, e.g. gc_map.log; is overridden by parameter logfile inlist: list a list of values, which is passed as interactive inputs via stdin void: bool return the stdout and stderr messages? shellscript: str a file to write the Gamma commands to in shell format Returns ------- tuple of str or None the stdout and stderr messages if void is False, otherwise None """ if logfile is not None: log = logfile else: log = os.path.join(logpath, os.path.basename(cmd[0]) + '.log') if logpath else None if shellscript is not None: if not os.path.isfile(shellscript): with open(shellscript, 'w') as init: pass line = ' '.join([str(x) for x in dissolve(cmd)]) if inlist is not None: line += ' <<< $"{}"'.format('\n'.join([str(x) for x in inlist]) + '\n') with open(shellscript, 'r+') as sh: if outdir is not None: content = sh.read() sh.seek(0) is_new = re.search( 'this script was created automatically by pyroSAR', content) is None if is_new: ts = datetime.now().strftime('%a %b %d %H:%M:%S %Y') sh.write( '# this script was created automatically by pyroSAR on {}\n\n' .format(ts)) sh.write('export base={}\n\n'.format(outdir)) sh.write(content) line = line.replace(outdir, '$base') sh.seek(0, 2) # set pointer to the end of the file sh.write(line + '\n\n') # create an environment containing the locations of all GAMMA submodules to be passed ot the subprocess calls gammaenv = {'GAMMA_HOME': ExamineGamma().home} for module in ['DIFF', 'DISP', 'IPTA', 'ISP', 'LAT']: loc = os.path.join(gammaenv['GAMMA_HOME'], module) if os.path.isdir(loc): gammaenv[module + '_HOME'] = loc # execute the command out, err = run(cmd, outdir=outdir, logfile=log, inlist=inlist, void=False, errorpass=True, env=gammaenv) gammaErrorHandler(out, err) if not void: return out, err
def process(cmd, outdir=None, logfile=None, logpath=None, inlist=None, void=True, shellscript=None): """ wrapper function to execute GAMMA commands via module :mod:`subprocess` Parameters ---------- cmd: list the command line arguments outdir: str the directory to execute the command in logfile: str a file to write the command log to; overrides parameter logpath logpath: str a directory to write logfiles to; the file will be named {GAMMA command}.log, e.g. gc_map.log; is overridden by parameter logfile inlist: list a list of values, which is passed as interactive inputs via stdin void: bool return the stdout and stderr messages? shellscript: str a file to write the Gamma commands to in shell format Returns ------- tuple of str or None the stdout and stderr messages if void is False, otherwise None """ if logfile is not None: log = logfile else: log = os.path.join(logpath, os.path.basename(cmd[0]) + '.log') if logpath else None if shellscript is not None: line = ' '.join([str(x) for x in dissolve(cmd)]) if inlist is not None: line += ' <<< $"{}"'.format('\n'.join([str(x) for x in inlist]) + '\n') with open(shellscript, 'a+') as sh: if outdir is not None: first = sh.read(1) if not first: ts = datetime.now().strftime('%a %b %d %H:%M:%S %Y') sh.write( '# this script was created automatically by pyroSAR on {}\n\n' .format(ts)) sh.write('export base={}\n\n'.format(outdir)) line = line.replace(outdir, '$base') sh.write(line + '\n\n') out, err = run(cmd, outdir=outdir, logfile=log, inlist=inlist, void=False, errorpass=True) gammaErrorHandler(out, err) if not void: return out, err