def getAuxdata(datasets, scenes): auxDataPath = os.path.join(expanduser("~"), '.snap/auxdata') scenes = [identify(scene) if isinstance(scene, str) else scene for scene in scenes] sensors = list(set([scene.sensor for scene in scenes])) for dataset in datasets: if dataset == 'SRTM 1Sec HGT': files = [x.replace('hgt', 'SRTMGL1.hgt.zip') for x in list(set(dissolve([scene.getHGT() for scene in scenes])))] for file in files: infile = os.path.join('http://step.esa.int/auxdata/dem/SRTMGL1', file) outfile = os.path.join(auxDataPath, 'dem/SRTM 1Sec HGT', file) if not os.path.isfile(outfile): print(infile) try: input = urlopen(infile) except HTTPError: print('-> not available') continue with open(outfile, 'wb') as output: output.write(input.read()) input.close() elif dataset == 'POEORB': for sensor in sensors: if re.search('S1[AB]', sensor): dates = [(scene.start[:4], scene.start[4:6]) for scene in scenes] years = list(set([x[0] for x in dates])) remote_contentVersion = urlopen( 'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/remote_contentVersion.txt') versions_remote = getOrbitContentVersions(remote_contentVersion) for year in years: dir_orb = os.path.join(auxDataPath, 'Orbits/Sentinel-1/POEORB', year) if not os.path.isdir(dir_orb): os.makedirs(dir_orb) contentVersionFile = os.path.join(dir_orb, 'contentVersion.txt') if os.path.isfile(contentVersionFile): contentVersion = open(contentVersionFile, 'r+') versions_local = getOrbitContentVersions(contentVersion) else: contentVersion = open(contentVersionFile, 'w') versions_local = {} combine = dict(set(versions_local.items()) & set(versions_remote.items())) dates_select = [x for x in dates if x[0] == year] months = list(set([x[1] for x in dates_select])) orb_ids = sorted( [x for x in ['{}-{}.zip'.format(year, month) for month in months] if not x in combine]) if len(orb_ids) > 0: contentVersion.write('#\n#{}\n'.format(strftime('%a %b %d %H:%M:%S %Z %Y', gmtime()))) for orb_id in orb_ids: orb_remote = urlopen( 'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/{}'.format(orb_id)) orb_remote_stream = zf.ZipFile(StringIO(orb_remote.read()), 'r') orb_remote.close() targets = [x for x in orb_remote_stream.namelist() if not os.path.isfile(os.path.join(dir_orb, x))] orb_remote_stream.extractall(dir_orb, targets) orb_remote_stream.close() versions_local[orb_id] = versions_remote[orb_id] for key, val in versions_local.iteritems(): contentVersion.write('{}={}\n'.format(key, val)) contentVersion.close() remote_contentVersion.close() else: print('not implemented yet') elif dataset == 'Delft Precise Orbits': path_server = 'dutlru2.lr.tudelft.nl' subdirs = {'ASAR:': 'ODR.ENVISAT1/eigen-cg03c', 'ERS1': 'ODR.ERS-1/dgm-e04', 'ERS2': 'ODR.ERS-2/dgm-e04'} ftp = FTP(path_server) ftp.login() for sensor in sensors: if sensor in subdirs.keys(): path_target = os.path.join('pub/orbits', subdirs[sensor]) path_local = os.path.join(auxDataPath, 'Orbits/Delft Precise Orbits', subdirs[sensor]) ftp.cwd(path_target) for item in ftp.nlst(): ftp.retrbinary('RETR ' + item, open(os.path.join(path_local, item), 'wb').write) ftp.quit() else: print('not implemented yet')
def test_identify_fail(testdir, testdata): with pytest.raises(OSError): pyroSAR.identify(os.path.join(testdir, 'foobar')) with pytest.raises(RuntimeError): pyroSAR.identify(testdata['tif'])
def test_filter_processed(tmpdir, testdata): scene = pyroSAR.identify(testdata['s1']) assert len(pyroSAR.filter_processed([scene], str(tmpdir))) == 1
def gpt(xmlfile, groups=None, cleanup=True, gpt_exceptions=None): """ wrapper for ESA SNAP's Graph Processing Tool GPT. Input is a readily formatted workflow XML file as created by function :func:`~pyroSAR.snap.util.geocode`. Additional to calling GPT, this function will * execute the workflow in groups as defined by `groups` * encode a nodata value into the output file if the format is GeoTiff-BigTIFF * convert output files to GeoTiff if the output format is ENVI Parameters ---------- xmlfile: str the name of the workflow XML file groups: list a list of lists each containing IDs for individual nodes cleanup: bool should all files written to the temporary directory during function execution be deleted after processing? gpt_exceptions: dict a dictionary to override the configured GPT executable for certain operators; each (sub-)workflow containing this operator will be executed with the define executable; - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}`` Returns ------- """ workflow = Workflow(xmlfile) write = workflow['Write'] outname = write.parameters['file'] suffix = workflow.suffix format = write.parameters['formatName'] dem_name = workflow.tree.find('.//demName').text if dem_name == 'External DEM': dem_nodata = float( workflow.tree.find('.//externalDEMNoDataValue').text) else: dem_nodata = 0 if 'Remove-GRD-Border-Noise' in workflow.ids: xmlfile = os.path.join(outname, os.path.basename(xmlfile.replace('_bnr', ''))) if not os.path.isdir(outname): os.makedirs(outname) # border noise removal is done outside of SNAP and the node is thus removed from the workflow del workflow['Remove-GRD-Border-Noise'] # remove the node name from the groups i = 0 while i < len(groups) - 1: if 'Remove-GRD-Border-Noise' in groups[i]: del groups[i][groups[i].index('Remove-GRD-Border-Noise')] if len(groups[i]) == 0: del groups[i] else: i += 1 # identify the input scene, unpack it and perform the custom border noise removal read = workflow['Read'] scene = identify(read.parameters['file']) print('unpacking scene') scene.unpack(outname) print('removing border noise..') scene.removeGRDBorderNoise() # change the name of the input file to that of the unpacked archive read.parameters['file'] = scene.scene # write a new workflow file workflow.write(xmlfile) print('executing node sequence{}..'.format( 's' if groups is not None else '')) if groups is not None: subs = split(xmlfile, groups) for sub in subs: execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions) else: execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions) if format == 'ENVI': print('converting to GTiff') translateoptions = { 'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'], 'format': 'GTiff' } for item in finder(outname, ['*.img'], recursive=False): if re.search('[HV]{2}', item): pol = re.search('[HV]{2}', item).group() name_new = outname.replace(suffix, '{0}_{1}.tif'.format(pol, suffix)) else: base = os.path.splitext(os.path.basename(item))[0] \ .replace('elevation', 'DEM') name_new = outname.replace(suffix, '{0}.tif'.format(base)) nodata = dem_nodata if re.search('elevation', item) else 0 translateoptions['noData'] = nodata gdal_translate(item, name_new, translateoptions) # by default the nodata value is not registered in the GTiff metadata elif format == 'GeoTiff-BigTIFF': ras = gdal.Open(outname + '.tif', GA_Update) for i in range(1, ras.RasterCount + 1): ras.GetRasterBand(i).SetNoDataValue(0) ras = None if cleanup: shutil.rmtree(outname) print('done')
'polarizations': ['HH', 'HV'], 'product': '1.5', 'samples': 12870, 'sensor': 'PSR2', 'spacing': (6.25, 6.25), 'start': '20140909T043342', 'stop': '20140909T043352' } } return cases @pytest.fixture def scene(testcases, testdata, request): case = testcases[request.param] case['pyro'] = pyroSAR.identify(testdata[request.param]) return case class Test_Metadata(): @pytest.mark.parametrize('scene', ['s1', 'psr2'], indirect=True) def test_attributes(self, scene): assert scene['pyro'].acquisition_mode == scene['acquisition_mode'] assert scene['pyro'].compression == scene['compression'] assert scene['pyro'].getCorners() == scene['corners'] assert scene['pyro'].lines == scene['lines'] assert scene['pyro'].outname_base() == scene['outname'] assert scene['pyro'].orbit == scene['orbit'] assert scene['pyro'].polarizations == scene['polarizations'] assert scene['pyro'].product == scene['product'] assert scene['pyro'].samples == scene['samples']
def split(xmlfile, groups): """ split a workflow file into groups and write them to separate workflows including source and write target linking. The new workflows are written to a sub-directory `temp` of the target directory defined in the input's `Write` node. Each new workflow is parameterized with a `Read` and `Write` node if they don't already exist. Temporary outputs are written to `BEAM-DIMAP` files named after the workflow suffix sequence. Parameters ---------- xmlfile: str the workflow to be split groups: list a list of lists each containing IDs for individual nodes Returns ------- list of str the names of the newly written temporary workflows Raises ------ RuntimeError """ workflow = Workflow(xmlfile) write = workflow['Write'] out = write.parameters['file'] tmp = os.path.join(out, 'temp') if not os.path.isdir(tmp): os.makedirs(tmp) # the temporary XML files outlist = [] # the names and format of temporary products prod_tmp = {} prod_tmp_format = {} for position, group in enumerate(groups): node_lookup = {} log.debug('creating new workflow for group {}'.format(group)) new = parse_recipe('blank') nodes = [workflow[x] for x in group] for node in nodes: id_old = node.id sources = node.source if sources is None: sources = [] resetSuccessorSource = False elif isinstance(sources, list): resetSuccessorSource = False else: resetSuccessorSource = True sources = [sources] reset = [] for source in sources: if source not in group: read = new.insert_node( parse_node('Read'), void=False, resetSuccessorSource=resetSuccessorSource) reset.append(read.id) read.parameters['file'] = prod_tmp[source] read.parameters['formatName'] = prod_tmp_format[source] node_lookup[read.id] = source else: reset.append(source) if isinstance(sources, list): sources_new_pos = [ list(node_lookup.values()).index(x) for x in sources ] sources_new = [ list(node_lookup.keys())[x] for x in sources_new_pos ] newnode = new.insert_node(node.copy(), before=sources_new, void=False, resetSuccessorSource=False) else: newnode = new.insert_node(node.copy(), void=False, resetSuccessorSource=False) node_lookup[newnode.id] = id_old if not resetSuccessorSource: newnode.source = reset # if possible, read the name of the SAR product for parsing names of temporary files # this was found necessary for SliceAssembly, which expects the names in a specific format products = [x.parameters['file'] for x in new['operator=Read']] try: id = identify(products[0]) filename = os.path.basename(id.scene) except (RuntimeError, OSError): filename = os.path.basename(products[0]) basename = os.path.splitext(filename)[0] basename = re.sub(r'_tmp[0-9]+', '', basename) # add a Write node to all dangling nodes counter = 0 for node in new: dependants = [ x for x in workflow.successors(node.id) if not x.startswith('Write') and not x in group ] if node.operator != 'Read' and len(dependants) > 0: write = parse_node('Write') new.insert_node(write, before=node.id, resetSuccessorSource=False) id = str(position) if counter == 0 else '{}-{}'.format( position, counter) tmp_out = os.path.join(tmp, '{}_tmp{}.dim'.format(basename, id)) prod_tmp[node_lookup[node.id]] = tmp_out prod_tmp_format[node_lookup[node.id]] = 'BEAM-DIMAP' write.parameters['file'] = tmp_out write.parameters['formatName'] = 'BEAM-DIMAP' counter += 1 if not is_consistent(new): message = 'inconsistent group:\n {}'.format(' -> '.join(group)) raise RuntimeError(message) outname = os.path.join(tmp, '{}_tmp{}.xml'.format(basename, position)) new.write(outname) outlist.append(outname) return outlist
def geocode(infile, outdir, t_srs=4326, tr=20, polarizations='all', shapefile=None, scaling='dB', geocoding_type='Range-Doppler', removeS1BorderNoise=True, removeS1BorderNoiseMethod='pyroSAR', removeS1ThermalNoise=True, offset=None, allow_RES_OSV=False, externalDEMFile=None, externalDEMNoDataValue=None, externalDEMApplyEGM=True, terrainFlattening=True, basename_extensions=None, test=False, export_extra=None, groupsize=1, cleanup=True, gpt_exceptions=None, gpt_args=None, returnWF=False, nodataValueAtSea=True, demResamplingMethod='BILINEAR_INTERPOLATION', imgResamplingMethod='BILINEAR_INTERPOLATION', speckleFilter=False, refarea='gamma0'): """ wrapper function for geocoding SAR images using ESA SNAP Parameters ---------- infile: str or ~pyroSAR.drivers.ID or list the SAR scene(s) to be processed; multiple scenes are treated as consecutive acquisitions, which will be mosaicked with SNAP's SliceAssembly operator outdir: str The directory to write the final files to. t_srs: int, str or osr.SpatialReference A target geographic reference system in WKT, EPSG, PROJ4 or OPENGIS format. See function :func:`spatialist.auxil.crsConvert()` for details. Default: `4326 <http://spatialreference.org/ref/epsg/4326/>`_. tr: int or float, optional The target resolution in meters. Default is 20 polarizations: list or str The polarizations to be processed; can be a string for a single polarization, e.g. 'VV', or a list of several polarizations, e.g. ['VV', 'VH']. With the special value 'all' (default) all available polarizations are processed. shapefile: str or :py:class:`~spatialist.vector.Vector`, optional A vector geometry for subsetting the SAR scene to a test site. Default is None. scaling: {'dB', 'db', 'linear'}, optional Should the output be in linear or decibel scaling? Default is 'dB'. geocoding_type: {'Range-Doppler', 'SAR simulation cross correlation'}, optional The type of geocoding applied; can be either 'Range-Doppler' (default) or 'SAR simulation cross correlation' removeS1BorderNoise: bool, optional Enables removal of S1 GRD border noise (default). removeS1BorderNoiseMethod: str the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following: - 'ESA': the pure implementation as described by ESA - 'pyroSAR': the ESA method plus the custom pyroSAR refinement removeS1ThermalNoise: bool, optional Enables removal of S1 thermal noise (default). offset: tuple, optional A tuple defining offsets for left, right, top and bottom in pixels, e.g. (100, 100, 0, 0); this variable is overridden if a shapefile is defined. Default is None. allow_RES_OSV: bool (only applies to Sentinel-1) Also allow the less accurate RES orbit files to be used? The function first tries to download a POE file for the scene. If this fails and RES files are allowed, it will download the RES file. The selected OSV type is written to the workflow XML file. Processing is aborted if the correction fails (Apply-Orbit-File parameter continueOnFail set to false). externalDEMFile: str or None, optional The absolute path to an external DEM file. Default is None. externalDEMNoDataValue: int, float or None, optional The no data value of the external DEM. If not specified (default) the function will try to read it from the specified external DEM. externalDEMApplyEGM: bool, optional Apply Earth Gravitational Model to external DEM? Default is True. terainFlattening: bool apply topographic normalization on the data? basename_extensions: list of str names of additional parameters to append to the basename, e.g. ['orbitNumber_rel'] test: bool, optional If set to True the workflow xml file is only written and not executed. Default is False. export_extra: list or None a list of image file IDs to be exported to outdir. The following IDs are currently supported: - incidenceAngleFromEllipsoid - localIncidenceAngle - projectedLocalIncidenceAngle - DEM groupsize: int the number of workers executed together in one gpt call cleanup: bool should all files written to the temporary directory during function execution be deleted after processing? gpt_exceptions: dict a dictionary to override the configured GPT executable for certain operators; each (sub-)workflow containing this operator will be executed with the define executable; - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}`` gpt_args: list or None a list of additional arguments to be passed to the gpt call - e.g. ``['-x', '-c', '2048M']`` for increased tile cache size and intermediate clearing returnWF: bool return the full name of the written workflow XML file? nodataValueAtSea: bool mask pixels acquired over sea? The sea mask depends on the selected DEM. demResamplingMethod: str one of the following: - 'NEAREST_NEIGHBOUR' - 'BILINEAR_INTERPOLATION' - 'CUBIC_CONVOLUTION' - 'BISINC_5_POINT_INTERPOLATION' - 'BISINC_11_POINT_INTERPOLATION' - 'BISINC_21_POINT_INTERPOLATION' - 'BICUBIC_INTERPOLATION' imgResamplingMethod: str the resampling method for geocoding the SAR image; the options are identical to demResamplingMethod speckleFilter: str one of the following: - 'Boxcar' - 'Median' - 'Frost' - 'Gamma Map' - 'Refined Lee' - 'Lee' - 'Lee Sigma' refarea: str one of the following: - 'beta0' - 'gamma0' - 'sigma0' Returns ------- str or None either the name of the workflow file if `returnWF == True` or None otherwise Note ---- If only one polarization is selected and not extra products are defined the results are directly written to GeoTiff. Otherwise the results are first written to a folder containing ENVI files and then transformed to GeoTiff files (one for each polarization/extra product). If GeoTiff would directly be selected as output format for multiple polarizations then a multilayer GeoTiff is written by SNAP which is considered an unfavorable format .. figure:: figures/snap_geocode.png :scale: 25% :align: center Workflow diagram for function geocode for processing a Sentinel-1 Ground Range Detected (GRD) scene to radiometrically terrain corrected (RTC) backscatter. An additional Subset node might be inserted in case a vector geometry is provided. Examples -------- geocode a Sentinel-1 scene and export the local incidence angle map with it >>> from pyroSAR.snap import geocode >>> filename = 'S1A_IW_GRDH_1SDV_20180829T170656_20180829T170721_023464_028DE0_F7BD.zip' >>> geocode(infile=filename, outdir='outdir', tr=20, scaling='dB', >>> export_extra=['DEM', 'localIncidenceAngle'], t_srs=4326) See Also -------- :class:`pyroSAR.drivers.ID`, :class:`spatialist.vector.Vector`, :func:`spatialist.auxil.crsConvert()` """ if isinstance(infile, pyroSAR.ID): id = infile elif isinstance(infile, str): id = pyroSAR.identify(infile) elif isinstance(infile, list): ids = pyroSAR.identify_many(infile, verbose=False, sortkey='start') id = ids[0] else: raise TypeError("'infile' must be of type str, list or pyroSAR.ID") if id.is_processed(outdir): print('scene {} already processed'.format(id.outname_base())) return # print(os.path.basename(id.scene)) if not os.path.isdir(outdir): os.makedirs(outdir) ############################################ # general setup if id.sensor in ['ASAR', 'ERS1', 'ERS2']: formatName = 'ENVISAT' elif id.sensor in ['S1A', 'S1B']: id.getOSV() if id.product == 'SLC': raise RuntimeError('Sentinel-1 SLC data is not supported yet') formatName = 'SENTINEL-1' else: raise RuntimeError('sensor not supported (yet)') ###################### # print('- assessing polarization selection') if isinstance(polarizations, str): if polarizations == 'all': polarizations = id.polarizations else: if polarizations in id.polarizations: polarizations = [polarizations] else: raise RuntimeError('polarization {} does not exists in the source product'.format(polarizations)) elif isinstance(polarizations, list): polarizations = [x for x in polarizations if x in id.polarizations] else: raise RuntimeError('polarizations must be of type str or list') format = 'GeoTiff-BigTIFF' if len(polarizations) == 1 and export_extra is None else 'ENVI' # print(polarizations) # print(format) bandnames = dict() bandnames['int'] = ['Intensity_' + x for x in polarizations] bandnames['beta0'] = ['Beta0_' + x for x in polarizations] bandnames['gamma0'] = ['Gamma0_' + x for x in polarizations] bandnames['sigma0'] = ['Sigma0_' + x for x in polarizations] ############################################ ############################################ # parse base workflow # print('- parsing base workflow') workflow = parse_recipe('base') ############################################ # Read node configuration # print('-- configuring Read Node') read = workflow['Read'] read.parameters['file'] = id.scene read.parameters['formatName'] = formatName readers = [read.id] if isinstance(infile, list): for i in range(1, len(infile)): readn = parse_node('Read') readn.parameters['file'] = ids[i].scene readn.parameters['formatName'] = formatName workflow.insert_node(readn, before=read.id, resetSuccessorSource=False) readers.append(readn.id) sliceAssembly = parse_node('SliceAssembly') sliceAssembly.parameters['selectedPolarisations'] = polarizations workflow.insert_node(sliceAssembly, before=readers) read = sliceAssembly ############################################ # Remove-GRD-Border-Noise node configuration # print('-- configuring Remove-GRD-Border-Noise Node') if id.sensor in ['S1A', 'S1B'] and removeS1BorderNoise: bn = parse_node('Remove-GRD-Border-Noise') workflow.insert_node(bn, before=read.id) bn.parameters['selectedPolarisations'] = polarizations ############################################ # ThermalNoiseRemoval node configuration # print('-- configuring ThermalNoiseRemoval Node') if id.sensor in ['S1A', 'S1B'] and removeS1ThermalNoise: for reader in readers: tn = parse_node('ThermalNoiseRemoval') workflow.insert_node(tn, before=reader) tn.parameters['selectedPolarisations'] = polarizations ############################################ # orbit file application node configuration # print('-- configuring Apply-Orbit-File Node') orbit_lookup = {'ENVISAT': 'DELFT Precise (ENVISAT, ERS1&2) (Auto Download)', 'SENTINEL-1': 'Sentinel Precise (Auto Download)'} orbitType = orbit_lookup[formatName] if formatName == 'ENVISAT' and id.acquisition_mode == 'WSM': orbitType = 'DORIS Precise VOR (ENVISAT) (Auto Download)' if formatName == 'SENTINEL-1': match = id.getOSV(osvType='POE', returnMatch=True) if match is None and allow_RES_OSV: id.getOSV(osvType='RES') orbitType = 'Sentinel Restituted (Auto Download)' orb = workflow['Apply-Orbit-File'] orb.parameters['orbitType'] = orbitType orb.parameters['continueOnFail'] = False ############################################ # calibration node configuration # print('-- configuring Calibration Node') cal = workflow['Calibration'] cal.parameters['selectedPolarisations'] = polarizations cal.parameters['sourceBands'] = bandnames['int'] if terrainFlattening: if refarea != 'gamma0': raise RuntimeError('if terrain flattening is applied refarea must be gamma0') cal.parameters['outputBetaBand'] = True else: refarea_options = ['sigma0', 'beta0', 'gamma0'] if refarea not in refarea_options: message = '{0} must be one of the following:\n- {1}' raise ValueError(message.format('refarea', '\n- '.join(refarea_options))) cal.parameters['output{}Band'.format(refarea[:-1].capitalize())] = True last = cal.id ############################################ # terrain flattening node configuration # print('-- configuring Terrain-Flattening Node') if terrainFlattening: tf = parse_node('Terrain-Flattening') workflow.insert_node(tf, before=last) if id.sensor in ['ERS1', 'ERS2'] or (id.sensor == 'ASAR' and id.acquisition_mode != 'APP'): tf.parameters['sourceBands'] = 'Beta0' else: tf.parameters['sourceBands'] = bandnames['beta0'] if externalDEMFile is None: tf.parameters['reGridMethod'] = True else: tf.parameters['reGridMethod'] = False last = tf.id ############################################ # speckle filtering node configuration speckleFilter_options = ['Boxcar', 'Median', 'Frost', 'Gamma Map', 'Refined Lee', 'Lee', 'Lee Sigma'] if speckleFilter: message = '{0} must be one of the following:\n- {1}' if speckleFilter not in speckleFilter_options: raise ValueError(message.format('speckleFilter', '\n- '.join(speckleFilter_options))) sf = parse_node('Speckle-Filter') workflow.insert_node(sf, before=last) sf.parameters['sourceBands'] = bandnames[refarea] sf.parameters['filter'] = speckleFilter last = sf.id ############################################ # configuration of node sequence for specific geocoding approaches # print('-- configuring geocoding approach Nodes') if geocoding_type == 'Range-Doppler': tc = parse_node('Terrain-Correction') workflow.insert_node(tc, before=last) tc.parameters['sourceBands'] = bandnames[refarea] elif geocoding_type == 'SAR simulation cross correlation': sarsim = parse_node('SAR-Simulation') workflow.insert_node(sarsim, before=last) sarsim.parameters['sourceBands'] = bandnames[refarea] workflow.insert_node(parse_node('Cross-Correlation'), before='SAR-Simulation') tc = parse_node('SARSim-Terrain-Correction') workflow.insert_node(tc, before='Cross-Correlation') else: raise RuntimeError('geocode_type not recognized') ############################################ # Multilook node configuration try: image_geometry = id.meta['image_geometry'] incidence = id.meta['incidence'] except KeyError: raise RuntimeError('This function does not yet support sensor {}'.format(id.sensor)) rlks, azlks = multilook_factors(sp_rg=id.spacing[0], sp_az=id.spacing[1], tr_rg=tr, tr_az=tr, geometry=image_geometry, incidence=incidence) if azlks > 1 or rlks > 1: workflow.insert_node(parse_node('Multilook'), before='Calibration') ml = workflow['Multilook'] ml.parameters['nAzLooks'] = azlks ml.parameters['nRgLooks'] = rlks ml.parameters['sourceBands'] = None # if cal.parameters['outputBetaBand']: # ml.parameters['sourceBands'] = bandnames['beta0'] # elif cal.parameters['outputGammaBand']: # ml.parameters['sourceBands'] = bandnames['gamma0'] # elif cal.parameters['outputSigmaBand']: # ml.parameters['sourceBands'] = bandnames['sigma0'] ############################################ # specify spatial resolution and coordinate reference system of the output dataset # print('-- configuring CRS') tc.parameters['pixelSpacingInMeter'] = tr try: t_srs = crsConvert(t_srs, 'epsg') except TypeError: raise RuntimeError("format of parameter 't_srs' not recognized") # the EPSG code 4326 is not supported by SNAP and thus the WKT string has to be defined; # in all other cases defining EPSG:{code} will do if t_srs == 4326: t_srs = 'GEOGCS["WGS84(DD)",' \ 'DATUM["WGS84",' \ 'SPHEROID["WGS84", 6378137.0, 298.257223563]],' \ 'PRIMEM["Greenwich", 0.0],' \ 'UNIT["degree", 0.017453292519943295],' \ 'AXIS["Geodetic longitude", EAST],' \ 'AXIS["Geodetic latitude", NORTH]]' else: t_srs = 'EPSG:{}'.format(t_srs) tc.parameters['mapProjection'] = t_srs ############################################ # (optionally) add node for conversion from linear to db scaling # print('-- configuring LinearToFromdB Node') if scaling not in ['dB', 'db', 'linear']: raise RuntimeError('scaling must be a string of either "dB", "db" or "linear"') if scaling in ['dB', 'db']: lin2db = parse_node('lin2db') workflow.insert_node(lin2db, before=tc.id) lin2db.parameters['sourceBands'] = bandnames[refarea] ############################################ # (optionally) add subset node and add bounding box coordinates of defined shapefile # print('-- configuring Subset Node') if shapefile: # print('--- read') shp = shapefile.clone() if isinstance(shapefile, Vector) else Vector(shapefile) # reproject the geometry to WGS 84 latlon # print('--- reproject') shp.reproject(4326) ext = shp.extent shp.close() # add an extra buffer of 0.01 degrees buffer = 0.01 ext['xmin'] -= buffer ext['ymin'] -= buffer ext['xmax'] += buffer ext['ymax'] += buffer # print('--- create bbox') with bbox(ext, 4326) as bounds: # print('--- intersect') inter = intersect(id.bbox(), bounds) if not inter: raise RuntimeError('no bounding box intersection between shapefile and scene') # print('--- close intersect') inter.close() # print('--- get wkt') wkt = bounds.convert2wkt()[0] # print('--- parse XML node') subset = parse_node('Subset') # print('--- insert node') workflow.insert_node(subset, before=read.id) subset.parameters['region'] = [0, 0, id.samples, id.lines] subset.parameters['geoRegion'] = wkt ############################################ # (optionally) configure subset node for pixel offsets if offset and not shapefile: subset = parse_node('Subset') workflow.insert_node(subset, before=read.id) # left, right, top and bottom offset in pixels l, r, t, b = offset subset_values = [l, t, id.samples - l - r, id.lines - t - b] subset.parameters['region'] = subset_values subset.parameters['geoRegion'] = '' ############################################ # parametrize write node # print('-- configuring Write Node') # create a suffix for the output file to identify processing steps performed in the workflow suffix = workflow.suffix basename = os.path.join(outdir, id.outname_base(basename_extensions)) extension = suffix if format == 'ENVI' else polarizations[0] + '_' + suffix outname = basename + '_' + extension write = workflow['Write'] write.parameters['file'] = outname write.parameters['formatName'] = format ############################################ ############################################ if export_extra is not None: options = ['incidenceAngleFromEllipsoid', 'localIncidenceAngle', 'projectedLocalIncidenceAngle', 'DEM'] write = parse_node('Write') workflow.insert_node(write, before=tc.id, resetSuccessorSource=False) write.parameters['file'] = outname write.parameters['formatName'] = format for item in export_extra: if item not in options: raise RuntimeError("ID '{}' not valid for argument 'export_extra'".format(item)) key = 'save{}{}'.format(item[0].upper(), item[1:]) tc.parameters[key] = True ############################################ ############################################ # select DEM type # print('-- configuring DEM') dempar = {'externalDEMFile': externalDEMFile, 'externalDEMApplyEGM': externalDEMApplyEGM} if externalDEMFile is not None: if os.path.isfile(externalDEMFile): if externalDEMNoDataValue is None: with Raster(externalDEMFile) as dem: dempar['externalDEMNoDataValue'] = dem.nodata if dempar['externalDEMNoDataValue'] is None: raise RuntimeError('Cannot read NoData value from DEM file. ' 'Please specify externalDEMNoDataValue') else: dempar['externalDEMNoDataValue'] = externalDEMNoDataValue dempar['reGridMethod'] = False else: raise RuntimeError('specified externalDEMFile does not exist') dempar['demName'] = 'External DEM' else: # SRTM 1arcsec is only available between -58 and +60 degrees. # If the scene exceeds those latitudes SRTM 3arcsec is selected. if id.getCorners()['ymax'] > 60 or id.getCorners()['ymin'] < -58: dempar['demName'] = 'SRTM 3Sec' else: dempar['demName'] = 'SRTM 1Sec HGT' dempar['externalDEMFile'] = None dempar['externalDEMNoDataValue'] = 0 for key, value in dempar.items(): workflow.set_par(key, value) # download the EGM lookup table if necessary if dempar['externalDEMApplyEGM']: get_egm96_lookup() ############################################ ############################################ # configure the resampling methods options = ['NEAREST_NEIGHBOUR', 'BILINEAR_INTERPOLATION', 'CUBIC_CONVOLUTION', 'BISINC_5_POINT_INTERPOLATION', 'BISINC_11_POINT_INTERPOLATION', 'BISINC_21_POINT_INTERPOLATION', 'BICUBIC_INTERPOLATION'] message = '{0} must be one of the following:\n- {1}' if demResamplingMethod not in options: raise ValueError(message.format('demResamplingMethod', '\n- '.join(options))) if imgResamplingMethod not in options: raise ValueError(message.format('imgResamplingMethod', '\n- '.join(options))) workflow.set_par('demResamplingMethod', demResamplingMethod) workflow.set_par('imgResamplingMethod', imgResamplingMethod) ############################################ ############################################ # additional parameter settings applied to the whole workflow workflow.set_par('nodataValueAtSea', nodataValueAtSea) ############################################ ############################################ # write workflow to file and optionally execute it # print('- writing workflow to file') workflow.write(outname + '_proc') # execute the newly written workflow if not test: try: groups = groupbyWorkers(outname + '_proc.xml', groupsize) gpt(outname + '_proc.xml', groups=groups, cleanup=cleanup, gpt_exceptions=gpt_exceptions, gpt_args=gpt_args, removeS1BorderNoiseMethod=removeS1BorderNoiseMethod) except RuntimeError as e: print(str(e)) with open(outname + '_error.log', 'w') as log: log.write(str(e)) if returnWF: return outname + '_proc.xml'
def test_filter_processed(): scene = pyroSAR.identify(testfile1) outdir = 'pyroSAR/tests' assert len(pyroSAR.filter_processed([scene], outdir)) == 1
def gpt(xmlfile, groups=None, cleanup=True, gpt_exceptions=None, gpt_args=None, removeS1BorderNoiseMethod='pyroSAR', basename_extensions=None): """ wrapper for ESA SNAP's Graph Processing Tool GPT. Input is a readily formatted workflow XML file as created by function :func:`~pyroSAR.snap.util.geocode`. Additional to calling GPT, this function will * execute the workflow in groups as defined by `groups` * encode a nodata value into the output file if the format is GeoTiff-BigTIFF * convert output files to GeoTiff if the output format is ENVI Parameters ---------- xmlfile: str the name of the workflow XML file groups: list a list of lists each containing IDs for individual nodes cleanup: bool should all files written to the temporary directory during function execution be deleted after processing? gpt_exceptions: dict a dictionary to override the configured GPT executable for certain operators; each (sub-)workflow containing this operator will be executed with the define executable; - e.g. ``{'Terrain-Flattening': '/home/user/snap/bin/gpt'}`` gpt_args: list or None a list of additional arguments to be passed to the gpt call - e.g. ``['-x', '-c', '2048M']`` for increased tile cache size and intermediate clearing removeS1BorderNoiseMethod: str the border noise removal method to be applied, See :func:`pyroSAR.S1.removeGRDBorderNoise` for details; one of the following: - 'ESA': the pure implementation as described by ESA - 'pyroSAR': the ESA method plus the custom pyroSAR refinement basename_extensions: list of str names of additional parameters to append to the basename, e.g. ['orbitNumber_rel'] Returns ------- Raises ------ RuntimeError """ workflow = Workflow(xmlfile) read = workflow['Read'] write = workflow['Write'] scene = identify(read.parameters['file']) outname = write.parameters['file'] suffix = workflow.suffix format = write.parameters['formatName'] dem_name = workflow.tree.find('.//demName') if dem_name is not None: if dem_name.text == 'External DEM': dem_nodata = float( workflow.tree.find('.//externalDEMNoDataValue').text) else: dem_nodata = 0 if 'Remove-GRD-Border-Noise' in workflow.ids \ and removeS1BorderNoiseMethod == 'pyroSAR' \ and scene.meta['IPF_version'] < 2.9: if 'SliceAssembly' in workflow.operators: raise RuntimeError( "pyroSAR's custom border noise removal is not yet implemented for multiple scene inputs" ) xmlfile = os.path.join(outname, os.path.basename(xmlfile.replace('_bnr', ''))) os.makedirs(outname, exist_ok=True) # border noise removal is done outside of SNAP and the node is thus removed from the workflow del workflow['Remove-GRD-Border-Noise'] # remove the node name from the groups i = 0 while i < len(groups) - 1: if 'Remove-GRD-Border-Noise' in groups[i]: del groups[i][groups[i].index('Remove-GRD-Border-Noise')] if len(groups[i]) == 0: del groups[i] else: i += 1 # unpack the scene if necessary and perform the custom border noise removal print('unpacking scene') if scene.compression is not None: scene.unpack(outname) print('removing border noise..') scene.removeGRDBorderNoise(method=removeS1BorderNoiseMethod) # change the name of the input file to that of the unpacked archive read.parameters['file'] = scene.scene # write a new workflow file workflow.write(xmlfile) print('executing node sequence{}..'.format( 's' if groups is not None else '')) try: if groups is not None: subs = split(xmlfile, groups) for sub in subs: execute(sub, cleanup=cleanup, gpt_exceptions=gpt_exceptions, gpt_args=gpt_args) else: execute(xmlfile, cleanup=cleanup, gpt_exceptions=gpt_exceptions, gpt_args=gpt_args) except RuntimeError as e: if cleanup and os.path.exists(outname): shutil.rmtree(outname, onerror=windows_fileprefix) raise RuntimeError(str(e) + '\nfailed: {}'.format(xmlfile)) if format == 'ENVI': print('converting to GTiff') translateoptions = { 'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'], 'format': 'GTiff' } for item in finder(outname, ['*.img'], recursive=False): if re.search('[HV]{2}', item): pol = re.search('[HV]{2}', item).group() name_new = outname.replace(suffix, '{0}_{1}.tif'.format(pol, suffix)) else: base = os.path.splitext(os.path.basename(item))[0] \ .replace('elevation', 'DEM') name_new = outname.replace(suffix, '{0}.tif'.format(base)) nodata = dem_nodata if re.search('elevation', item) else 0 translateoptions['noData'] = nodata gdal_translate(item, name_new, translateoptions) # by default the nodata value is not registered in the GTiff metadata elif format == 'GeoTiff-BigTIFF': ras = gdal.Open(outname + '.tif', GA_Update) for i in range(1, ras.RasterCount + 1): ras.GetRasterBand(i).SetNoDataValue(0) ras = None ########################################################################### # write the Sentinel-1 manifest.safe file as addition to the actual product readers = workflow['operator=Read'] for reader in readers: infile = reader.parameters['file'] try: id = identify(infile) if id.sensor in ['S1A', 'S1B']: manifest = id.getFileObj(id.findfiles('manifest.safe')[0]) basename = id.outname_base(basename_extensions) basename = '{0}_{1}_manifest.safe'.format(basename, suffix) outdir = os.path.dirname(outname) outname_manifest = os.path.join(outdir, basename) with open(outname_manifest, 'wb') as out: out.write(manifest.read()) except RuntimeError: continue ########################################################################### if cleanup and os.path.exists(outname): shutil.rmtree(outname, onerror=windows_fileprefix) print('done')
def test_infile_type(self, tmpdir, testdata): scene = testdata['s1'] with pytest.raises(TypeError): geocode(infile=123, outdir=str(tmpdir), test=True) id = identify(scene) geocode(infile=id, outdir=str(tmpdir), test=True)
def geocode(infile, outdir, t_srs=4326, tr=20, polarizations='all', shapefile=None, scaling='dB', geocoding_type='Range-Doppler', removeS1BoderNoise=True, offset=None, externalDEMFile=None, externalDEMNoDataValue=None, externalDEMApplyEGM=True, test=False): """ wrapper function for geocoding SAR images using ESA SNAP Parameters ---------- infile: str or ~pyroSAR.drivers.ID the SAR scene to be processed outdir: str The directory to write the final files to. t_srs: int, str or osr.SpatialReference A target geographic reference system in WKT, EPSG, PROJ4 or OPENGIS format. See function :func:`spatialist.auxil.crsConvert()` for details. Default: `4326 <http://spatialreference.org/ref/epsg/4326/>`_. tr: int or float, optional The target resolution in meters. Default is 20 polarizations: list or {'VV', 'HH', 'VH', 'HV', 'all'}, optional The polarizations to be processed; can be a string for a single polarization e.g. 'VV' or a list of several polarizations e.g. ['VV', 'VH']. Default is 'all'. shapefile: str or :py:class:`~spatialist.vector.Vector`, optional A vector geometry for subsetting the SAR scene to a test site. Default is None. scaling: {'dB', 'db', 'linear'}, optional Should the output be in linear or decibel scaling? Default is 'dB'. geocoding_type: {'Range-Doppler', 'SAR simulation cross correlation'}, optional The type of geocoding applied; can be either 'Range-Doppler' (default) or 'SAR simulation cross correlation' removeS1BoderNoise: bool, optional Enables removal of S1 GRD border noise (default). offset: tuple, optional A tuple defining offsets for left, right, top and bottom in pixels, e.g. (100, 100, 0, 0); this variable is overridden if a shapefile is defined. Default is None. externalDEMFile: str or None, optional The absolute path to an external DEM file. Default is None. externalDEMNoDataValue: int, float or None, optional The no data value of the external DEM. If not specified (default) the function will try to read it from the specified external DEM. externalDEMApplyEGM: bool, optional Apply Earth Gravitational Model to external DEM? Default is True. test: bool, optional If set to True the workflow xml file is only written and not executed. Default is False. Note ---- If only one polarization is selected the results are directly written to GeoTiff. Otherwise the results are first written to a folder containing ENVI files and then transformed to GeoTiff files (one for each polarization). If GeoTiff would directly be selected as output format for multiple polarizations then a multilayer GeoTiff is written by SNAP which is considered an unfavorable format See Also -------- :class:`pyroSAR.drivers.ID`, :class:`spatialist.vector.Vector`, :func:`spatialist.auxil.crsConvert()` """ id = infile if isinstance(infile, pyroSAR.ID) else pyroSAR.identify(infile) if id.is_processed(outdir): print('scene {} already processed'.format(id.outname_base())) return ############################################ # general setup if id.sensor in ['ASAR', 'ERS1', 'ERS2']: formatName = 'ENVISAT' elif id.sensor in ['S1A', 'S1B']: if id.product == 'SLC': raise RuntimeError('Sentinel-1 SLC data is not supported yet') formatName = 'SENTINEL-1' else: raise RuntimeError('sensor not supported (yet)') if polarizations == 'all': polarizations = id.polarizations else: polarizations = [x for x in polarizations if x in id.polarizations] print("Polarizations:", polarizations) format = 'GeoTiff-BigTIFF' if len(polarizations) == 1 else 'ENVI' ############################################ # parse base workflow workflow = parse_recipe('geocode') ############################################ # Read node configuration read = workflow.find('.//node[@id="Read"]') read.find('.//parameters/file').text = id.scene read.find('.//parameters/formatName').text = formatName ############################################ # Remove-GRD-Border-Noise node configuration if id.sensor in ['S1A', 'S1B'] and removeS1BoderNoise: insert_node(workflow, 'Read', parse_node('Remove-GRD-Border-Noise')) bn = workflow.find('.//node[@id="Remove-GRD-Border-Noise"]') bn.find('.//parameters/selectedPolarisations').text = ','.join(polarizations) ############################################ # orbit file application node configuration orbit_lookup = {'ENVISAT': 'DELFT Precise (ENVISAT, ERS1&2) (Auto Download)', 'SENTINEL-1': 'Sentinel Precise (Auto Download)'} orbitType = orbit_lookup[formatName] if formatName == 'ENVISAT' and id.acquisition_mode == 'WSM': orbitType = 'DORIS Precise VOR (ENVISAT) (Auto Download)' orb = workflow.find('.//node[@id="Apply-Orbit-File"]') orb.find('.//parameters/orbitType').text = orbitType ############################################ # calibration node configuration cal = workflow.find('.//node[@id="Calibration"]') cal.find('.//parameters/selectedPolarisations').text = ','.join(polarizations) cal.find('.//parameters/sourceBands').text = ','.join(['Intensity_' + x for x in polarizations]) ############################################ # terrain flattening node configuration tf = workflow.find('.//node[@id="Terrain-Flattening"]') if id.sensor in ['ERS1', 'ERS2'] or (id.sensor == 'ASAR' and id.acquisition_mode != 'APP'): tf.find('.//parameters/sourceBands').text = 'Beta0' else: tf.find('.//parameters/sourceBands').text = ','.join(['Beta0_' + x for x in polarizations]) ############################################ # configuration of node sequence for specific geocoding approaches if geocoding_type == 'Range-Doppler': insert_node(workflow, 'Terrain-Flattening', parse_node('Terrain-Correction')) tc = workflow.find('.//node[@id="Terrain-Correction"]') tc.find('.//parameters/sourceBands').text = ','.join(['Gamma0_' + x for x in polarizations]) elif geocoding_type == 'SAR simulation cross correlation': insert_node(workflow, 'Terrain-Flattening', parse_node('SAR-Simulation')) insert_node(workflow, 'SAR-Simulation', parse_node('Cross-Correlation')) insert_node(workflow, 'Cross-Correlation', parse_node('SARSim-Terrain-Correction')) tc = workflow.find('.//node[@id="SARSim-Terrain-Correction"]') sarsim = workflow.find('.//node[@id="SAR-Simulation"]') sarsim.find('.//parameters/sourceBands').text = ','.join(['Gamma0_' + x for x in polarizations]) else: raise RuntimeError('geocode_type not recognized') tc.find('.//parameters/pixelSpacingInMeter').text = str(tr) try: t_srs = spatial.crsConvert(t_srs, 'epsg') except TypeError: raise RuntimeError("format of parameter 't_srs' not recognized") if t_srs == 4326: t_srs = 'GEOGCS["WGS84(DD)",' \ 'DATUM["WGS84",' \ 'SPHEROID["WGS84", 6378137.0, 298.257223563]],' \ 'PRIMEM["Greenwich", 0.0],' \ 'UNIT["degree", 0.017453292519943295],' \ 'AXIS["Geodetic longitude", EAST],' \ 'AXIS["Geodetic latitude", NORTH]]' else: t_srs = 'EPSG:{}'.format(t_srs) tc.find('.//parameters/mapProjection').text = t_srs ############################################ # add node for conversion from linear to db scaling if scaling not in ['dB', 'db', 'linear']: raise RuntimeError('scaling must be a string of either "dB", "db" or "linear"') if scaling in ['dB', 'db']: lin2db = parse_node('lin2db') sourceNode = 'Terrain-Correction' if geocoding_type == 'Range-Doppler' else 'SARSim-Terrain-Correction' insert_node(workflow, sourceNode, lin2db) lin2db = workflow.find('.//node[@id="LinearToFromdB"]') lin2db.find('.//parameters/sourceBands').text = ','.join(['Gamma0_' + x for x in polarizations]) ############################################ # add subset node and add bounding box coordinates of defined shapefile if shapefile: shp = shapefile if isinstance(shapefile, spatial.vector.Vector) else spatial.vector.Vector(shapefile) bounds = spatial.bbox(shp.extent, shp.wkt) bounds.reproject(id.projection) intersect = spatial.intersect(id.bbox(), bounds) if not intersect: raise RuntimeError('no bounding box intersection between shapefile and scene') wkt = bounds.convert2wkt()[0] subset = parse_node('Subset') insert_node(workflow, 'Read', subset) subset = workflow.find('.//node[@id="Subset"]') subset.find('.//parameters/region').text = ','.join(map(str, [0, 0, id.samples, id.lines])) subset.find('.//parameters/geoRegion').text = wkt ############################################ # configure subset node for pixel offsets if offset and not shapefile: subset = parse_node('Subset') insert_node(workflow, 'Read', subset) # left, right, top and bottom offset in pixels l, r, t, b = offset subset = workflow.find('.//node[@id="Subset"]') subset.find('.//parameters/region').text = ','.join(map(str, [l, t, id.samples - l - r, id.lines - t - b])) subset.find('.//parameters/geoRegion').text = '' ############################################ # parametrize write node # create a suffix for the output file to identify processing steps performed in the workflow suffix = parse_suffix(workflow) if format == 'ENVI': outname = os.path.join(outdir, id.outname_base() + '_' + suffix) else: outname = os.path.join(outdir, '{}_{}_{}'.format(id.outname_base(), polarizations[0], suffix)) write = workflow.find('.//node[@id="Write"]') write.find('.//parameters/file').text = outname write.find('.//parameters/formatName').text = format ############################################ ############################################ # select DEM type if externalDEMFile is not None: if os.path.isfile(externalDEMFile): if externalDEMNoDataValue is None: dem = spatial.raster.Raster(externalDEMFile) if dem.nodata is not None: externalDEMNoDataValue = dem.nodata else: raise RuntimeError('Cannot read NoData value from DEM file. Please specify externalDEMNoDataValue') else: raise RuntimeError('specified externalDEMFile does not exist') demname = 'External DEM' else: # SRTM 1arcsec is only available between -58 and +60 degrees. # If the scene exceeds those latitudes SRTM 3arcsec is selected. if id.getCorners()['ymax'] > 60 or id.getCorners()['ymin'] < -58: demname = 'SRTM 3Sec' else: demname = 'SRTM 1Sec HGT' externalDEMFile = None externalDEMNoDataValue = 0 for demName in workflow.findall('.//parameters/demName'): demName.text = demname for externalDEM in workflow.findall('.//parameters/externalDEMFile'): externalDEM.text = externalDEMFile for demNodata in workflow.findall('.//parameters/externalDEMNoDataValue'): demNodata.text = str(externalDEMNoDataValue) for egm in workflow.findall('.//parameters/externalDEMApplyEGM'): egm.text = str(externalDEMApplyEGM).lower() ############################################ ############################################ # write workflow to file write_recipe(workflow, outname + '_proc') # execute the newly written workflow if not test: try: gpt(outname + '_proc.xml') except RuntimeError: os.remove(outname + '_proc.xml')
def test_identify_fail(): with pytest.raises(IOError): pyroSAR.identify(os.path.join(testdir, 'foobar'))
def main(s1_archive, out_path, t_srs='utm', tr=10, polarization='all', bbox=None, shapefile=None, scaling='db', geocoding_type='Range-Doppler', remove_s1_border_noise=True, remove_s1_thermal_noise=False, offset_left=None, offset_right=None, offset_top=None, offset_bottom=None, external_dem_file=None, external_dem_no_data_value=None, external_dem_apply_egm=True, terrain_flattening=True, basename_extension=None, test=False, export_extra=None, group_size=2, cleanup=True, return_wf=False): # identify the scene from input archive print("Identifying S1 scene from input archive.") scene = pyroSAR.identify(s1_archive) print("Success") # deal with offsets offset = (offset_left, offset_right, offset_top, offset_bottom) if set(offset) == set([None]): offset = None if t_srs.lower() == 'utm': # get the bbox scene_extent = scene.bbox().extent scene_bbox = box(scene_extent['xmin'], scene_extent['ymin'], scene_extent['xmax'], scene_extent['ymax']) # get the lat and long from the aoi centroid long = scene_bbox.centroid.x lat = scene_bbox.centroid.y # use the longitude to get the utm zone (3rd element in returned list c = utm.from_latlon(lat, long) utm_zone = c[2] # build the epsg code if lat >= 0.0: t_srs = int("326%s" % (str(utm_zone))) else: t_srs = int("327%s" % (str(utm_zone))) else: # try to convert to an integer -- if it doesn't work, leave as is try: t_srs = int(t_srs) except ValueError: pass # deal with basename extensions if basename_extension is None: basename_extensions = None else: basename_extensions = [basename_extension] # deal with export_extra if export_extra is not None: export_extra = list(map(str, export_extra.split(','))) # make the output directory if it doesn't exist if os.path.exists(out_path) is False: os.mkdir(out_path) # if bbox is provided, convert to format expected by pyroSAR if bbox is not None: bbox = map(float, bbox.split(',')) shapefile = spatialist.bbox( dict(zip(['xmin', 'ymin', 'xmax', 'ymax'], bbox)), crs=spatialist.auxil.crsConvert(4326, 'epsg')) print("Geocoding image according to input options.") geocode(infile=scene, outdir=out_path, t_srs=t_srs, tr=tr, polarizations=[polarization], shapefile=shapefile, scaling=scaling, geocoding_type=geocoding_type, removeS1BoderNoise=remove_s1_border_noise, removeS1ThermalNoise=remove_s1_thermal_noise, offset=offset, externalDEMFile=external_dem_file, externalDEMNoDataValue=external_dem_no_data_value, externalDEMApplyEGM=external_dem_apply_egm, terrainFlattening=terrain_flattening, basename_extensions=basename_extensions, test=test, export_extra=export_extra, groupsize=group_size, cleanup=cleanup, returnWF=return_wf) print("Processing completed successfully.")
def geocode(infile, outdir, t_srs=4326, tr=20, polarizations='all', shapefile=None, scaling='dB', geocoding_type='Range-Doppler', removeS1BoderNoise=True, removeS1ThermalNoise=True, offset=None, externalDEMFile=None, externalDEMNoDataValue=None, externalDEMApplyEGM=True, basename_extensions=None, test=False, export_extra=None, groupsize=2, cleanup=True): """ wrapper function for geocoding SAR images using ESA SNAP Parameters ---------- infile: str or ~pyroSAR.drivers.ID the SAR scene to be processed outdir: str The directory to write the final files to. t_srs: int, str or osr.SpatialReference A target geographic reference system in WKT, EPSG, PROJ4 or OPENGIS format. See function :func:`spatialist.auxil.crsConvert()` for details. Default: `4326 <http://spatialreference.org/ref/epsg/4326/>`_. tr: int or float, optional The target resolution in meters. Default is 20 polarizations: list or {'VV', 'HH', 'VH', 'HV', 'all'}, optional The polarizations to be processed; can be a string for a single polarization e.g. 'VV' or a list of several polarizations e.g. ['VV', 'VH']. Default is 'all'. shapefile: str or :py:class:`~spatialist.vector.Vector`, optional A vector geometry for subsetting the SAR scene to a test site. Default is None. scaling: {'dB', 'db', 'linear'}, optional Should the output be in linear or decibel scaling? Default is 'dB'. geocoding_type: {'Range-Doppler', 'SAR simulation cross correlation'}, optional The type of geocoding applied; can be either 'Range-Doppler' (default) or 'SAR simulation cross correlation' removeS1BoderNoise: bool, optional Enables removal of S1 GRD border noise (default). removeS1ThermalNoise: bool, optional Enables removal of S1 thermal noise (default). offset: tuple, optional A tuple defining offsets for left, right, top and bottom in pixels, e.g. (100, 100, 0, 0); this variable is overridden if a shapefile is defined. Default is None. externalDEMFile: str or None, optional The absolute path to an external DEM file. Default is None. externalDEMNoDataValue: int, float or None, optional The no data value of the external DEM. If not specified (default) the function will try to read it from the specified external DEM. externalDEMApplyEGM: bool, optional Apply Earth Gravitational Model to external DEM? Default is True. basename_extensions: list of str names of additional parameters to append to the basename, e.g. ['orbitNumber_rel'] test: bool, optional If set to True the workflow xml file is only written and not executed. Default is False. export_extra: list or None a list of image file IDs to be exported to outdir. The following IDs are currently supported: * incidenceAngleFromEllipsoid * localIncidenceAngle * projectedLocalIncidenceAngle * DEM groupsize: int the number of workers executed together in one gpt call cleanup: bool should all files written to the temporary directory during function execution be deleted after processing? Note ---- If only one polarization is selected and not extra products are defined the results are directly written to GeoTiff. Otherwise the results are first written to a folder containing ENVI files and then transformed to GeoTiff files (one for each polarization/extra product). If GeoTiff would directly be selected as output format for multiple polarizations then a multilayer GeoTiff is written by SNAP which is considered an unfavorable format Examples -------- geocode a Sentinel-1 scene and export the local incidence angle map with it >>> from pyroSAR.snap import geocode >>> filename = 'S1A_IW_GRDH_1SDV_20180829T170656_20180829T170721_023464_028DE0_F7BD.zip' >>> geocode(infile=filename, outdir='outdir', tr=20, scaling='dB', >>> export_extra=['DEM', 'localIncidenceAngle'], t_srs=4326) See Also -------- :class:`pyroSAR.drivers.ID`, :class:`spatialist.vector.Vector`, :func:`spatialist.auxil.crsConvert()` """ id = infile if isinstance(infile, pyroSAR.ID) else pyroSAR.identify(infile) if id.is_processed(outdir): print('scene {} already processed'.format(id.outname_base())) return # print(os.path.basename(id.scene)) if not os.path.isdir(outdir): os.makedirs(outdir) ############################################ # general setup if id.sensor in ['ASAR', 'ERS1', 'ERS2']: formatName = 'ENVISAT' elif id.sensor in ['S1A', 'S1B']: if id.product == 'SLC': raise RuntimeError('Sentinel-1 SLC data is not supported yet') formatName = 'SENTINEL-1' else: raise RuntimeError('sensor not supported (yet)') ###################### # print('- assessing polarization selection') if isinstance(polarizations, str): if polarizations == 'all': polarizations = id.polarizations else: if polarizations in id.polarizations: polarizations = [polarizations] else: raise RuntimeError( 'polarization {} does not exists in the source product'. format(polarizations)) elif isinstance(polarizations, list): polarizations = [x for x in polarizations if x in id.polarizations] else: raise RuntimeError('polarizations must be of type str or list') format = 'GeoTiff-BigTIFF' if len( polarizations) == 1 and export_extra is None else 'ENVI' # print(polarizations) # print(format) bands_int = ['Intensity_' + x for x in polarizations] bands_beta = ['Beta0_' + x for x in polarizations] bands_gamma = ['Gamma0_' + x for x in polarizations] ############################################ ############################################ # parse base workflow # print('- parsing base workflow') workflow = parse_recipe('geocode') ############################################ # Read node configuration # print('-- configuring Read Node') read = workflow['Read'] read.parameters['file'] = id.scene read.parameters['formatName'] = formatName ############################################ # Remove-GRD-Border-Noise node configuration # print('-- configuring Remove-GRD-Border-Noise Node') if id.sensor in ['S1A', 'S1B'] and removeS1BoderNoise: bn = parse_node('Remove-GRD-Border-Noise') workflow.insert_node(bn, before='Read') bn.parameters['selectedPolarisations'] = polarizations ############################################ # ThermalNoiseRemoval node configuration # print('-- configuring ThermalNoiseRemoval Node') if id.sensor in ['S1A', 'S1B'] and removeS1ThermalNoise: tn = parse_node('ThermalNoiseRemoval') workflow.insert_node(tn, before='Read') tn.parameters['selectedPolarisations'] = polarizations ############################################ # orbit file application node configuration # print('-- configuring Apply-Orbit-File Node') orbit_lookup = { 'ENVISAT': 'DELFT Precise (ENVISAT, ERS1&2) (Auto Download)', 'SENTINEL-1': 'Sentinel Precise (Auto Download)' } orbitType = orbit_lookup[formatName] if formatName == 'ENVISAT' and id.acquisition_mode == 'WSM': orbitType = 'DORIS Precise VOR (ENVISAT) (Auto Download)' orb = workflow['Apply-Orbit-File'] orb.parameters['orbitType'] = orbitType ############################################ # calibration node configuration # print('-- configuring Calibration Node') cal = workflow['Calibration'] cal.parameters['selectedPolarisations'] = polarizations cal.parameters['sourceBands'] = bands_int ############################################ # terrain flattening node configuration # print('-- configuring Terrain-Flattening Node') tf = workflow['Terrain-Flattening'] if id.sensor in ['ERS1', 'ERS2'] or (id.sensor == 'ASAR' and id.acquisition_mode != 'APP'): tf.parameters['sourceBands'] = 'Beta0' else: tf.parameters['sourceBands'] = bands_beta ############################################ # configuration of node sequence for specific geocoding approaches # print('-- configuring geocoding approach Nodes') if geocoding_type == 'Range-Doppler': tc = parse_node('Terrain-Correction') workflow.insert_node(tc, before='Terrain-Flattening') tc.parameters['sourceBands'] = bands_gamma elif geocoding_type == 'SAR simulation cross correlation': sarsim = parse_node('SAR-Simulation') workflow.insert_node(sarsim, before='Terrain-Flattening') sarsim.parameters['sourceBands'] = bands_gamma workflow.insert_node(parse_node('Cross-Correlation'), before='SAR-Simulation') tc = parse_node('SARSim-Terrain-Correction') workflow.insert_node(tc, before='Cross-Correlation') else: raise RuntimeError('geocode_type not recognized') ############################################ # Multilook node configuration try: image_geometry = id.meta['image_geometry'] incidence = id.meta['incidence'] except KeyError: raise RuntimeError( 'This function does not yet support sensor {}'.format(id.sensor)) rlks, azlks = multilook_factors(sp_rg=id.spacing[0], sp_az=id.spacing[1], tr_rg=tr, tr_az=tr, geometry=image_geometry, incidence=incidence) if azlks > 1 or rlks > 1: workflow.insert_node(parse_node('Multilook'), after=tf.id) ml = workflow['Multilook'] ml.parameters['sourceBands'] = bands_beta ml.parameters['nAzLooks'] = azlks ml.parameters['nRgLooks'] = rlks ############################################ # specify spatial resolution and coordinate reference system of the output dataset # print('-- configuring CRS') tc.parameters['pixelSpacingInMeter'] = tr try: t_srs = crsConvert(t_srs, 'epsg') except TypeError: raise RuntimeError("format of parameter 't_srs' not recognized") # the EPSG code 4326 is not supported by SNAP and thus the WKT string has to be defined; # in all other cases defining EPSG:{code} will do if t_srs == 4326: t_srs = 'GEOGCS["WGS84(DD)",' \ 'DATUM["WGS84",' \ 'SPHEROID["WGS84", 6378137.0, 298.257223563]],' \ 'PRIMEM["Greenwich", 0.0],' \ 'UNIT["degree", 0.017453292519943295],' \ 'AXIS["Geodetic longitude", EAST],' \ 'AXIS["Geodetic latitude", NORTH]]' else: t_srs = 'EPSG:{}'.format(t_srs) tc.parameters['mapProjection'] = t_srs ############################################ # (optionally) add node for conversion from linear to db scaling # print('-- configuring LinearToFromdB Node') if scaling not in ['dB', 'db', 'linear']: raise RuntimeError( 'scaling must be a string of either "dB", "db" or "linear"') if scaling in ['dB', 'db']: lin2db = parse_node('lin2db') sourceNode = 'Terrain-Correction' if geocoding_type == 'Range-Doppler' else 'SARSim-Terrain-Correction' workflow.insert_node(lin2db, before=sourceNode) lin2db.parameters['sourceBands'] = bands_gamma ############################################ # (optionally) add subset node and add bounding box coordinates of defined shapefile # print('-- configuring Subset Node') if shapefile: # print('--- read') shp = shapefile.clone() if isinstance(shapefile, Vector) else Vector(shapefile) # reproject the geometry to WGS 84 latlon # print('--- reproject') shp.reproject(4326) ext = shp.extent shp.close() # add an extra buffer of 0.01 degrees buffer = 0.01 ext['xmin'] -= buffer ext['ymin'] -= buffer ext['xmax'] += buffer ext['ymax'] += buffer # print('--- create bbox') with bbox(ext, 4326) as bounds: # print('--- intersect') inter = intersect(id.bbox(), bounds) if not inter: raise RuntimeError( 'no bounding box intersection between shapefile and scene') # print('--- close intersect') inter.close() # print('--- get wkt') wkt = bounds.convert2wkt()[0] # print('--- parse XML node') subset = parse_node('Subset') # print('--- insert node') workflow.insert_node(subset, before='Read') subset.parameters['region'] = [0, 0, id.samples, id.lines] subset.parameters['geoRegion'] = wkt ############################################ # (optionally) configure subset node for pixel offsets if offset and not shapefile: subset = parse_node('Subset') workflow.insert_node(subset, before='Read') # left, right, top and bottom offset in pixels l, r, t, b = offset subset_values = [l, t, id.samples - l - r, id.lines - t - b] subset.parameters['region'] = subset_values subset.parameters['geoRegion'] = '' ############################################ # parametrize write node # print('-- configuring Write Node') # create a suffix for the output file to identify processing steps performed in the workflow suffix = workflow.suffix basename = os.path.join(outdir, id.outname_base(basename_extensions)) extension = suffix if format == 'ENVI' else polarizations[0] + '_' + suffix outname = basename + '_' + extension write = workflow['Write'] write.parameters['file'] = outname write.parameters['formatName'] = format ############################################ ############################################ if export_extra is not None: options = [ 'incidenceAngleFromEllipsoid', 'localIncidenceAngle', 'projectedLocalIncidenceAngle', 'DEM' ] write = parse_node('Write') workflow.insert_node(write, before=tc.id, resetSuccessorSource=False) write.parameters['file'] = outname write.parameters['formatName'] = format for item in export_extra: if item not in options: raise RuntimeError( "ID '{}' not valid for argument 'export_extra'".format( item)) key = 'save{}{}'.format(item[0].upper(), item[1:]) tc.parameters[key] = True ############################################ ############################################ # select DEM type # print('-- configuring DEM') dempar = { 'externalDEMFile': externalDEMFile, 'externalDEMApplyEGM': externalDEMApplyEGM } if externalDEMFile is not None: if os.path.isfile(externalDEMFile): if externalDEMNoDataValue is None: with Raster(externalDEMFile) as dem: dempar['externalDEMNoDataValue'] = dem.nodata if dempar['externalDEMNoDataValue'] is None: raise RuntimeError( 'Cannot read NoData value from DEM file. ' 'Please specify externalDEMNoDataValue') else: dempar['externalDEMNoDataValue'] = externalDEMNoDataValue else: raise RuntimeError('specified externalDEMFile does not exist') dempar['demName'] = 'External DEM' else: # SRTM 1arcsec is only available between -58 and +60 degrees. # If the scene exceeds those latitudes SRTM 3arcsec is selected. if id.getCorners()['ymax'] > 60 or id.getCorners()['ymin'] < -58: dempar['demName'] = 'SRTM 3Sec' else: dempar['demName'] = 'SRTM 1Sec HGT' dempar['externalDEMFile'] = None dempar['externalDEMNoDataValue'] = 0 for node in workflow.nodes(): for key, value in dempar.items(): if key in node.parameters.keys(): node.parameters[key] = value ############################################ ############################################ # write workflow to file and optionally execute it # print('- writing workflow to file') workflow.write(outname + '_proc') # execute the newly written workflow if not test: try: groups = groupbyWorkers(outname + '_proc.xml', groupsize) gpt(outname + '_proc.xml', groups=groups, cleanup=cleanup) except RuntimeError: if cleanup: os.remove(outname + '_proc.xml')