def azimuth_y(self, reductionFactor=1): """Calculate the angle of each pixel position vector with respect to the Y-axis (azimuth). In general, azimuth is the angle from a reference vector (e.g., the direction to North) to the chosen position vector. The azimuth increases clockwise from direction to North. http://en.wikipedia.org/wiki/Azimuth Parameters ----------- reductionFactor : integer factor by which the size of the output array is reduced Returns ------- azimuth : numpy array Values of azimuth in degrees in range 0 - 360 """ lon_grd, lat_grd = self.get_geolocation_grids(reductionFactor) a = initial_bearing(lon_grd[1:, :], lat_grd[1:, :], lon_grd[:-1:, :], lat_grd[:-1:, :]) # Repeat last row once to match size of lon-lat grids a = np.vstack((a, a[-1, :])) return a
def azimuth_y(self, reductionFactor=1): '''Calculate the angle of each pixel position vector with respect to the Y-axis (azimuth). In general, azimuth is the angle from a reference vector (e.g., the direction to North) to the chosen position vector. The azimuth increases clockwise from direction to North. http://en.wikipedia.org/wiki/Azimuth Parameters ----------- reductionFactor : integer factor by which the size of the output array is reduced Returns ------- azimuth : numpy array Values of azimuth in degrees in range 0 - 360 ''' lon, lat = self.get_geolocation_grids(reductionFactor) a = initial_bearing(lon[1:, :], lat[1:, :], lon[:-1:, :], lat[:-1:, :]) # Repeat last row once to match size of lon-lat grids a = np.vstack((a, a[-1, :])) return a
def azimuth_y(self, reductionFactor=1): '''Calculate the azimuth of 'upward' direction in each pixel Generaly speaking, azimuth is angle from the reference vector (direction to North) to the chosen direction. Azimuth increases clockwise from direction to North. http://en.wikipedia.org/wiki/Azimuth Here we calcluate azimuth of 'upward' direction. 'Upward' direction coincides with Y-axis direction (and hence is opposite to the ROW-axis direction). For lon-lat (cylindrical, Plate Caree) and Mercator projections 'upward' direction coincides with direction to North, hence azimuth is 0. Parameters ----------- reductionFactor : integer factor by which the size of the output array is reduced Returns ------- azimuth : numpy array Values of azimuth in degrees in range 0 - 360 ''' lon, lat = self.get_geolocation_grids(reductionFactor) a = initial_bearing(lon[1:, :], lat[1:, :], lon[:-1:, :], lat[:-1:, :]) # Repeat last row once to match size of lon-lat grids a = np.vstack((a, a[-1, :])) return a
def add_look_direction_band(self): lon, lat = self.get_full_size_GCPs() """ TODO: Also in mapper_sentinel1_l1.py... Use this code there.. """ sat_heading = initial_bearing(lon[:-1, :], lat[:-1, :], lon[1:, :], lat[1:, :]) look_direction = scipy.ndimage.interpolation.zoom( np.mod(sat_heading + 90, 360), (np.shape(lon)[0] / (np.shape(lon)[0] - 1.), 1)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT.from_array(look_direction_u) look_v_VRT = VRT.from_array(look_direction_v) lookVRT = VRT.from_lonlat(lon, lat) lookVRT.create_band([{ 'SourceFilename': look_u_VRT.filename, 'SourceBand': 1 }, { 'SourceFilename': look_v_VRT.filename, 'SourceBand': 1 }], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, 1) # Store VRTs so that they are accessible later self.band_vrts['look_u_VRT'] = look_u_VRT self.band_vrts['look_v_VRT'] = look_v_VRT self.band_vrts['lookVRT'] = lookVRT src = { 'SourceFilename': self.band_vrts['lookVRT'].filename, 'SourceBand': 1 } dst = {'wkv': 'sensor_azimuth_angle', 'name': 'look_direction'} self.create_band(src, dst) self.dataset.FlushCache() """ End repetition """
def __init__(self, filename, gdalDataset, gdalMetadata, fast=False, **kwargs): if kwargs.get('manifestonly', False): fast = True NansatFutureWarning( 'manifestonly option will be deprecated. Use: fast=True') if not os.path.split(filename.rstrip('/'))[1][:3] in ['S1A', 'S1B']: raise WrongMapperError('%s: Not Sentinel 1A or 1B' % filename) if not IMPORT_SCIPY: raise NansatReadError( 'Sentinel-1 data cannot be read because scipy is not installed' ) if zipfile.is_zipfile(filename): zz = zipfile.PyZipFile(filename) # Assuming the file names are consistent, the polarization # dependent data should be sorted equally such that we can use the # same indices consistently for all the following lists # THIS IS NOT THE CASE... mds_files = [ '/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'measurement/s1' in fn ] calibration_files = [ '/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'annotation/calibration/calibration-s1' in fn ] noise_files = [ '/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'annotation/calibration/noise-s1' in fn ] annotation_files = [ '/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'annotation/s1' in fn ] manifest_files = [ '/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'manifest.safe' in fn ] zz.close() else: mds_files = glob.glob('%s/measurement/s1*' % filename) calibration_files = glob.glob( '%s/annotation/calibration/calibration-s1*' % filename) noise_files = glob.glob('%s/annotation/calibration/noise-s1*' % filename) annotation_files = glob.glob('%s/annotation/s1*' % filename) manifest_files = glob.glob('%s/manifest.safe' % filename) if (not mds_files or not calibration_files or not noise_files or not annotation_files or not manifest_files): raise WrongMapperError(filename) # convert list of MDS files into dictionary. Keys - polarizations in upper case. mds_files = { os.path.basename(ff).split('-')[3].upper(): ff for ff in mds_files } polarizations = list(mds_files.keys()) # read annotation files annotation_data = self.read_annotation(annotation_files) if not fast: annotation_data = Mapper.correct_geolocation_data(annotation_data) # read manifest file manifest_data = self.read_manifest_data(manifest_files[0]) # very fast constructor without any bands only with some metadata and geolocation self._init_empty(manifest_data, annotation_data) # skip adding bands in the fast mode and RETURN if fast: return # Open data files with GDAL gdalDatasets = {} for pol in polarizations: gdalDatasets[pol] = gdal.Open(mds_files[pol]) if not gdalDatasets[pol]: raise WrongMapperError('%s: No Sentinel-1 datasets found' % mds_files[pol]) # Check metadata to confirm it is Sentinel-1 L1 metadata = gdalDatasets[polarizations[0]].GetMetadata() # create full size VRTs with incidenceAngle and elevationAngle annotation_vrts = self.vrts_from_arrays( annotation_data, ['incidenceAngle', 'elevationAngle']) self.band_vrts.update(annotation_vrts) # create full size VRTS with calibration LUT calibration_names = ['sigmaNought', 'betaNought'] calibration_list_tag = 'calibrationVectorList' for calibration_file in calibration_files: pol = '_' + os.path.basename(calibration_file).split( '-')[4].upper() xml = self.read_vsi(calibration_file) calibration_data = self.read_calibration(xml, calibration_list_tag, calibration_names, pol) calibration_vrts = self.vrts_from_arrays(calibration_data, calibration_names, pol, True, 1) self.band_vrts.update(calibration_vrts) # create full size VRTS with noise LUT for noise_file in noise_files: pol = '_' + os.path.basename(noise_file).split('-')[4].upper() xml = self.read_vsi(noise_file) if '<noiseVectorList' in xml: noise_list_tag = 'noiseVectorList' noise_name = 'noiseLut' elif '<noiseRangeVectorList' in xml: noise_list_tag = 'noiseRangeVectorList' noise_name = 'noiseRangeLut' noise_data = self.read_calibration(xml, noise_list_tag, [noise_name], pol) noise_vrts = self.vrts_from_arrays(noise_data, [noise_name], pol, True, 1) self.band_vrts.update(noise_vrts) #### Create metaDict: dict with metadata for all bands metaDict = [] bandNumberDict = {} bnmax = 0 for pol in polarizations: dsPath, dsName = os.path.split(mds_files[pol]) name = 'DN_%s' % pol # A dictionary of band numbers is needed for the pixel function # bands further down. This is not the best solution. It would be # better to have a function in VRT that returns the number given a # band name. This function exists in Nansat but could perhaps be # moved to VRT? The existing nansat function could just call the # VRT one... bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] band = gdalDatasets[pol].GetRasterBand(1) dtype = band.DataType metaDict.append({ 'src': { 'SourceFilename': mds_files[pol], 'SourceBand': 1, 'DataType': dtype, }, 'dst': { 'name': name, }, }) # add bands with metadata and corresponding values to the empty VRT self.create_bands(metaDict) ''' Calibration should be performed as s0 = DN^2/sigmaNought^2, where sigmaNought is from e.g. annotation/calibration/calibration-s1a-iw-grd-hh-20140811t151231-20140811t151301-001894-001cc7-001.xml, and DN is the Digital Numbers in the tiff files. Also the noise should be subtracted. See https://sentinel.esa.int/web/sentinel/sentinel-1-sar-wiki/-/wiki/Sentinel%20One/Application+of+Radiometric+Calibration+LUT The noise correction/subtraction is implemented in an independent package "sentinel1denoised" See https://github.com/nansencenter/sentinel1denoised ''' # Get look direction longitude, latitude = self.transform_points( calibration_data['pixel'].flatten(), calibration_data['line'].flatten()) longitude.shape = calibration_data['pixel'].shape latitude.shape = calibration_data['pixel'].shape sat_heading = initial_bearing(longitude[:-1, :], latitude[:-1, :], longitude[1:, :], latitude[1:, :]) look_direction = scipy.ndimage.interpolation.zoom( np.mod(sat_heading + 90, 360), (np.shape(longitude)[0] / (np.shape(longitude)[0] - 1.), 1)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT.from_array(look_direction_u) look_v_VRT = VRT.from_array(look_direction_v) lookVRT = VRT.from_lonlat(longitude, latitude) lookVRT.create_band([{ 'SourceFilename': look_u_VRT.filename, 'SourceBand': 1 }, { 'SourceFilename': look_v_VRT.filename, 'SourceBand': 1 }], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, 1) # Store VRTs so that they are accessible later self.band_vrts['look_u_VRT'] = look_u_VRT self.band_vrts['look_v_VRT'] = look_v_VRT self.band_vrts['lookVRT'] = lookVRT metaDict = [] # Add bands to full size VRT for pol in polarizations: name = 'sigmaNought_%s' % pol bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': (self.band_vrts[name].filename), 'SourceBand': 1 }, 'dst': { 'name': name } }) name = 'noise_%s' % pol bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.band_vrts['%s_%s' % (noise_name, pol)].filename, 'SourceBand': 1 }, 'dst': { 'name': name } }) name = 'look_direction' bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.band_vrts['lookVRT'].filename, 'SourceBand': 1 }, 'dst': { 'wkv': 'sensor_azimuth_angle', 'name': name } }) for pol in polarizations: dsPath, dsName = os.path.split(mds_files[pol]) name = 'sigma0_%s' % pol bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] metaDict.append({ 'src': [{ 'SourceFilename': self.filename, 'SourceBand': bandNumberDict['DN_%s' % pol], }, { 'SourceFilename': self.band_vrts['sigmaNought_%s' % pol].filename, 'SourceBand': 1 }], 'dst': { 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol, 'suffix': pol, }, }) name = 'beta0_%s' % pol bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] metaDict.append({ 'src': [{ 'SourceFilename': self.filename, 'SourceBand': bandNumberDict['DN_%s' % pol] }, { 'SourceFilename': self.band_vrts['betaNought_%s' % pol].filename, 'SourceBand': 1 }], 'dst': { 'wkv': 'surface_backwards_brightness_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol, 'suffix': pol, }, }) self.create_bands(metaDict) # Add incidence angle as band name = 'incidence_angle' bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] src = { 'SourceFilename': self.band_vrts['incidenceAngle'].filename, 'SourceBand': 1 } dst = {'wkv': 'angle_of_incidence', 'name': name} self.create_band(src, dst) self.dataset.FlushCache() # Add elevation angle as band name = 'elevation_angle' bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] src = { 'SourceFilename': self.band_vrts['elevationAngle'].filename, 'SourceBand': 1 } dst = {'wkv': 'angle_of_elevation', 'name': name} self.create_band(src, dst) self.dataset.FlushCache() # Add sigma0_VV if 'VV' not in polarizations and 'HH' in polarizations: name = 'sigma0_VV' bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] src = [{ 'SourceFilename': self.filename, 'SourceBand': bandNumberDict['DN_HH'], }, { 'SourceFilename': (self.band_vrts['sigmaNought_HH'].filename), 'SourceBand': 1, }, { 'SourceFilename': self.band_vrts['incidenceAngle'].filename, 'SourceBand': 1 }] dst = { 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV' } self.create_band(src, dst) self.dataset.FlushCache()
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs): ''' Create Radarsat2 VRT ''' fPathName, fExt = os.path.splitext(fileName) if zipfile.is_zipfile(fileName): # Open zip file using VSI fPath, fName = os.path.split(fPathName) fileName = '/vsizip/%s/%s' % (fileName, fName) if not 'RS' in fName[0:2]: raise WrongMapperError('Provided data is not Radarsat-2') gdalDataset = gdal.Open(fileName) gdalMetadata = gdalDataset.GetMetadata() #if it is not RADARSAT-2, return if (not gdalMetadata or not 'SATELLITE_IDENTIFIER' in gdalMetadata.keys()): raise WrongMapperError elif gdalMetadata['SATELLITE_IDENTIFIER'] != 'RADARSAT-2': raise WrongMapperError # read product.xml productXmlName = os.path.join(fileName, 'product.xml') productXml = self.read_xml(productXmlName) # Get additional metadata from product.xml rs2_0 = Node.create(productXml) rs2_1 = rs2_0.node('sourceAttributes') rs2_2 = rs2_1.node('radarParameters') if rs2_2['antennaPointing'].lower() == 'right': antennaPointing = 90 else: antennaPointing = -90 rs2_3 = rs2_1.node('orbitAndAttitude').node('orbitInformation') passDirection = rs2_3['passDirection'] # create empty VRT dataset with geolocation only VRT.__init__(self, gdalDataset) #define dictionary of metadata and band specific parameters pol = [] metaDict = [] # Get the subdataset with calibrated sigma0 only for dataset in gdalDataset.GetSubDatasets(): if dataset[1] == 'Sigma Nought calibrated': s0dataset = gdal.Open(dataset[0]) s0datasetName = dataset[0][:] band = s0dataset.GetRasterBand(1) s0datasetPol = band.GetMetadata()['POLARIMETRIC_INTERP'] for i in range(1, s0dataset.RasterCount + 1): iBand = s0dataset.GetRasterBand(i) polString = iBand.GetMetadata()['POLARIMETRIC_INTERP'] suffix = polString # The nansat data will be complex # if the SAR data is of type 10 dtype = iBand.DataType if dtype == 10: # add intensity band metaDict.append({ 'src': { 'SourceFilename': ('RADARSAT_2_CALIB:SIGMA0:' + fileName + '/product.xml'), 'SourceBand': i, 'DataType': dtype }, 'dst': { 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'intensity', 'SourceTransferType': gdal.GetDataTypeName(dtype), 'suffix': suffix, 'polarization': polString, 'dataType': 6 } }) # modify suffix for adding the compled band below suffix = polString + '_complex' pol.append(polString) metaDict.append({ 'src': { 'SourceFilename': ('RADARSAT_2_CALIB:SIGMA0:' + fileName + '/product.xml'), 'SourceBand': i, 'DataType': dtype }, 'dst': { 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'suffix': suffix, 'polarization': polString } }) if dataset[1] == 'Beta Nought calibrated': b0dataset = gdal.Open(dataset[0]) b0datasetName = dataset[0][:] for j in range(1, b0dataset.RasterCount + 1): jBand = b0dataset.GetRasterBand(j) polString = jBand.GetMetadata()['POLARIMETRIC_INTERP'] if polString == s0datasetPol: b0datasetBand = j ############################### # Add SAR look direction ############################### d = Domain(ds=gdalDataset) lon, lat = d.get_geolocation_grids(100) ''' (GDAL?) Radarsat-2 data is stored with maximum latitude at first element of each column and minimum longitude at first element of each row (e.g. np.shape(lat)=(59,55) -> latitude maxima are at lat[0,:], and longitude minima are at lon[:,0]) In addition, there is an interpolation error for direct estimate along azimuth. We therefore estimate the heading along range and add 90 degrees to get the "satellite" heading. ''' if str(passDirection).upper() == 'DESCENDING': sat_heading = initial_bearing(lon[:, :-1], lat[:, :-1], lon[:, 1:], lat[:, 1:]) + 90 elif str(passDirection).upper() == 'ASCENDING': sat_heading = initial_bearing(lon[:, 1:], lat[:, 1:], lon[:, :-1], lat[:, :-1]) + 90 else: print 'Can not decode pass direction: ' + str(passDirection) # Calculate SAR look direction SAR_look_direction = sat_heading + antennaPointing # Interpolate to regain lost row SAR_look_direction = np.mod(SAR_look_direction, 360) SAR_look_direction = scipy.ndimage.interpolation.zoom( SAR_look_direction, (1, 11. / 10.)) # Decompose, to avoid interpolation errors around 0 <-> 360 SAR_look_direction_u = np.sin(np.deg2rad(SAR_look_direction)) SAR_look_direction_v = np.cos(np.deg2rad(SAR_look_direction)) look_u_VRT = VRT(array=SAR_look_direction_u, lat=lat, lon=lon) look_v_VRT = VRT(array=SAR_look_direction_v, lat=lat, lon=lon) # Note: If incidence angle and look direction are stored in # same VRT, access time is about twice as large lookVRT = VRT(lat=lat, lon=lon) lookVRT._create_band([{ 'SourceFilename': look_u_VRT.fileName, 'SourceBand': 1 }, { 'SourceFilename': look_v_VRT.fileName, 'SourceBand': 1 }], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) # Store VRTs so that they are accessible later self.bandVRTs['look_u_VRT'] = look_u_VRT self.bandVRTs['look_v_VRT'] = look_v_VRT self.bandVRTs['lookVRT'] = lookVRT # Add band to full sized VRT lookFileName = self.bandVRTs['lookVRT'].fileName metaDict.append({ 'src': { 'SourceFilename': lookFileName, 'SourceBand': 1 }, 'dst': { 'wkv': 'sensor_azimuth_angle', 'name': 'SAR_look_direction' } }) ############################### # Create bands ############################### self._create_bands(metaDict) ################################################### # Add derived band (incidence angle) calculated # using pixel function "BetaSigmaToIncidence": ################################################### src = [{ 'SourceFilename': b0datasetName, 'SourceBand': b0datasetBand, 'DataType': dtype }, { 'SourceFilename': s0datasetName, 'SourceBand': 1, 'DataType': dtype }] dst = { 'wkv': 'angle_of_incidence', 'PixelFunctionType': 'BetaSigmaToIncidence', 'SourceTransferType': gdal.GetDataTypeName(dtype), '_FillValue': -10000, # NB: this is also hard-coded in # pixelfunctions.c 'dataType': 6, 'name': 'incidence_angle' } self._create_band(src, dst) self.dataset.FlushCache() ################################################################### # Add sigma0_VV - pixel function of sigma0_HH and beta0_HH # incidence angle is calculated within pixel function # It is assummed that HH is the first band in sigma0 and # beta0 sub datasets ################################################################### if 'VV' not in pol and 'HH' in pol: s0datasetNameHH = pol.index('HH') + 1 src = [{ 'SourceFilename': s0datasetName, 'SourceBand': s0datasetNameHH, 'DataType': 6 }, { 'SourceFilename': b0datasetName, 'SourceBand': b0datasetBand, 'DataType': 6 }] dst = { 'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sigma0HHBetaToSigma0VV', 'polarization': 'VV', 'suffix': 'VV' } self._create_band(src, dst) self.dataset.FlushCache() ############################################ # Add SAR metadata ############################################ if antennaPointing == 90: self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT') if antennaPointing == -90: self.dataset.SetMetadataItem('ANTENNA_POINTING', 'LEFT') self.dataset.SetMetadataItem('ORBIT_DIRECTION', str(passDirection).upper()) # Set time validTime = gdalDataset.GetMetadata()['ACQUISITION_START_TIME'] self.logger.info('Valid time: %s', str(validTime)) self._set_time(parse(validTime)) # set SADCAT specific metadata self.dataset.SetMetadataItem( 'start_date', (parse(gdalMetadata['FIRST_LINE_TIME']).isoformat())) self.dataset.SetMetadataItem( 'stop_date', (parse(gdalMetadata['LAST_LINE_TIME']).isoformat())) self.dataset.SetMetadataItem('sensor', 'SAR') self.dataset.SetMetadataItem('satellite', 'Radarsat2') self.dataset.SetMetadataItem('mapper', 'radarsat2')
def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): ''' Parameters ----------- filename : string gdalDataset : gdal dataset gdalMetadata : gdal metadata ''' self.setup_ads_parameters(filename, gdalMetadata) if self.product[0:4] != "ASA_": raise WrongMapperError if not IMPORT_SCIPY: raise NansatReadError('ASAR data cannot be read because scipy is not installed') # get channel string (remove '/', since NetCDF # does not support that in metadata) polarization = [{'channel': gdalMetadata['SPH_MDS1_TX_RX_POLAR'] .replace("/", ""), 'bandNum': 1}] # if there is the 2nd band, get channel string if 'SPH_MDS2_TX_RX_POLAR' in gdalMetadata.keys(): channel = gdalMetadata['SPH_MDS2_TX_RX_POLAR'].replace("/", "") if not(channel.isspace()): polarization.append({'channel': channel, 'bandNum': 2}) # create empty VRT dataset with geolocation only self._init_from_gdal_dataset(gdalDataset, metadata=gdalMetadata) # get calibration constant gotCalibration = True try: for iPolarization in polarization: metaKey = ('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS.%d.EXT_CAL_FACT' % (iPolarization['bandNum'])) iPolarization['calibrationConst'] = float( gdalDataset.GetMetadataItem(metaKey, 'records')) except: try: for iPolarization in polarization: # Apparently some ASAR files have calibration # constant stored in another place metaKey = ('MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS.%d.EXT_CAL_FACT' % (iPolarization['bandNum'])) iPolarization['calibrationConst'] = float( gdalDataset.GetMetadataItem(metaKey, 'records')) except: self.logger.warning('Cannot get calibrationConst') gotCalibration = False # add dictionary for raw counts metaDict = [] for iPolarization in polarization: iBand = gdalDataset.GetRasterBand(iPolarization['bandNum']) dtype = iBand.DataType shortName = 'RawCounts_%s' %iPolarization['channel'] bandName = shortName dstName = 'raw_counts_%s' % iPolarization['channel'] if (8 <= dtype and dtype < 12): bandName = shortName+'_complex' dstName = dstName + '_complex' metaDict.append({'src': {'SourceFilename': filename, 'SourceBand': iPolarization['bandNum']}, 'dst': {'name': dstName}}) ''' metaDict.append({'src': {'SourceFilename': filename, 'SourceBand': iPolarization['bandNum']}, 'dst': {'name': 'raw_counts_%s' % iPolarization['channel']}}) ''' # if raw data is complex, add the intensity band if (8 <= dtype and dtype < 12): # choose pixelfunction type if (dtype == 8 or dtype == 9): pixelFunctionType = 'IntensityInt' else: pixelFunctionType = 'intensity' # get data type of the intensity band intensityDataType = {'8': 3, '9': 4, '10': 5, '11': 6}.get(str(dtype), 4) # add intensity band metaDict.append( {'src': {'SourceFilename': filename, 'SourceBand': iPolarization['bandNum'], 'DataType': dtype}, 'dst': {'name': 'raw_counts_%s' % iPolarization['channel'], 'PixelFunctionType': pixelFunctionType, 'SourceTransferType': gdal.GetDataTypeName(dtype), 'dataType': intensityDataType}}) ##################################################################### # Add incidence angle and look direction through small VRT objects ##################################################################### lon = self.get_array_from_ADS('first_line_longs') lat = self.get_array_from_ADS('first_line_lats') inc = self.get_array_from_ADS('first_line_incidence_angle') # Calculate SAR look direction (ASAR is always right-looking) look_direction = initial_bearing(lon[:, :-1], lat[:, :-1], lon[:, 1:], lat[:, 1:]) # Interpolate to regain lost row look_direction = scipy.ndimage.interpolation.zoom( look_direction, (1, 11./10.)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT.from_array(look_direction_u) look_v_VRT = VRT.from_array(look_direction_v) # Note: If incidence angle and look direction are stored in # same VRT, access time is about twice as large incVRT = VRT.from_array(inc) lookVRT = VRT.from_lonlat(lon, lat) lookVRT.create_band([{'SourceFilename': look_u_VRT.filename, 'SourceBand': 1}, {'SourceFilename': look_v_VRT.filename, 'SourceBand': 1}], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up bands to full size incVRT = incVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) # Store VRTs so that they are accessible later self.band_vrts = {'incVRT': incVRT, 'look_u_VRT': look_u_VRT, 'look_v_VRT': look_v_VRT, 'lookVRT': lookVRT} # Add band to full sized VRT incFileName = self.band_vrts['incVRT'].filename lookFileName = self.band_vrts['lookVRT'].filename metaDict.append({'src': {'SourceFilename': incFileName, 'SourceBand': 1}, 'dst': {'wkv': 'angle_of_incidence', 'name': 'incidence_angle'}}) metaDict.append({'src': {'SourceFilename': lookFileName, 'SourceBand': 1}, 'dst': {'wkv': 'sensor_azimuth_angle', 'name': 'look_direction'}}) #################### # Add Sigma0-bands #################### if gotCalibration: for iPolarization in polarization: # add dictionary for sigma0, ice and water short_names = ['sigma0', 'sigma0_normalized_ice', 'sigma0_normalized_water'] wkt = [ 'surface_backwards_scattering_coefficient_of_radar_wave', 'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_ice', 'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_water'] sphPass = [gdalMetadata['SPH_PASS'], '', ''] sourceFileNames = [filename, incFileName] pixelFunctionTypes = ['RawcountsIncidenceToSigma0', 'Sigma0NormalizedIce'] if iPolarization['channel'] == 'HH': pixelFunctionTypes.append('Sigma0HHNormalizedWater') elif iPolarization['channel'] == 'VV': pixelFunctionTypes.append('Sigma0VVNormalizedWater') # add pixelfunction bands to metaDict for iPixFunc in range(len(pixelFunctionTypes)): srcFiles = [] for j, jFileName in enumerate(sourceFileNames): sourceFile = {'SourceFilename': jFileName} if j == 0: sourceFile['SourceBand'] = iPolarization['bandNum'] # if ASA_full_incAng, # set 'ScaleRatio' into source file dict sourceFile['ScaleRatio'] = np.sqrt( 1.0 / iPolarization['calibrationConst']) else: sourceFile['SourceBand'] = 1 srcFiles.append(sourceFile) metaDict.append({ 'src': srcFiles, 'dst': {'short_name': short_names[iPixFunc], 'wkv': wkt[iPixFunc], 'PixelFunctionType': ( pixelFunctionTypes[iPixFunc]), 'polarization': iPolarization['channel'], 'suffix': iPolarization['channel'], 'pass': sphPass[iPixFunc], 'dataType': 6}}) # add bands with metadata and corresponding values to the empty VRT self.create_bands(metaDict) # Add oribit and look information to metadata domain # ASAR is always right-looking self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT') self.dataset.SetMetadataItem('ORBIT_DIRECTION', gdalMetadata['SPH_PASS'].upper().strip()) ################################################################### # Estimate sigma0_VV from sigma0_HH ################################################################### polarizations = [] for pp in polarization: polarizations.append(pp['channel']) if 'VV' not in polarizations and 'HH' in polarizations: srcFiles = [] for j, jFileName in enumerate(sourceFileNames): sourceFile = {'SourceFilename': jFileName} if j == 0: sourceFile['SourceBand'] = iPolarization['bandNum'] # if ASA_full_incAng, # set 'ScaleRatio' into source file dict sourceFile['ScaleRatio'] = np.sqrt( 1.0 / iPolarization['calibrationConst']) else: sourceFile['SourceBand'] = 1 srcFiles.append(sourceFile) dst = {'wkv': ( 'surface_backwards_scattering_coefficient_of_radar_wave'), 'PixelFunctionType': 'Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV'} self.create_band(srcFiles, dst) self.dataset.FlushCache() # set time self._set_envisat_time(gdalMetadata) # When using TPS for reprojection, use only every 3rd GCP # to improve performance (tradeoff vs accuracy) self.dataset.SetMetadataItem('skip_gcps', '3') self.dataset.SetMetadataItem('time_coverage_start', (parse(gdalMetadata['MPH_SENSING_START']). isoformat())) self.dataset.SetMetadataItem('time_coverage_end', (parse(gdalMetadata['MPH_SENSING_STOP']). isoformat())) # Get dictionary describing the instrument and platform according to # the GCMD keywords mm = pti.get_gcmd_instrument('asar') ee = pti.get_gcmd_platform('envisat') # TODO: Validate that the found instrument and platform are indeed what # we want.... self.dataset.SetMetadataItem('instrument', json.dumps(mm)) self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, manifestonly=False, **kwargs): if zipfile.is_zipfile(fileName): zz = zipfile.PyZipFile(fileName) # Assuming the file names are consistent, the polarization # dependent data should be sorted equally such that we can use the # same indices consistently for all the following lists # THIS IS NOT THE CASE... mdsFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'measurement/s1a' in fn] calFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'annotation/calibration/calibration-s1a' in fn] noiseFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'annotation/calibration/noise-s1a' in fn] annotationFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'annotation/s1a' in fn] manifestFile = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'manifest.safe' in fn] zz.close() else: mdsFiles = glob.glob('%s/measurement/s1a*' % fileName) calFiles = glob.glob('%s/annotation/calibration/calibration-s1a*' % fileName) noiseFiles = glob.glob('%s/annotation/calibration/noise-s1a*' % fileName) annotationFiles = glob.glob('%s/annotation/s1a*' % fileName) manifestFile = glob.glob('%s/manifest.safe' % fileName) if (not mdsFiles or not calFiles or not noiseFiles or not annotationFiles or not manifestFile): raise WrongMapperError mdsDict = {} for ff in mdsFiles: mdsDict[ os.path.splitext(os.path.basename(ff))[0].split('-')[3]] = ff self.calXMLDict = {} for ff in calFiles: self.calXMLDict[ os.path.splitext( os.path.basename(ff))[0].split('-')[4]] = self.read_xml(ff) self.noiseXMLDict = {} for ff in noiseFiles: self.noiseXMLDict[ os.path.splitext( os.path.basename(ff))[0].split('-')[4]] = self.read_xml(ff) self.annotationXMLDict = {} for ff in annotationFiles: self.annotationXMLDict[ os.path.splitext( os.path.basename(ff))[0].split('-')[3]] = self.read_xml(ff) self.manifestXML = self.read_xml(manifestFile[0]) # very fast constructor without any bands if manifestonly: self.init_from_manifest_only(self.manifestXML, self.annotationXMLDict[ self.annotationXMLDict.keys()[0]]) return gdalDatasets = {} for key in mdsDict.keys(): # Open data files gdalDatasets[key] = gdal.Open(mdsDict[key]) if not gdalDatasets: raise WrongMapperError('No Sentinel-1 datasets found') # Check metadata to confirm it is Sentinel-1 L1 for key in gdalDatasets: metadata = gdalDatasets[key].GetMetadata() break if not 'TIFFTAG_IMAGEDESCRIPTION' in metadata.keys(): raise WrongMapperError if (not 'Sentinel-1' in metadata['TIFFTAG_IMAGEDESCRIPTION'] and not 'L1' in metadata['TIFFTAG_IMAGEDESCRIPTION']): raise WrongMapperError warnings.warn('Sentinel-1 level-1 mapper is not yet adapted to ' 'complex data. In addition, the band names should be ' 'updated for multi-swath data - ' 'and there might be other issues.') # create empty VRT dataset with geolocation only for key in gdalDatasets: VRT.__init__(self, gdalDatasets[key]) break # Read annotation, noise and calibration xml-files pol = {} it = 0 for key in self.annotationXMLDict: xml = Node.create(self.annotationXMLDict[key]) pol[key] = (xml.node('product'). node('adsHeader')['polarisation'].upper()) it += 1 if it == 1: # Get incidence angle pi = xml.node('generalAnnotation').node('productInformation') self.dataset.SetMetadataItem('ORBIT_DIRECTION', str(pi['pass'])) (X, Y, lon, lat, inc, ele, numberOfSamples, numberOfLines) = self.read_geolocation_lut( self.annotationXMLDict[key]) X = np.unique(X) Y = np.unique(Y) lon = np.array(lon).reshape(len(Y), len(X)) lat = np.array(lat).reshape(len(Y), len(X)) inc = np.array(inc).reshape(len(Y), len(X)) ele = np.array(ele).reshape(len(Y), len(X)) incVRT = VRT(array=inc, lat=lat, lon=lon) eleVRT = VRT(array=ele, lat=lat, lon=lon) incVRT = incVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=2) eleVRT = eleVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=2) self.bandVRTs['incVRT'] = incVRT self.bandVRTs['eleVRT'] = eleVRT for key in self.calXMLDict: calibration_LUT_VRTs, longitude, latitude = ( self.get_LUT_VRTs(self.calXMLDict[key], 'calibrationVectorList', ['sigmaNought', 'betaNought', 'gamma', 'dn'] )) self.bandVRTs['LUT_sigmaNought_VRT_'+pol[key]] = ( calibration_LUT_VRTs['sigmaNought']. get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1)) self.bandVRTs['LUT_betaNought_VRT_'+pol[key]] = ( calibration_LUT_VRTs['betaNought']. get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1)) self.bandVRTs['LUT_gamma_VRT'] = calibration_LUT_VRTs['gamma'] self.bandVRTs['LUT_dn_VRT'] = calibration_LUT_VRTs['dn'] for key in self.noiseXMLDict: noise_LUT_VRT = self.get_LUT_VRTs(self.noiseXMLDict[key], 'noiseVectorList', ['noiseLut'])[0] self.bandVRTs['LUT_noise_VRT_'+pol[key]] = ( noise_LUT_VRT['noiseLut'].get_resized_vrt( self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1)) metaDict = [] bandNumberDict = {} bnmax = 0 for key in gdalDatasets.keys(): dsPath, dsName = os.path.split(mdsDict[key]) name = 'DN_%s' % pol[key] # A dictionary of band numbers is needed for the pixel function # bands further down. This is not the best solution. It would be # better to have a function in VRT that returns the number given a # band name. This function exists in Nansat but could perhaps be # moved to VRT? The existing nansat function could just call the # VRT one... bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] band = gdalDatasets[key].GetRasterBand(1) dtype = band.DataType metaDict.append({ 'src': { 'SourceFilename': mdsDict[key], 'SourceBand': 1, 'DataType': dtype, }, 'dst': { 'name': name, #'SourceTransferType': gdal.GetDataTypeName(dtype), #'dataType': 6, }, }) # add bands with metadata and corresponding values to the empty VRT self._create_bands(metaDict) ''' Calibration should be performed as s0 = DN^2/sigmaNought^2, where sigmaNought is from e.g. annotation/calibration/calibration-s1a-iw-grd-hh-20140811t151231-20140811t151301-001894-001cc7-001.xml, and DN is the Digital Numbers in the tiff files. Also the noise should be subtracted. See https://sentinel.esa.int/web/sentinel/sentinel-1-sar-wiki/-/wiki/Sentinel%20One/Application+of+Radiometric+Calibration+LUT ''' # Get look direction sat_heading = initial_bearing(longitude[:-1, :], latitude[:-1, :], longitude[1:, :], latitude[1:, :]) look_direction = scipy.ndimage.interpolation.zoom( np.mod(sat_heading + 90, 360), (np.shape(longitude)[0] / (np.shape(longitude)[0]-1.), 1)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT(array=look_direction_u, lat=latitude, lon=longitude) look_v_VRT = VRT(array=look_direction_v, lat=latitude, lon=longitude) lookVRT = VRT(lat=latitude, lon=longitude) lookVRT._create_band([{'SourceFilename': look_u_VRT.fileName, 'SourceBand': 1}, {'SourceFilename': look_v_VRT.fileName, 'SourceBand': 1}], {'PixelFunctionType': 'UVToDirectionTo'} ) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1) # Store VRTs so that they are accessible later self.bandVRTs['look_u_VRT'] = look_u_VRT self.bandVRTs['look_v_VRT'] = look_v_VRT self.bandVRTs['lookVRT'] = lookVRT metaDict = [] # Add bands to full size VRT for key in pol: name = 'LUT_sigmaNought_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': {'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_' + pol[key]].fileName), 'SourceBand': 1 }, 'dst': {'name': name } }) name = 'LUT_noise_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.bandVRTs['LUT_noise_VRT_' + pol[key]].fileName, 'SourceBand': 1 }, 'dst': { 'name': name } }) name = 'look_direction' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.bandVRTs['lookVRT'].fileName, 'SourceBand': 1 }, 'dst': { 'wkv': 'sensor_azimuth_angle', 'name': name } }) for key in gdalDatasets.keys(): dsPath, dsName = os.path.split(mdsDict[key]) name = 'sigma0_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': [{'SourceFilename': self.fileName, 'SourceBand': bandNumberDict['DN_%s' % pol[key]], }, {'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_%s' % pol[key]].fileName), 'SourceBand': 1 } ], 'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol[key], 'suffix': pol[key], }, }) name = 'beta0_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': [{'SourceFilename': self.fileName, 'SourceBand': bandNumberDict['DN_%s' % pol[key]] }, {'SourceFilename': (self.bandVRTs['LUT_betaNought_VRT_%s' % pol[key]].fileName), 'SourceBand': 1 } ], 'dst': {'wkv': 'surface_backwards_brightness_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol[key], 'suffix': pol[key], }, }) self._create_bands(metaDict) # Add incidence angle as band name = 'incidence_angle' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = {'SourceFilename': self.bandVRTs['incVRT'].fileName, 'SourceBand': 1} dst = {'wkv': 'angle_of_incidence', 'name': name} self._create_band(src, dst) self.dataset.FlushCache() # Add elevation angle as band name = 'elevation_angle' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = {'SourceFilename': self.bandVRTs['eleVRT'].fileName, 'SourceBand': 1} dst = {'wkv': 'angle_of_elevation', 'name': name} self._create_band(src, dst) self.dataset.FlushCache() # Add sigma0_VV pp = [pol[key] for key in pol] if 'VV' not in pp and 'HH' in pp: name = 'sigma0_VV' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = [{'SourceFilename': self.fileName, 'SourceBand': bandNumberDict['DN_HH'], }, {'SourceFilename': (self.bandVRTs['LUT_noise_VRT_HH']. fileName), 'SourceBand': 1 }, {'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_HH']. fileName), 'SourceBand': 1, }, {'SourceFilename': self.bandVRTs['incVRT'].fileName, 'SourceBand': 1} ] dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV'} self._create_band(src, dst) self.dataset.FlushCache() # set time as acquisition start time n = Node.create(self.manifestXML) meta = n.node('metadataSection') for nn in meta.children: if nn.getAttribute('ID') == u'acquisitionPeriod': # set valid time self.dataset.SetMetadataItem( 'time_coverage_start', parse((nn.node('metadataWrap'). node('xmlData'). node('safe:acquisitionPeriod')['safe:startTime']) ).isoformat()) self.dataset.SetMetadataItem( 'time_coverage_end', parse((nn.node('metadataWrap'). node('xmlData'). node('safe:acquisitionPeriod')['safe:stopTime']) ).isoformat()) # Get dictionary describing the instrument and platform according to # the GCMD keywords mm = pti.get_gcmd_instrument('sar') ee = pti.get_gcmd_platform('sentinel-1a') # TODO: Validate that the found instrument and platform are indeed what we # want.... self.dataset.SetMetadataItem('instrument', json.dumps(mm)) self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): ''' Parameters ----------- filename : string gdalDataset : gdal dataset gdalMetadata : gdal metadata ''' self.setup_ads_parameters(filename, gdalMetadata) if self.product[0:4] != "ASA_": raise WrongMapperError if not IMPORT_SCIPY: raise NansatReadError( 'ASAR data cannot be read because scipy is not installed') # get channel string (remove '/', since NetCDF # does not support that in metadata) polarization = [{ 'channel': gdalMetadata['SPH_MDS1_TX_RX_POLAR'].replace("/", ""), 'bandNum': 1 }] # if there is the 2nd band, get channel string if 'SPH_MDS2_TX_RX_POLAR' in gdalMetadata.keys(): channel = gdalMetadata['SPH_MDS2_TX_RX_POLAR'].replace("/", "") if not (channel.isspace()): polarization.append({'channel': channel, 'bandNum': 2}) # create empty VRT dataset with geolocation only self._init_from_gdal_dataset(gdalDataset, metadata=gdalMetadata) # get calibration constant gotCalibration = True try: for iPolarization in polarization: metaKey = ( 'MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS.%d.EXT_CAL_FACT' % (iPolarization['bandNum'])) iPolarization['calibrationConst'] = float( gdalDataset.GetMetadataItem(metaKey, 'records')) except: try: for iPolarization in polarization: # Apparently some ASAR files have calibration # constant stored in another place metaKey = ( 'MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS.%d.EXT_CAL_FACT' % (iPolarization['bandNum'])) iPolarization['calibrationConst'] = float( gdalDataset.GetMetadataItem(metaKey, 'records')) except: self.logger.warning('Cannot get calibrationConst') gotCalibration = False # add dictionary for raw counts metaDict = [] for iPolarization in polarization: iBand = gdalDataset.GetRasterBand(iPolarization['bandNum']) dtype = iBand.DataType shortName = 'RawCounts_%s' % iPolarization['channel'] bandName = shortName dstName = 'raw_counts_%s' % iPolarization['channel'] if (8 <= dtype and dtype < 12): bandName = shortName + '_complex' dstName = dstName + '_complex' metaDict.append({ 'src': { 'SourceFilename': filename, 'SourceBand': iPolarization['bandNum'] }, 'dst': { 'name': dstName } }) ''' metaDict.append({'src': {'SourceFilename': filename, 'SourceBand': iPolarization['bandNum']}, 'dst': {'name': 'raw_counts_%s' % iPolarization['channel']}}) ''' # if raw data is complex, add the intensity band if (8 <= dtype and dtype < 12): # choose pixelfunction type if (dtype == 8 or dtype == 9): pixelFunctionType = 'IntensityInt' else: pixelFunctionType = 'intensity' # get data type of the intensity band intensityDataType = { '8': 3, '9': 4, '10': 5, '11': 6 }.get(str(dtype), 4) # add intensity band metaDict.append({ 'src': { 'SourceFilename': filename, 'SourceBand': iPolarization['bandNum'], 'DataType': dtype }, 'dst': { 'name': 'raw_counts_%s' % iPolarization['channel'], 'PixelFunctionType': pixelFunctionType, 'SourceTransferType': gdal.GetDataTypeName(dtype), 'dataType': intensityDataType } }) ##################################################################### # Add incidence angle and look direction through small VRT objects ##################################################################### lon = self.get_array_from_ADS('first_line_longs') lat = self.get_array_from_ADS('first_line_lats') inc = self.get_array_from_ADS('first_line_incidence_angle') # Calculate SAR look direction (ASAR is always right-looking) look_direction = initial_bearing(lon[:, :-1], lat[:, :-1], lon[:, 1:], lat[:, 1:]) # Interpolate to regain lost row look_direction = scipy.ndimage.interpolation.zoom( look_direction, (1, 11. / 10.)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT.from_array(look_direction_u) look_v_VRT = VRT.from_array(look_direction_v) # Note: If incidence angle and look direction are stored in # same VRT, access time is about twice as large incVRT = VRT.from_array(inc) lookVRT = VRT.from_lonlat(lon, lat) lookVRT.create_band([{ 'SourceFilename': look_u_VRT.filename, 'SourceBand': 1 }, { 'SourceFilename': look_v_VRT.filename, 'SourceBand': 1 }], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up bands to full size incVRT = incVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) # Store VRTs so that they are accessible later self.band_vrts = { 'incVRT': incVRT, 'look_u_VRT': look_u_VRT, 'look_v_VRT': look_v_VRT, 'lookVRT': lookVRT } # Add band to full sized VRT incFileName = self.band_vrts['incVRT'].filename lookFileName = self.band_vrts['lookVRT'].filename metaDict.append({ 'src': { 'SourceFilename': incFileName, 'SourceBand': 1 }, 'dst': { 'wkv': 'angle_of_incidence', 'name': 'incidence_angle' } }) metaDict.append({ 'src': { 'SourceFilename': lookFileName, 'SourceBand': 1 }, 'dst': { 'wkv': 'sensor_azimuth_angle', 'name': 'look_direction' } }) #################### # Add Sigma0-bands #################### if gotCalibration: for iPolarization in polarization: # add dictionary for sigma0, ice and water short_names = [ 'sigma0', 'sigma0_normalized_ice', 'sigma0_normalized_water' ] wkt = [ 'surface_backwards_scattering_coefficient_of_radar_wave', 'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_ice', 'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_water' ] sphPass = [gdalMetadata['SPH_PASS'], '', ''] sourceFileNames = [filename, incFileName] pixelFunctionTypes = [ 'RawcountsIncidenceToSigma0', 'Sigma0NormalizedIce' ] if iPolarization['channel'] == 'HH': pixelFunctionTypes.append('Sigma0HHNormalizedWater') elif iPolarization['channel'] == 'VV': pixelFunctionTypes.append('Sigma0VVNormalizedWater') # add pixelfunction bands to metaDict for iPixFunc in range(len(pixelFunctionTypes)): srcFiles = [] for j, jFileName in enumerate(sourceFileNames): sourceFile = {'SourceFilename': jFileName} if j == 0: sourceFile['SourceBand'] = iPolarization['bandNum'] # if ASA_full_incAng, # set 'ScaleRatio' into source file dict sourceFile['ScaleRatio'] = np.sqrt( 1.0 / iPolarization['calibrationConst']) else: sourceFile['SourceBand'] = 1 srcFiles.append(sourceFile) metaDict.append({ 'src': srcFiles, 'dst': { 'short_name': short_names[iPixFunc], 'wkv': wkt[iPixFunc], 'PixelFunctionType': (pixelFunctionTypes[iPixFunc]), 'polarization': iPolarization['channel'], 'suffix': iPolarization['channel'], 'pass': sphPass[iPixFunc], 'dataType': 6 } }) # add bands with metadata and corresponding values to the empty VRT self.create_bands(metaDict) # Add oribit and look information to metadata domain # ASAR is always right-looking self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT') self.dataset.SetMetadataItem('ORBIT_DIRECTION', gdalMetadata['SPH_PASS'].upper().strip()) ################################################################### # Estimate sigma0_VV from sigma0_HH ################################################################### polarizations = [] for pp in polarization: polarizations.append(pp['channel']) if 'VV' not in polarizations and 'HH' in polarizations: srcFiles = [] for j, jFileName in enumerate(sourceFileNames): sourceFile = {'SourceFilename': jFileName} if j == 0: sourceFile['SourceBand'] = iPolarization['bandNum'] # if ASA_full_incAng, # set 'ScaleRatio' into source file dict sourceFile['ScaleRatio'] = np.sqrt( 1.0 / iPolarization['calibrationConst']) else: sourceFile['SourceBand'] = 1 srcFiles.append(sourceFile) dst = { 'wkv': ('surface_backwards_scattering_coefficient_of_radar_wave'), 'PixelFunctionType': 'Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV' } self.create_band(srcFiles, dst) self.dataset.FlushCache() # set time self._set_envisat_time(gdalMetadata) # When using TPS for reprojection, use only every 3rd GCP # to improve performance (tradeoff vs accuracy) self.dataset.SetMetadataItem('skip_gcps', '3') self.dataset.SetMetadataItem( 'time_coverage_start', (parse(gdalMetadata['MPH_SENSING_START']).isoformat())) self.dataset.SetMetadataItem( 'time_coverage_end', (parse(gdalMetadata['MPH_SENSING_STOP']).isoformat())) # Get dictionary describing the instrument and platform according to # the GCMD keywords mm = pti.get_gcmd_instrument('asar') ee = pti.get_gcmd_platform('envisat') # TODO: Validate that the found instrument and platform are indeed what # we want.... self.dataset.SetMetadataItem('instrument', json.dumps(mm)) self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs): ''' Parameters ----------- fileName : string gdalDataset : gdal dataset gdalMetadata : gdal metadata ''' self.setup_ads_parameters(fileName, gdalMetadata) if self.product[0:4] != "ASA_": raise WrongMapperError # get channel string (remove '/', since NetCDF # does not support that in metadata) polarization = [{'channel': gdalMetadata['SPH_MDS1_TX_RX_POLAR'] .replace("/", ""), 'bandNum': 1}] # if there is the 2nd band, get channel string if 'SPH_MDS2_TX_RX_POLAR' in gdalMetadata.keys(): channel = gdalMetadata['SPH_MDS2_TX_RX_POLAR'].replace("/", "") if not(channel.isspace()): polarization.append({'channel': channel, 'bandNum': 2}) # create empty VRT dataset with geolocation only VRT.__init__(self, gdalDataset) # get calibration constant gotCalibration = True try: for iPolarization in polarization: metaKey = ('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS.%d.EXT_CAL_FACT' % (iPolarization['bandNum'])) iPolarization['calibrationConst'] = float( gdalDataset.GetMetadataItem(metaKey, 'records')) except: try: for iPolarization in polarization: # Apparently some ASAR files have calibration # constant stored in another place metaKey = ('MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS.%d.EXT_CAL_FACT' % (iPolarization['bandNum'])) iPolarization['calibrationConst'] = float( gdalDataset.GetMetadataItem(metaKey, 'records')) except: self.logger.warning('Cannot get calibrationConst') gotCalibration = False # add dictionary for raw counts metaDict = [] for iPolarization in polarization: metaDict.append({'src': {'SourceFilename': fileName, 'SourceBand': iPolarization['bandNum']}, 'dst': {'name': 'raw_counts_%s' % iPolarization['channel']}}) ##################################################################### # Add incidence angle and look direction through small VRT objects ##################################################################### lon = self.get_array_from_ADS('first_line_longs') lat = self.get_array_from_ADS('first_line_lats') inc = self.get_array_from_ADS('first_line_incidence_angle') # Calculate SAR look direction (ASAR is always right-looking) SAR_look_direction = initial_bearing(lon[:, :-1], lat[:, :-1], lon[:, 1:], lat[:, 1:]) # Interpolate to regain lost row SAR_look_direction = scipy.ndimage.interpolation.zoom( SAR_look_direction, (1, 11./10.)) # Decompose, to avoid interpolation errors around 0 <-> 360 SAR_look_direction_u = np.sin(np.deg2rad(SAR_look_direction)) SAR_look_direction_v = np.cos(np.deg2rad(SAR_look_direction)) look_u_VRT = VRT(array=SAR_look_direction_u, lat=lat, lon=lon) look_v_VRT = VRT(array=SAR_look_direction_v, lat=lat, lon=lon) # Note: If incidence angle and look direction are stored in # same VRT, access time is about twice as large incVRT = VRT(array=inc, lat=lat, lon=lon) lookVRT = VRT(lat=lat, lon=lon) lookVRT._create_band([{'SourceFilename': look_u_VRT.fileName, 'SourceBand': 1}, {'SourceFilename': look_v_VRT.fileName, 'SourceBand': 1}], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up bands to full size incVRT = incVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) # Store VRTs so that they are accessible later self.bandVRTs = {'incVRT': incVRT, 'look_u_VRT': look_u_VRT, 'look_v_VRT': look_v_VRT, 'lookVRT': lookVRT} # Add band to full sized VRT incFileName = self.bandVRTs['incVRT'].fileName lookFileName = self.bandVRTs['lookVRT'].fileName metaDict.append({'src': {'SourceFilename': incFileName, 'SourceBand': 1}, 'dst': {'wkv': 'angle_of_incidence', 'name': 'incidence_angle'}}) metaDict.append({'src': {'SourceFilename': lookFileName, 'SourceBand': 1}, 'dst': {'wkv': 'sensor_azimuth_angle', 'name': 'SAR_look_direction'}}) #################### # Add Sigma0-bands #################### if gotCalibration: for iPolarization in polarization: # add dictionary for sigma0, ice and water short_names = ['sigma0', 'sigma0_normalized_ice', 'sigma0_normalized_water'] wkt = [ 'surface_backwards_scattering_coefficient_of_radar_wave', 'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_ice', 'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_water'] sphPass = [gdalMetadata['SPH_PASS'], '', ''] sourceFileNames = [fileName, incFileName] pixelFunctionTypes = ['RawcountsIncidenceToSigma0', 'Sigma0NormalizedIce'] if iPolarization['channel'] == 'HH': pixelFunctionTypes.append('Sigma0HHNormalizedWater') elif iPolarization['channel'] == 'VV': pixelFunctionTypes.append('Sigma0VVNormalizedWater') # add pixelfunction bands to metaDict for iPixFunc in range(len(pixelFunctionTypes)): srcFiles = [] for j, jFileName in enumerate(sourceFileNames): sourceFile = {'SourceFilename': jFileName} if j == 0: sourceFile['SourceBand'] = iPolarization['bandNum'] # if ASA_full_incAng, # set 'ScaleRatio' into source file dict sourceFile['ScaleRatio'] = np.sqrt( 1.0 / iPolarization['calibrationConst']) else: sourceFile['SourceBand'] = 1 srcFiles.append(sourceFile) metaDict.append({ 'src': srcFiles, 'dst': {'short_name': short_names[iPixFunc], 'wkv': wkt[iPixFunc], 'PixelFunctionType': ( pixelFunctionTypes[iPixFunc]), 'polarization': iPolarization['channel'], 'suffix': iPolarization['channel'], 'pass': sphPass[iPixFunc], 'dataType': 6}}) # add bands with metadata and corresponding values to the empty VRT self._create_bands(metaDict) # Add oribit and look information to metadata domain # ASAR is always right-looking self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT') self.dataset.SetMetadataItem('ORBIT_DIRECTION', gdalMetadata['SPH_PASS'].upper()) ################################################################### # Add sigma0_VV ################################################################### polarizations = [] for pp in polarization: polarizations.append(pp['channel']) if 'VV' not in polarizations and 'HH' in polarizations: srcFiles = [] for j, jFileName in enumerate(sourceFileNames): sourceFile = {'SourceFilename': jFileName} if j == 0: sourceFile['SourceBand'] = iPolarization['bandNum'] # if ASA_full_incAng, # set 'ScaleRatio' into source file dict sourceFile['ScaleRatio'] = np.sqrt( 1.0 / iPolarization['calibrationConst']) else: sourceFile['SourceBand'] = 1 srcFiles.append(sourceFile) dst = {'wkv': ( 'surface_backwards_scattering_coefficient_of_radar_wave'), 'PixelFunctionType': 'Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV'} self._create_band(srcFiles, dst) self.dataset.FlushCache() # set time self._set_envisat_time(gdalMetadata) # When using TPS for reprojection, use only every 3rd GCP # to improve performance (tradeoff vs accuracy) self.dataset.SetMetadataItem('skip_gcps', '3') # set SADCAT specific metadata self.dataset.SetMetadataItem('start_date', (parse(gdalMetadata['MPH_SENSING_START']). isoformat())) self.dataset.SetMetadataItem('stop_date', (parse(gdalMetadata['MPH_SENSING_STOP']). isoformat())) self.dataset.SetMetadataItem('sensor', 'ASAR') self.dataset.SetMetadataItem('satellite', 'Envisat') self.dataset.SetMetadataItem('mapper', 'asar')
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs): if zipfile.is_zipfile(fileName): zz = zipfile.PyZipFile(fileName) # Assuming the file names are consistent, the polarization # dependent data should be sorted equally such that we can use the # same indices consistently for all the following lists # THIS IS NOT THE CASE... mdsFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'measurement/s1a' in fn] calFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'annotation/calibration/calibration-s1a' in fn] noiseFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'annotation/calibration/noise-s1a' in fn] annotationFiles = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'annotation/s1a' in fn] manifestFile = ['/vsizip/%s/%s' % (fileName, fn) for fn in zz.namelist() if 'manifest.safe' in fn] zz.close() else: mdsFiles = glob.glob('%s/measurement/s1a*' % fileName) calFiles = glob.glob('%s/annotation/calibration/calibration-s1a*' % fileName) noiseFiles = glob.glob('%s/annotation/calibration/noise-s1a*' % fileName) annotationFiles = glob.glob('%s/annotation/s1a*' % fileName) manifestFile = glob.glob('%s/manifest.safe' % fileName) if (not mdsFiles or not calFiles or not noiseFiles or not annotationFiles or not manifestFile): raise WrongMapperError mdsDict = {} for mds in mdsFiles: mdsDict[int((os.path.splitext(os.path.basename(mds))[0]. split('-'))[-1:][0])] = mds calDict = {} for ff in calFiles: calDict[int((os.path.splitext(os.path.basename(ff))[0]. split('-'))[-1:][0])] = ff noiseDict = {} for ff in noiseFiles: noiseDict[int((os.path.splitext(os.path.basename(ff))[0]. split('-'))[-1:][0])] = ff annotationDict = {} for ff in annotationFiles: annotationDict[int((os.path.splitext(os.path.basename(ff))[0]. split('-'))[-1:][0])] = ff manifestXML = self.read_xml(manifestFile[0]) gdalDatasets = {} for key in mdsDict.keys(): # Open data files gdalDatasets[key] = gdal.Open(mdsDict[key]) if not gdalDatasets: raise WrongMapperError('No Sentinel-1 datasets found') # Check metadata to confirm it is Sentinel-1 L1 for key in gdalDatasets: metadata = gdalDatasets[key].GetMetadata() break if not 'TIFFTAG_IMAGEDESCRIPTION' in metadata.keys(): raise WrongMapperError if (not 'Sentinel-1' in metadata['TIFFTAG_IMAGEDESCRIPTION'] and not 'L1' in metadata['TIFFTAG_IMAGEDESCRIPTION']): raise WrongMapperError warnings.warn('Sentinel-1 level-1 mapper is not yet adapted to ' 'complex data. In addition, the band names should be ' 'updated for multi-swath data - ' 'and there might be other issues.') # create empty VRT dataset with geolocation only for key in gdalDatasets: VRT.__init__(self, gdalDatasets[key]) break # Read annotation, noise and calibration xml-files pol = {} it = 0 for key in annotationDict.keys(): xml = Node.create(self.read_xml(annotationDict[key])) pol[key] = (xml.node('product'). node('adsHeader')['polarisation'].upper()) it += 1 if it == 1: # Get incidence angle pi = xml.node('generalAnnotation').node('productInformation') self.dataset.SetMetadataItem('ORBIT_DIRECTION', str(pi['pass'])) # Incidence angles are found in #<geolocationGrid> # <geolocationGridPointList count="#"> # <geolocationGridPoint> geolocationGridPointList = (xml.node('geolocationGrid'). children[0]) X = [] Y = [] lon = [] lat = [] inc = [] ele = [] for gridPoint in geolocationGridPointList.children: X.append(int(gridPoint['pixel'])) Y.append(int(gridPoint['line'])) lon.append(float(gridPoint['longitude'])) lat.append(float(gridPoint['latitude'])) inc.append(float(gridPoint['incidenceAngle'])) ele.append(float(gridPoint['elevationAngle'])) X = np.unique(X) Y = np.unique(Y) lon = np.array(lon).reshape(len(Y), len(X)) lat = np.array(lat).reshape(len(Y), len(X)) inc = np.array(inc).reshape(len(Y), len(X)) ele = np.array(ele).reshape(len(Y), len(X)) incVRT = VRT(array=inc, lat=lat, lon=lon) eleVRT = VRT(array=ele, lat=lat, lon=lon) incVRT = incVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=2) eleVRT = eleVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=2) self.bandVRTs['incVRT'] = incVRT self.bandVRTs['eleVRT'] = eleVRT for key in calDict.keys(): xml = self.read_xml(calDict[key]) calibration_LUT_VRTs, longitude, latitude = ( self.get_LUT_VRTs(xml, 'calibrationVectorList', ['sigmaNought', 'betaNought', 'gamma', 'dn'] )) self.bandVRTs['LUT_sigmaNought_VRT_'+pol[key]] = ( calibration_LUT_VRTs['sigmaNought']. get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1)) self.bandVRTs['LUT_betaNought_VRT_'+pol[key]] = ( calibration_LUT_VRTs['betaNought']. get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1)) self.bandVRTs['LUT_gamma_VRT'] = calibration_LUT_VRTs['gamma'] self.bandVRTs['LUT_dn_VRT'] = calibration_LUT_VRTs['dn'] for key in noiseDict.keys(): xml = self.read_xml(noiseDict[key]) noise_LUT_VRT = self.get_LUT_VRTs(xml, 'noiseVectorList', ['noiseLut'])[0] self.bandVRTs['LUT_noise_VRT_'+pol[key]] = ( noise_LUT_VRT['noiseLut'].get_resized_vrt( self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1)) metaDict = [] bandNumberDict = {} bnmax = 0 for key in gdalDatasets.keys(): dsPath, dsName = os.path.split(mdsDict[key]) name = 'DN_%s' % pol[key] # A dictionary of band numbers is needed for the pixel function # bands further down. This is not the best solution. It would be # better to have a function in VRT that returns the number given a # band name. This function exists in Nansat but could perhaps be # moved to VRT? The existing nansat function could just call the # VRT one... bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] band = gdalDatasets[key].GetRasterBand(1) dtype = band.DataType metaDict.append({ 'src': { 'SourceFilename': mdsDict[key], 'SourceBand': 1, 'DataType': dtype, }, 'dst': { 'name': name, 'SourceTransferType': gdal.GetDataTypeName(dtype), 'dataType': 6, }, }) # add bands with metadata and corresponding values to the empty VRT self._create_bands(metaDict) ''' Calibration should be performed as s0 = DN^2/sigmaNought^2, where sigmaNought is from e.g. annotation/calibration/calibration-s1a-iw-grd-hh-20140811t151231-20140811t151301-001894-001cc7-001.xml, and DN is the Digital Numbers in the tiff files. Also the noise should be subtracted. See https://sentinel.esa.int/web/sentinel/sentinel-1-sar-wiki/-/wiki/Sentinel%20One/Application+of+Radiometric+Calibration+LUT ''' # Get look direction sat_heading = initial_bearing(longitude[:-1, :], latitude[:-1, :], longitude[1:, :], latitude[1:, :]) look_direction = scipy.ndimage.interpolation.zoom( np.mod(sat_heading + 90, 360), (np.shape(longitude)[0] / (np.shape(longitude)[0]-1.), 1)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT(array=look_direction_u, lat=latitude, lon=longitude) look_v_VRT = VRT(array=look_direction_v, lat=latitude, lon=longitude) lookVRT = VRT(lat=latitude, lon=longitude) lookVRT._create_band([{'SourceFilename': look_u_VRT.fileName, 'SourceBand': 1}, {'SourceFilename': look_v_VRT.fileName, 'SourceBand': 1}], {'PixelFunctionType': 'UVToDirectionTo'} ) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, eResampleAlg=1) # Store VRTs so that they are accessible later self.bandVRTs['look_u_VRT'] = look_u_VRT self.bandVRTs['look_v_VRT'] = look_v_VRT self.bandVRTs['lookVRT'] = lookVRT metaDict = [] # Add bands to full size VRT for key in pol: name = 'LUT_sigmaNought_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': {'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_' + pol[key]].fileName), 'SourceBand': 1 }, 'dst': {'name': name } }) name = 'LUT_noise_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.bandVRTs['LUT_noise_VRT_' + pol[key]].fileName, 'SourceBand': 1 }, 'dst': { 'name': name } }) name = 'look_direction' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.bandVRTs['lookVRT'].fileName, 'SourceBand': 1 }, 'dst': { 'wkv': 'sensor_azimuth_angle', 'name': name } }) for key in gdalDatasets.keys(): dsPath, dsName = os.path.split(mdsDict[key]) name = 'sigma0_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': [{'SourceFilename': self.fileName, 'SourceBand': bandNumberDict['DN_%s' % pol[key]], }, {'SourceFilename': (self.bandVRTs['LUT_noise_VRT_%s' % pol[key]].fileName), 'SourceBand': 1 }, {'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_%s' % pol[key]].fileName), 'SourceBand': 1 } ], 'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol[key], 'suffix': pol[key], }, }) name = 'beta0_%s' % pol[key] bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': [{'SourceFilename': self.fileName, 'SourceBand': bandNumberDict['DN_%s' % pol[key]] }, {'SourceFilename': (self.bandVRTs['LUT_noise_VRT_%s' % pol[key]].fileName), 'SourceBand': 1 }, {'SourceFilename': (self.bandVRTs['LUT_betaNought_VRT_%s' % pol[key]].fileName), 'SourceBand': 1 } ], 'dst': {'wkv': 'surface_backwards_brightness_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol[key], 'suffix': pol[key], }, }) self._create_bands(metaDict) # Add incidence angle as band name = 'incidence_angle' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = {'SourceFilename': self.bandVRTs['incVRT'].fileName, 'SourceBand': 1} dst = {'wkv': 'angle_of_incidence', 'name': name} self._create_band(src, dst) self.dataset.FlushCache() # Add elevation angle as band name = 'elevation_angle' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = {'SourceFilename': self.bandVRTs['eleVRT'].fileName, 'SourceBand': 1} dst = {'wkv': 'angle_of_elevation', 'name': name} self._create_band(src, dst) self.dataset.FlushCache() # Add sigma0_VV pp = [pol[key] for key in pol] if 'VV' not in pp and 'HH' in pp: name = 'sigma0_VV' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = [{'SourceFilename': self.fileName, 'SourceBand': bandNumberDict['DN_HH'], }, {'SourceFilename': (self.bandVRTs['LUT_noise_VRT_HH']. fileName), 'SourceBand': 1 }, {'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_HH']. fileName), 'SourceBand': 1, }, {'SourceFilename': self.bandVRTs['incVRT'].fileName, 'SourceBand': 1} ] dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV'} self._create_band(src, dst) self.dataset.FlushCache() # set time as acquisition start time n = Node.create(manifestXML) meta = n.node('metadataSection') for nn in meta.children: if nn.getAttribute('ID') == u'acquisitionPeriod': # set valid time self.dataset.SetMetadataItem( 'time_coverage_start', parse((nn.node('metadataWrap'). node('xmlData'). node('safe:acquisitionPeriod')['safe:startTime']) ).isoformat()) self.dataset.SetMetadataItem( 'time_coverage_end', parse((nn.node('metadataWrap'). node('xmlData'). node('safe:acquisitionPeriod')['safe:stopTime']) ).isoformat()) # Get dictionary describing the instrument and platform according to # the GCMD keywords mm = pti.get_gcmd_instrument('sar') ee = pti.get_gcmd_platform('sentinel-1a') # TODO: Validate that the found instrument and platform are indeed what we # want.... self.dataset.SetMetadataItem('instrument', json.dumps(mm)) self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs): ''' Create Radarsat2 VRT ''' fPathName, fExt = os.path.splitext(fileName) if zipfile.is_zipfile(fileName): # Open zip file using VSI fPath, fName = os.path.split(fPathName) fileName = '/vsizip/%s/%s' % (fileName, fName) if not 'RS' in fName[0:2]: raise WrongMapperError('Provided data is not Radarsat-2') gdalDataset = gdal.Open(fileName) gdalMetadata = gdalDataset.GetMetadata() #if it is not RADARSAT-2, return if (not gdalMetadata or not 'SATELLITE_IDENTIFIER' in gdalMetadata.keys()): raise WrongMapperError elif gdalMetadata['SATELLITE_IDENTIFIER'] != 'RADARSAT-2': raise WrongMapperError # read product.xml productXmlName = os.path.join(fileName, 'product.xml') productXml = self.read_xml(productXmlName) # Get additional metadata from product.xml rs2_0 = Node.create(productXml) rs2_1 = rs2_0.node('sourceAttributes') rs2_2 = rs2_1.node('radarParameters') if rs2_2['antennaPointing'].lower() == 'right': antennaPointing = 90 else: antennaPointing = -90 rs2_3 = rs2_1.node('orbitAndAttitude').node('orbitInformation') passDirection = rs2_3['passDirection'] # create empty VRT dataset with geolocation only VRT.__init__(self, gdalDataset) #define dictionary of metadata and band specific parameters pol = [] metaDict = [] # Get the subdataset with calibrated sigma0 only for dataset in gdalDataset.GetSubDatasets(): if dataset[1] == 'Sigma Nought calibrated': s0dataset = gdal.Open(dataset[0]) s0datasetName = dataset[0][:] band = s0dataset.GetRasterBand(1) s0datasetPol = band.GetMetadata()['POLARIMETRIC_INTERP'] for i in range(1, s0dataset.RasterCount+1): iBand = s0dataset.GetRasterBand(i) polString = iBand.GetMetadata()['POLARIMETRIC_INTERP'] suffix = polString # The nansat data will be complex # if the SAR data is of type 10 dtype = iBand.DataType if dtype == 10: # add intensity band metaDict.append( {'src': {'SourceFilename': ('RADARSAT_2_CALIB:SIGMA0:' + fileName + '/product.xml'), 'SourceBand': i, 'DataType': dtype}, 'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'intensity', 'SourceTransferType': gdal.GetDataTypeName(dtype), 'suffix': suffix, 'polarization': polString, 'dataType': 6}}) # modify suffix for adding the compled band below suffix = polString+'_complex' pol.append(polString) metaDict.append( {'src': {'SourceFilename': ('RADARSAT_2_CALIB:SIGMA0:' + fileName + '/product.xml'), 'SourceBand': i, 'DataType': dtype}, 'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'suffix': suffix, 'polarization': polString}}) if dataset[1] == 'Beta Nought calibrated': b0dataset = gdal.Open(dataset[0]) b0datasetName = dataset[0][:] for j in range(1, b0dataset.RasterCount+1): jBand = b0dataset.GetRasterBand(j) polString = jBand.GetMetadata()['POLARIMETRIC_INTERP'] if polString == s0datasetPol: b0datasetBand = j ############################### # Add SAR look direction ############################### d = Domain(ds=gdalDataset) lon, lat = d.get_geolocation_grids(100) ''' (GDAL?) Radarsat-2 data is stored with maximum latitude at first element of each column and minimum longitude at first element of each row (e.g. np.shape(lat)=(59,55) -> latitude maxima are at lat[0,:], and longitude minima are at lon[:,0]) In addition, there is an interpolation error for direct estimate along azimuth. We therefore estimate the heading along range and add 90 degrees to get the "satellite" heading. ''' if str(passDirection).upper() == 'DESCENDING': sat_heading = initial_bearing(lon[:, :-1], lat[:, :-1], lon[:, 1:], lat[:, 1:]) + 90 elif str(passDirection).upper() == 'ASCENDING': sat_heading = initial_bearing(lon[:, 1:], lat[:, 1:], lon[:, :-1], lat[:, :-1]) + 90 else: print 'Can not decode pass direction: ' + str(passDirection) # Calculate SAR look direction look_direction = sat_heading + antennaPointing # Interpolate to regain lost row look_direction = np.mod(look_direction, 360) look_direction = scipy.ndimage.interpolation.zoom( look_direction, (1, 11./10.)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT(array=look_direction_u, lat=lat, lon=lon) look_v_VRT = VRT(array=look_direction_v, lat=lat, lon=lon) # Note: If incidence angle and look direction are stored in # same VRT, access time is about twice as large lookVRT = VRT(lat=lat, lon=lon) lookVRT._create_band( [{'SourceFilename': look_u_VRT.fileName, 'SourceBand': 1}, {'SourceFilename': look_v_VRT.fileName, 'SourceBand': 1}], {'PixelFunctionType': 'UVToDirectionTo'}) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize, gdalDataset.RasterYSize) # Store VRTs so that they are accessible later self.bandVRTs['look_u_VRT'] = look_u_VRT self.bandVRTs['look_v_VRT'] = look_v_VRT self.bandVRTs['lookVRT'] = lookVRT # Add band to full sized VRT lookFileName = self.bandVRTs['lookVRT'].fileName metaDict.append({'src': {'SourceFilename': lookFileName, 'SourceBand': 1}, 'dst': {'wkv': 'sensor_azimuth_angle', 'name': 'look_direction'}}) ############################### # Create bands ############################### self._create_bands(metaDict) ################################################### # Add derived band (incidence angle) calculated # using pixel function "BetaSigmaToIncidence": ################################################### src = [{'SourceFilename': b0datasetName, 'SourceBand': b0datasetBand, 'DataType': dtype}, {'SourceFilename': s0datasetName, 'SourceBand': 1, 'DataType': dtype}] dst = {'wkv': 'angle_of_incidence', 'PixelFunctionType': 'BetaSigmaToIncidence', 'SourceTransferType': gdal.GetDataTypeName(dtype), '_FillValue': -10000, # NB: this is also hard-coded in # pixelfunctions.c 'dataType': 6, 'name': 'incidence_angle'} self._create_band(src, dst) self.dataset.FlushCache() ################################################################### # Add sigma0_VV - pixel function of sigma0_HH and beta0_HH # incidence angle is calculated within pixel function # It is assummed that HH is the first band in sigma0 and # beta0 sub datasets ################################################################### if 'VV' not in pol and 'HH' in pol: s0datasetNameHH = pol.index('HH')+1 src = [{'SourceFilename': s0datasetName, 'SourceBand': s0datasetNameHH, 'DataType': 6}, {'SourceFilename': b0datasetName, 'SourceBand': b0datasetBand, 'DataType': 6}] dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sigma0HHBetaToSigma0VV', 'polarization': 'VV', 'suffix': 'VV'} self._create_band(src, dst) self.dataset.FlushCache() ############################################ # Add SAR metadata ############################################ if antennaPointing == 90: self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT') if antennaPointing == -90: self.dataset.SetMetadataItem('ANTENNA_POINTING', 'LEFT') self.dataset.SetMetadataItem('ORBIT_DIRECTION', str(passDirection).upper()) # set valid time self.dataset.SetMetadataItem('time_coverage_start', (parse(gdalMetadata['FIRST_LINE_TIME']). isoformat())) self.dataset.SetMetadataItem('time_coverage_end', (parse(gdalMetadata['LAST_LINE_TIME']). isoformat())) # Get dictionary describing the instrument and platform according to # the GCMD keywords mm = pti.get_gcmd_instrument('sar') ee = pti.get_gcmd_platform('radarsat-2') # TODO: Validate that the found instrument and platform are indeed what we # want.... self.dataset.SetMetadataItem('instrument', json.dumps(mm)) self.dataset.SetMetadataItem('platform', json.dumps(ee)) self._add_swath_mask_band()
def __init__(self, filename, gdalDataset, gdalMetadata, fast=False, fixgcp=True, **kwargs): if not os.path.split(filename.rstrip('/'))[1][:3] in ['S1A', 'S1B']: raise WrongMapperError('%s: Not Sentinel 1A or 1B' %filename) if not IMPORT_SCIPY: raise NansatReadError('Sentinel-1 data cannot be read because scipy is not installed') if zipfile.is_zipfile(filename): zz = zipfile.PyZipFile(filename) # Assuming the file names are consistent, the polarization # dependent data should be sorted equally such that we can use the # same indices consistently for all the following lists # THIS IS NOT THE CASE... mds_files = ['/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'measurement/s1' in fn] calibration_files = ['/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'annotation/calibration/calibration-s1' in fn] noise_files = ['/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'annotation/calibration/noise-s1' in fn] annotation_files = ['/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'annotation/s1' in fn] manifest_files = ['/vsizip/%s/%s' % (filename, fn) for fn in zz.namelist() if 'manifest.safe' in fn] zz.close() else: mds_files = glob.glob('%s/measurement/s1*' % filename) calibration_files = glob.glob('%s/annotation/calibration/calibration-s1*' % filename) noise_files = glob.glob('%s/annotation/calibration/noise-s1*' % filename) annotation_files = glob.glob('%s/annotation/s1*' % filename) manifest_files = glob.glob('%s/manifest.safe' % filename) if (not mds_files or not calibration_files or not noise_files or not annotation_files or not manifest_files): raise WrongMapperError(filename) # convert list of MDS files into dictionary. Keys - polarizations in upper case. mds_files = {os.path.basename(ff).split('-')[3].upper():ff for ff in mds_files} polarizations = list(mds_files.keys()) # read annotation files self.annotation_data = self.read_annotation(annotation_files) if not fast and fixgcp: self.correct_geolocation_data() # read manifest file manifest_data = self.read_manifest_data(manifest_files[0]) # very fast constructor without any bands only with some metadata and geolocation self._init_empty(manifest_data, self.annotation_data) # skip adding bands in the fast mode and RETURN if fast: return # Open data files with GDAL gdalDatasets = {} for pol in polarizations: gdalDatasets[pol] = gdal.Open(mds_files[pol]) if not gdalDatasets[pol]: raise WrongMapperError('%s: No Sentinel-1 datasets found' % mds_files[pol]) # Check metadata to confirm it is Sentinel-1 L1 metadata = gdalDatasets[polarizations[0]].GetMetadata() # create full size VRTs with incidenceAngle and elevationAngle annotation_vrts = self.vrts_from_arrays(self.annotation_data, ['incidenceAngle', 'elevationAngle']) self.band_vrts.update(annotation_vrts) # create full size VRTS with calibration LUT calibration_names = ['sigmaNought', 'betaNought'] calibration_list_tag = 'calibrationVectorList' for calibration_file in calibration_files: pol = '_' + os.path.basename(calibration_file).split('-')[4].upper() xml = self.read_vsi(calibration_file) calibration_data = self.read_calibration(xml, calibration_list_tag, calibration_names, pol) calibration_vrts = self.vrts_from_arrays(calibration_data, calibration_names, pol, True, 1) self.band_vrts.update(calibration_vrts) # create full size VRTS with noise LUT for noise_file in noise_files: pol = '_' + os.path.basename(noise_file).split('-')[4].upper() xml = self.read_vsi(noise_file) if '<noiseVectorList' in xml: noise_list_tag = 'noiseVectorList' noise_name = 'noiseLut' elif '<noiseRangeVectorList' in xml: noise_list_tag = 'noiseRangeVectorList' noise_name = 'noiseRangeLut' noise_data = self.read_calibration(xml, noise_list_tag, [noise_name], pol) noise_vrts = self.vrts_from_arrays(noise_data, [noise_name], pol, True, 1) self.band_vrts.update(noise_vrts) #### Create metaDict: dict with metadata for all bands metaDict = [] bandNumberDict = {} bnmax = 0 for pol in polarizations: dsPath, dsName = os.path.split(mds_files[pol]) name = 'DN_%s' % pol # A dictionary of band numbers is needed for the pixel function # bands further down. This is not the best solution. It would be # better to have a function in VRT that returns the number given a # band name. This function exists in Nansat but could perhaps be # moved to VRT? The existing nansat function could just call the # VRT one... bandNumberDict[name] = bnmax + 1 bnmax = bandNumberDict[name] band = gdalDatasets[pol].GetRasterBand(1) dtype = band.DataType metaDict.append({ 'src': { 'SourceFilename': mds_files[pol], 'SourceBand': 1, 'DataType': dtype, }, 'dst': { 'name': name, }, }) # add bands with metadata and corresponding values to the empty VRT self.create_bands(metaDict) ''' Calibration should be performed as s0 = DN^2/sigmaNought^2, where sigmaNought is from e.g. annotation/calibration/calibration-s1a-iw-grd-hh-20140811t151231-20140811t151301-001894-001cc7-001.xml, and DN is the Digital Numbers in the tiff files. Also the noise should be subtracted. See https://sentinel.esa.int/web/sentinel/sentinel-1-sar-wiki/-/wiki/Sentinel%20One/Application+of+Radiometric+Calibration+LUT The noise correction/subtraction is implemented in an independent package "sentinel1denoised" See https://github.com/nansencenter/sentinel1denoised ''' # Get look direction longitude, latitude = self.transform_points(calibration_data['pixel'].flatten(), calibration_data['line'].flatten()) longitude.shape = calibration_data['pixel'].shape latitude.shape = calibration_data['pixel'].shape sat_heading = initial_bearing(longitude[:-1, :], latitude[:-1, :], longitude[1:, :], latitude[1:, :]) look_direction = scipy.ndimage.interpolation.zoom( np.mod(sat_heading + 90, 360), (np.shape(longitude)[0] / (np.shape(longitude)[0]-1.), 1)) # Decompose, to avoid interpolation errors around 0 <-> 360 look_direction_u = np.sin(np.deg2rad(look_direction)) look_direction_v = np.cos(np.deg2rad(look_direction)) look_u_VRT = VRT.from_array(look_direction_u) look_v_VRT = VRT.from_array(look_direction_v) lookVRT = VRT.from_lonlat(longitude, latitude) lookVRT.create_band([{'SourceFilename': look_u_VRT.filename, 'SourceBand': 1}, {'SourceFilename': look_v_VRT.filename, 'SourceBand': 1}], {'PixelFunctionType': 'UVToDirectionTo'} ) # Blow up to full size lookVRT = lookVRT.get_resized_vrt(self.dataset.RasterXSize, self.dataset.RasterYSize, 1) # Store VRTs so that they are accessible later self.band_vrts['look_u_VRT'] = look_u_VRT self.band_vrts['look_v_VRT'] = look_v_VRT self.band_vrts['lookVRT'] = lookVRT metaDict = [] # Add bands to full size VRT for pol in polarizations: name = 'sigmaNought_%s' % pol bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': {'SourceFilename': (self.band_vrts[name].filename), 'SourceBand': 1 }, 'dst': {'name': name } }) name = 'noise_%s' % pol bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.band_vrts['%s_%s' % (noise_name, pol)].filename, 'SourceBand': 1 }, 'dst': { 'name': name } }) name = 'look_direction' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append({ 'src': { 'SourceFilename': self.band_vrts['lookVRT'].filename, 'SourceBand': 1 }, 'dst': { 'wkv': 'sensor_azimuth_angle', 'name': name } }) for pol in polarizations: dsPath, dsName = os.path.split(mds_files[pol]) name = 'sigma0_%s' % pol bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': [{'SourceFilename': self.filename, 'SourceBand': bandNumberDict['DN_%s' % pol], }, {'SourceFilename': self.band_vrts['sigmaNought_%s' % pol].filename, 'SourceBand': 1 } ], 'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol, 'suffix': pol, }, }) name = 'beta0_%s' % pol bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] metaDict.append( {'src': [{'SourceFilename': self.filename, 'SourceBand': bandNumberDict['DN_%s' % pol] }, {'SourceFilename': self.band_vrts['betaNought_%s' % pol].filename, 'SourceBand': 1 } ], 'dst': {'wkv': 'surface_backwards_brightness_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Calibration', 'polarization': pol, 'suffix': pol, }, }) self.create_bands(metaDict) # Add incidence angle as band name = 'incidence_angle' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = {'SourceFilename': self.band_vrts['incidenceAngle'].filename, 'SourceBand': 1} dst = {'wkv': 'angle_of_incidence', 'name': name} self.create_band(src, dst) self.dataset.FlushCache() # Add elevation angle as band name = 'elevation_angle' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = {'SourceFilename': self.band_vrts['elevationAngle'].filename, 'SourceBand': 1} dst = {'wkv': 'angle_of_elevation', 'name': name} self.create_band(src, dst) self.dataset.FlushCache() # Add sigma0_VV if 'VV' not in polarizations and 'HH' in polarizations: name = 'sigma0_VV' bandNumberDict[name] = bnmax+1 bnmax = bandNumberDict[name] src = [{'SourceFilename': self.filename, 'SourceBand': bandNumberDict['DN_HH'], }, {'SourceFilename': (self.band_vrts['sigmaNought_HH']. filename), 'SourceBand': 1, }, {'SourceFilename': self.band_vrts['incidenceAngle'].filename, 'SourceBand': 1} ] dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', 'PixelFunctionType': 'Sentinel1Sigma0HHToSigma0VV', 'polarization': 'VV', 'suffix': 'VV'} self.create_band(src, dst) self.dataset.FlushCache()