def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create LANDSAT VRT '''
# try to open .tar or .tar.gz or .tgz file with tar
tarFile = tarfile.open(fileName)
tarNames = tarFile.getnames()
#print tarNames
metaDict = []
for tarName in tarNames:
if ((tarName[0] == 'L' or tarName[0] == 'M')
and (tarName[-4:] == '.TIF' or tarName[-4:] == '.tif')):
#print tarName
bandNo = tarName[-6:-4]
metaDict.append({
'src': {
'SourceFilename':
'/vsitar/%s/%s' % (fileName, tarName),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'suffix': bandNo
}
})
#print metaDict
sizeDiffBands = []
for iFile in range(len(metaDict)):
tmpName = metaDict[iFile]['src']['SourceFilename']
gdalDatasetTmp = gdal.Open(tmpName)
if iFile == 0:
gdalDatasetTmp0 = gdalDatasetTmp
xSize = gdalDatasetTmp.RasterXSize
ySize = gdalDatasetTmp.RasterYSize
elif xSize != gdalDatasetTmp.RasterXSize or ySize != gdalDatasetTmp.RasterYSize:
sizeDiffBands.append(iFile)
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDatasetTmp0, **kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# 8th band of LANDSAT8 is a double size band. Reduce the size to same as the 1st band.
if len(sizeDiffBands) != 0:
vrtXML = self.read_xml()
node0 = Node.create(vrtXML)
for iBand in sizeDiffBands:
iNodeDstRect = node0.nodeList('VRTRasterBand')[iBand].node(
'DstRect')
iNodeDstRect.replaceAttribute('xSize', str(xSize))
iNodeDstRect.replaceAttribute('ySize', str(ySize))
self.write_xml(str(node0.rawxml()))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create NCEP VRT '''
if gdalDataset.GetGeoTransform() != (-0.25, 0.5, 0.0, 90.25, 0.0, -0.5) or gdalDataset.RasterCount != 9: # Not water proof
raise AttributeError("NCEP BAD MAPPER")
metaDict = [
{'src': {'SourceFilename': fileName,
'SourceBand': 8},
'dst': {'wkv': 'eastward_wind',
'name': 'east_wind',
'height': '10 m'}},
{'src': {'SourceFilename': fileName,
'SourceBand': 9},
'dst': {'wkv': 'northward_wind',
'name': 'north_wind',
'height': '10 m'}},
{'src': [{'SourceFilename': fileName,
'SourceBand': 8},
{'SourceFilename': fileName,
'SourceBand': 9}],
'dst': {'wkv': 'wind_speed',
'PixelFunctionType': 'UVToMagnitude',
'name': 'windspeed',
'height': '2 m'}},
{'src': [{'SourceFilename': fileName,
'SourceBand': 8},
{'SourceFilename': fileName,
'SourceBand': 9}],
'dst': {'wkv': 'wind_from_direction',
'PixelFunctionType': 'UVToDirectionFrom',
'name': 'winddirection',
'height': '2 m'}},
{'src': {'SourceFilename': fileName,
'SourceBand': 6},
'dst': {'wkv': 'air_temperature',
'name': 'air_t',
'height': '2 m'}}
]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time from the GRIB file to dataset
validTime = gdalDataset.GetRasterBand(8).GetMetadata()['GRIB_VALID_TIME']
self._set_time(datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0])))
return
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create LANDSAT VRT '''
# try to open .tar or .tar.gz or .tgz file with tar
tarFile = tarfile.open(fileName)
tarNames = tarFile.getnames()
#print tarNames
metaDict = []
for tarName in tarNames:
if ((tarName[0] == 'L' or tarName[0] == 'M') and
(tarName[-4:] == '.TIF' or tarName[-4:] == '.tif')):
#print tarName
bandNo = tarName[-6:-4]
metaDict.append({
'src': {'SourceFilename': '/vsitar/%s/%s' % (fileName,
tarName),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'suffix': bandNo}})
#print metaDict
sizeDiffBands = []
for iFile in range(len(metaDict)):
tmpName = metaDict[iFile]['src']['SourceFilename']
gdalDatasetTmp = gdal.Open(tmpName)
if iFile == 0:
gdalDatasetTmp0 = gdalDatasetTmp
xSize = gdalDatasetTmp.RasterXSize
ySize = gdalDatasetTmp.RasterYSize
elif (xSize != gdalDatasetTmp.RasterXSize or
ySize != gdalDatasetTmp.RasterYSize):
sizeDiffBands.append(iFile)
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDatasetTmp0)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# 8th band of LANDSAT8 is a double size band.
# Reduce the size to same as the 1st band.
if len(sizeDiffBands) != 0:
vrtXML = self.read_xml()
node0 = Node.create(vrtXML)
for iBand in sizeDiffBands:
iBandNode = node0.nodeList('VRTRasterBand')[iBand]
iNodeDstRect = iBandNode.node('DstRect')
iNodeDstRect.replaceAttribute('xSize', str(xSize))
iNodeDstRect.replaceAttribute('ySize', str(ySize))
self.write_xml(str(node0.rawxml()))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create MODIS_L1 VRT '''
# raise error in case of not GOCI L1B
title = gdalMetadata['HDFEOS_POINTS_Scene_Header_Scene_Title']
assert title == 'GOCI Level-1B Data'
# set GOCI projection parameters
lat_0 = gdalMetadata['HDFEOS_POINTS_Map_Projection_Central_Latitude_(parallel)']
lon_0 = gdalMetadata['HDFEOS_POINTS_Map_Projection_Central_Longitude_(meridian)']
rasterXSize = int(gdalMetadata['HDFEOS_POINTS_Scene_Header_number_of_columns'].split(' ')[0])
rasterYSize = int(gdalMetadata['HDFEOS_POINTS_Scene_Header_number_of_rows'].split(' ')[0])
proj4 = '+proj=ortho +lat_0=%s +lon_0=%s units=m +ellps=WGS84 +datum=WGS84 +no_defs' % (lat_0, lon_0)
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4)
projection = srs.ExportToWkt()
geoTransform = (-1391500.0, 500.0, 0.0, 1349500.0, 0.0, -500.0)
# create empty VRT dataset with georeference only
VRT.__init__(self, srcGeoTransform=geoTransform,
srcProjection=projection,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
# add bands from subdatasets
subDatasets = gdalDataset.GetSubDatasets()
metaDict = []
wavelengths = ['412', '443', '490', '555', '660', '680', '745', '865']
for subDSI in range(8):
metaEntry = {'src': {'SourceFilename': subDatasets[subDSI][0],
'SourceBand': 1,
'ScaleRatio': 1e-6},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': wavelengths[subDSI],
'suffix': wavelengths[subDSI]}}
metaDict.append(metaEntry)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create SMOS VRT '''
# check extension
fName = os.path.split(fileName)[1]
fExt = os.path.splitext(fileName)[1]
if fExt == '.MAT' or fExt == '.mat' and 'OSUDP2' in fName:
# load file
matFile = loadmat(fileName)
else:
AttributeError("SMOS BAD MAPPER")
# get geolocation
geolocArray = matFile['geolocation'][0]
srcProj4 = '+proj=stere +lon_0=%f +lat_0=%f +datum=WGS84 +ellps=WGS84 +units=km +no_defs' % (geolocArray[0], geolocArray[1])
srcProjection = osr.SpatialReference()
srcProjection.ImportFromProj4(srcProj4)
srcProjection = srcProjection.ExportToWkt()
srcGeotransform = (geolocArray[2], geolocArray[4], 0,
geolocArray[3], 0, geolocArray[5])
lon = matFile['longitude']
#lat = matFile['latitude']
srcRasterYSize, srcRasterXSize = lon.shape
# create VRT from lat/lon
# VRT.__init__(self, lon=lon, lat=lat)
VRT.__init__(self,
srcGeoTransform=srcGeotransform,
srcProjection=srcProjection,
srcRasterXSize=srcRasterXSize,
srcRasterYSize=srcRasterYSize)
# add the following variables
varNames = ['SSS1', 'SSS2', 'SSS3', 'SST',
'Sigma_SSS1', 'Sigma_SSS2', 'Sigma_SSS3',
'Control_Flags_1', 'Control_Flags_2',
'Control_Flags_3', 'Control_Flags_4',
'Science_Flags_1', 'Science_Flags_2',
'Science_Flags_3', 'Science_Flags_4']
self.varVRTs = {}
metaDict = []
for varName in varNames:
var = matFile[varName]
self.varVRTs[varName] = VRT(array=var)
metaDict.append({'src': {'SourceFilename':
self.varVRTs[varName].fileName,
'sourceBand': 1},
'dst': {'name': varName}})
# create mask
cloudBits = [2, 3, 4, 5, 6]
maxSigma = 3.0
mask = np.zeros(lon.shape, 'uint16')
mask[:] = 128
mask[np.isnan(matFile['SSS1'])] = 0
mask[matFile['Sigma_SSS1'] > maxSigma] = 1
mask[matFile['Sigma_SSS2'] > maxSigma] = 1
mask[matFile['Sigma_SSS3'] > maxSigma] = 1
for cloudBit in cloudBits:
for cfi in range(1, 5):
bitMap = np.bitwise_and(matFile['Control_Flags_%d' % cfi],
np.power(2, cloudBit))
mask[bitMap > 0] = 1
self.varVRTs['mask'] = VRT(array=mask)
metaDict.append({'src': {'SourceFilename':
self.varVRTs['mask'].fileName,
'sourceBand': 1},
'dst': {'name': 'mask'}})
self.logger.debug('metaDict: %s' % metaDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create MER1 VRT '''
# get ENVISAT MPH_PRODUCT
product = gdalMetadata.get("MPH_PRODUCT")
if product[0:9] != "MER_FRS_1" and product[0:9] != "MER_RR__1":
raise AttributeError("MERIS_L1 BAD MAPPER")
kwDict = {'geolocation' : True}
# choose kwargs for envisat and asar and change keyname
for key in kwargs:
if key.startswith('envisat') or key.startswith('meris'):
keyName = key.replace('envisat_', '').replace('meris_', '')
kwDict[keyName] = kwargs[key]
else:
kwDict[key] = kwargs[key]
# init ADS parameters
Envisat.__init__(self, fileName, product[0:4], **kwDict)
metaDict = [
{'src': {'SourceFilename': fileName, 'SourceBand': 1}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '413'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 2}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '443'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 3}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '490'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 4}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '510'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 5}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '560'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 6}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '620'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 7}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '665'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 8}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '681'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 9}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '709'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 10}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '753'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 11}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '761'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 12}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '778'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 13}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '864'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 14}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '849'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 15}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': '900'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 16, 'DataType': 1}, 'dst': {'wkv': 'quality_flags', 'suffix': 'l1'}}
]
# add DataType into 'src' and suffix into 'dst'
for bandDict in metaDict:
if 'DataType' not in bandDict['src']:
bandDict['src']['DataType'] = 2 # default for meris L1 DataType UInt16
if bandDict['dst'].has_key('wavelength'):
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
# get scaling GADS from header
scales = self.read_scaling_gads(range(7, 22))
# set scale factor to the band metadata (only radiances)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
# get list with resized VRTs from ADS
self.adsVRTs = self.get_ads_vrts(gdalDataset, ['sun zenith angles', 'sun azimuth angles', 'zonal winds', 'meridional winds'])
# add bands from the ADS VRTs
for adsVRT in self.adsVRTs:
metaDict.append({'src': {'SourceFilename': adsVRT.fileName,
'SourceBand': 1},
'dst': {'name': adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name'),
'units': adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units')}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if self.d['geolocation']:
self.add_geolocation_from_ads(gdalDataset)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create CSKS VRT '''
if fileName[0:4] != "CSKS":
raise AttributeError("COSMO-SKYMED BAD MAPPER")
# Get coordinates
bottom_left_lon = float(
gdalMetadata['Estimated_Bottom_Left_Geodetic_Coordinates'].split(
' ')[1])
bottom_left_lat = float(
gdalMetadata['Estimated_Bottom_Left_Geodetic_Coordinates'].split(
' ')[0])
bottom_right_lon = float(
gdalMetadata['Estimated_Bottom_Right_Geodetic_Coordinates'].split(
' ')[1])
bottom_right_lat = float(
gdalMetadata['Estimated_Bottom_Right_Geodetic_Coordinates'].split(
' ')[0])
top_left_lon = float(
gdalMetadata['Estimated_Top_Left_Geodetic_Coordinates'].split(
' ')[1])
top_left_lat = float(
gdalMetadata['Estimated_Top_Left_Geodetic_Coordinates'].split(
' ')[0])
top_right_lon = float(
gdalMetadata['Estimated_Top_Right_Geodetic_Coordinates'].split(
' ')[1])
top_right_lat = float(
gdalMetadata['Estimated_Top_Right_Geodetic_Coordinates'].split(
' ')[0])
center_lon = float(
gdalMetadata['Scene_Centre_Geodetic_Coordinates'].split(' ')[1])
center_lat = float(
gdalMetadata['Scene_Centre_Geodetic_Coordinates'].split(' ')[0])
# Get sub-datasets
subDatasets = gdalDataset.GetSubDatasets()
# Get file names from dataset or subdataset
if subDatasets.__len__() == 1:
fileNames = [fileName]
else:
fileNames = [f[0] for f in subDatasets]
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'QLK':
fileNames.pop(i)
#print fileNames
subDataset = gdal.Open(fileNames[0])
# generate list of GCPs
gcps = []
# create GCP with X,Y,Z(?),pixel,line from lat/lon matrices
gcp = gdal.GCP(float(bottom_left_lon), float(bottom_left_lat), 0, 0, 0)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 0, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(bottom_right_lon), float(bottom_right_lat), 0,
subDataset.RasterXSize, 0)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 1, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_left_lon), float(top_left_lat), 0, 0,
subDataset.RasterYSize)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 2, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_right_lon), float(top_right_lat), 0,
subDataset.RasterXSize, subDataset.RasterYSize)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 3, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(center_lon), float(center_lat), 0,
int(np.round(subDataset.RasterXSize / 2.)),
int(round(subDataset.RasterYSize / 2.)))
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 4, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
# append GCPs and lat/lon projection to the vsiDataset
latlongSRS = osr.SpatialReference()
latlongSRS.ImportFromProj4(
"+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
latlongSRSWKT = latlongSRS.ExportToWkt()
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
srcGCPs=gcps,
srcGCPProjection=latlongSRSWKT)
#print self.fileName
# Read all bands later
#band='S01'
#res='SBI'
# Use only full size "original" datasets
for i, elem in enumerate(fileNames):
band_number = i
if fileNames[i][-3:] == 'SBI':
# Add real and imaginary raw counts as bands
src = {
'SourceFilename': fileNames[i],
'SourceBand': 1,
'DataType': gdal.GDT_Int16
}
dst = {
'dataType':
gdal.GDT_Float32,
'name':
'RawCounts_%s_real' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation']
}
self._create_band(src, dst)
src = {
'SourceFilename': fileNames[i],
'SourceBand': 2,
'DataType': gdal.GDT_Int16
}
dst = {
'dataType':
gdal.GDT_Float32,
'name':
'RawCounts_%s_imaginary' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation']
}
self._create_band(src, dst)
self.dataset.FlushCache()
for i, elem in enumerate(fileNames):
band_number = i
if fileNames[i][-3:] == 'SBI':
# Calculate sigma0 scaling factor
Rref = float(gdalMetadata['Reference_Slant_Range'])
Rexp = float(gdalMetadata['Reference_Slant_Range_Exponent'])
alphaRef = float(gdalMetadata['Reference_Incidence_Angle'])
F = float(gdalMetadata['Rescaling_Factor'])
K = float(gdalMetadata[fileNames[i][-7:-4] +
'_Calibration_Constant'])
Ftot = Rref**(2. * Rexp)
Ftot *= np.sin(alphaRef * np.pi / 180.0)
Ftot /= F**2.
Ftot /= K
#print Ftot
src = [{
'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2 * i + 1,
'ScaleRatio': np.sqrt(Ftot)
}, {
'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2 * i + 2,
'ScaleRatio': np.sqrt(Ftot)
}]
dst = {
'wkv':
'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType':
'RawcountsToSigma0_CosmoSkymed_SBI',
'polarisation':
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation'],
'name':
'sigma0_%s' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation'],
'SatelliteID':
gdalMetadata['Satellite_ID'],
'dataType':
gdal.GDT_Float32
}
#'pass': gdalMetadata[''] - I can't find this in the metadata...
self._create_band(src, dst)
self.dataset.FlushCache()
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_COUNT0=5, GCP_COUNT1=20, pixelStep=1,
lineStep=1, **kwargs):
''' Create VIIRS VRT '''
assert 'GMTCO_npp_' in fileName, 'viirs_l1 BAD MAPPER'
ifiledir = os.path.split(fileName)[0]
ifiles = glob.glob(ifiledir + 'SVM??_npp_d*_obpg_ops.h5')
ifiles.sort()
viirsWavelengths = [None, 412, 445, 488, 555, 672, 746, 865, 1240,
1378, 1610, 2250, 3700, 4050, 8550, 10736, 12013]
# create empty VRT dataset with geolocation only
xDatasetSource = 'HDF5:"%s"://All_Data/VIIRS-MOD-GEO-TC_All/Longitude' % fileName
xDatasetBand = 1
xDataset = gdal.Open(xDatasetSource)
VRT.__init__(self, xDataset)
metaDict = []
for ifile in ifiles:
ifilename = os.path.split(ifile)[1]
print ifilename
bNumber = int(ifilename[3:5])
print bNumber
bWavelength = viirsWavelengths[bNumber]
print bWavelength
SourceFilename = 'HDF5:"%s"://All_Data/VIIRS-M%d-SDR_All/Radiance' % (ifile, bNumber)
print SourceFilename
metaEntry = {'src': {'SourceFilename': SourceFilename,
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': str(bWavelength),
'suffix': str(bWavelength)}
}
metaDict.append(metaEntry)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
xVRTArray = xDataset.ReadAsArray()
xVRTArray = gaussian_filter(xVRTArray, 5).astype('float32')
xVRT = VRT(array=xVRTArray)
yDatasetSource = 'HDF5:"%s"://All_Data/VIIRS-MOD-GEO-TC_All/Latitude' % fileName
yDatasetBand = 1
yDataset = gdal.Open(yDatasetSource)
yVRTArray = yDataset.ReadAsArray()
yVRTArray = gaussian_filter(yVRTArray, 5).astype('float32')
yVRT = VRT(array=yVRTArray)
#self.add_geolocationArray(GeolocationArray(xDatasetSource, yDatasetSource))
#"""
# estimate pixel/line step
self.logger.debug('pixel/lineStep %f %f' % (pixelStep, lineStep))
# ==== ADD GCPs and Pojection ====
# get lat/lon matrices
longitude = xVRT.dataset.GetRasterBand(1).ReadAsArray()
latitude = yVRT.dataset.GetRasterBand(1).ReadAsArray()
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT0))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT1))
self.logger.debug('gcpCount: %d %d %d %d, %d %d',
latitude.shape[0], latitude.shape[1],
GCP_COUNT0, GCP_COUNT1, step0, step1)
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0,
i1 * pixelStep, i0 * lineStep)
self.logger.debug('%d %d %d %f %f',
k, gcp.GCPPixel, gcp.GCPLine,
gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, latlongSRS.ExportToWkt())
# remove geolocation array
self.remove_geolocationArray()
def __init__(self, fileName, gdalDataset, gdalMetadata,
geolocation=False, zoomSize=500, step=1, **kwargs):
''' Create MER1 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
geolocation : bool (default is False)
if True, add gdal geolocation
zoomSize: int (used in envisat.py)
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int (used in envisat.py)
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
'''
# get ENVISAT MPH_PRODUCT
product = gdalMetadata.get("MPH_PRODUCT")
if product[0:9] != "MER_FRS_1" and product[0:9] != "MER_RR__1":
raise AttributeError("MERIS_L1 BAD MAPPER")
# init ADS parameters
Envisat.__init__(self, fileName, product[0:4])
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '413'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '443'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 3},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '490'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 4},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '510'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 5},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '560'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 6},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '620'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 7},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '665'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 8},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '681'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 9},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '709'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 10},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '753'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 11},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '761'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 12},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '778'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 13},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '864'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 14},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '849'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 15},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '900'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 16,
'DataType': 1},
'dst': {'wkv': 'quality_flags', 'suffix': 'l1'}}
]
# add DataType into 'src' and suffix into 'dst'
for bandDict in metaDict:
if 'DataType' not in bandDict['src']:
bandDict['src']['DataType'] = 2 # default for meris L1 DataType UInt16
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
# get scaling GADS from header
scales = self.read_scaling_gads(range(7, 22))
# set scale factor to the band metadata (only radiances)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
# get list with resized VRTs from ADS
self.adsVRTs = self.get_ads_vrts(gdalDataset,
['sun zenith angles',
'sun azimuth angles',
'zonal winds',
'meridional winds'],
zoomSize=zoomSize,
step=step)
# add bands from the ADS VRTs
for adsVRT in self.adsVRTs:
metaDict.append({'src': {'SourceFilename': adsVRT.fileName,
'SourceBand': 1},
'dst': {'name': (adsVRT.dataset.GetRasterBand(1).
GetMetadataItem('name')),
'units': (adsVRT.dataset.GetRasterBand(1).
GetMetadataItem('units'))}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if geolocation:
self.add_geolocation_from_ads(gdalDataset,
zoomSize=zoomSize, step=step)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create MER1 VRT '''
# get ENVISAT MPH_PRODUCT
product = gdalMetadata.get("MPH_PRODUCT")
if product[0:9] != "MER_FRS_1" and product[0:9] != "MER_RR__1":
raise AttributeError("MERIS_L1 BAD MAPPER")
kwDict = {'geolocation': True}
# choose kwargs for envisat and asar and change keyname
for key in kwargs:
if key.startswith('envisat') or key.startswith('meris'):
keyName = key.replace('envisat_', '').replace('meris_', '')
kwDict[keyName] = kwargs[key]
else:
kwDict[key] = kwargs[key]
# init ADS parameters
Envisat.__init__(self, fileName, product[0:4], **kwDict)
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '413'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '443'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 3
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '490'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 4
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '510'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 5
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '560'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 6
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '620'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 7
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '665'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 8
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '681'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 9
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '709'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 10
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '753'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 11
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '761'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 12
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '778'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 13
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '864'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 14
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '849'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 15
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '900'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 16,
'DataType': 1
},
'dst': {
'wkv': 'quality_flags',
'suffix': 'l1'
}
}]
# add DataType into 'src' and suffix into 'dst'
for bandDict in metaDict:
if 'DataType' not in bandDict['src']:
bandDict['src'][
'DataType'] = 2 # default for meris L1 DataType UInt16
if bandDict['dst'].has_key('wavelength'):
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
# get scaling GADS from header
scales = self.read_scaling_gads(range(7, 22))
# set scale factor to the band metadata (only radiances)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
# get list with resized VRTs from ADS
self.adsVRTs = self.get_ads_vrts(gdalDataset, [
'sun zenith angles', 'sun azimuth angles', 'zonal winds',
'meridional winds'
])
# add bands from the ADS VRTs
for adsVRT in self.adsVRTs:
metaDict.append({
'src': {
'SourceFilename': adsVRT.fileName,
'SourceBand': 1
},
'dst': {
'name':
adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name'),
'units':
adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units')
}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if self.d['geolocation']:
self.add_geolocation_from_ads(gdalDataset)
def __init__(self, fileName, gdalDataset, gdalMetadata,
geolocation=False, zoomSize=500, step=1, **kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
geolocation : bool (default is False)
if True, add gdal geolocation
zoomSize: int (used in envisat.py)
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int (used in envisat.py)
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
'''
product = gdalMetadata["MPH_PRODUCT"]
if product[0:9] != "MER_FRS_2" and product[0:9] != "MER_RR__2":
raise AttributeError("MERIS_L2 BAD MAPPER")
Envisat.__init__(self, fileName, product[0:4])
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '412'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 2},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '443'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 3},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '490'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 4},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '510'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 5},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '560'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 6},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '620'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 7},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '665'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 8},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '680'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 9},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '708'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 10},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '753'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 11},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '761'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 12},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '778'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 13},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '864'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 15},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1_log', 'case': 'I'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 16},
'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 17},
'dst': {'wkv': 'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 18},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 22},
'dst': {'wkv': 'quality_flags', 'suffix': 'l2'}}
]
# add 'name' to 'parameters'
for bandDict in metaDict:
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
#get GADS from header
scales = self.read_scaling_gads(range(7, 20) + [20, 21, 22, 20])
offsets = self.read_scaling_gads(range(33, 46) + [46, 47, 48, 46])
# set scale/offset to the band metadata (only reflectance)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
bandDict['src']['ScaleOffset'] = str(offsets[i])
# add log10-scaled variables
metaDict += [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1', 'case': 'I',
'expression': 'np.power(10., self["chlor_a_1_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2', 'case': 'II',
'expression': 'np.power(10., self["chlor_a_2_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2', 'case': 'II',
'expression': 'np.power(10., self["cdom_a_2_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2', 'case': 'II',
'expression': 'np.power(10., self["tsm_2_log"])'}}
]
# get list with resized VRTs from ADS
self.adsVRTs = []
self.adsVRTs = self.get_ads_vrts(gdalDataset,
['sun zenith angles',
'sun azimuth angles',
'zonal winds',
'meridional winds'],
zoomSize=zoomSize,
step=step)
# add bands from the ADS VRTs
for adsVRT in self.adsVRTs:
metaDict.append({'src': {'SourceFilename': adsVRT.fileName,
'SourceBand': 1},
'dst': {'name': (adsVRT.dataset.GetRasterBand(1).
GetMetadataItem('name')),
'units': (adsVRT.dataset.GetRasterBand(1).
GetMetadataItem('units'))}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if geolocation:
self.add_geolocation_from_ads(gdalDataset,
zoomSize=zoomSize, step=step)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create SMOS VRT '''
# check extension
fName = os.path.split(fileName)[1]
fExt = os.path.splitext(fileName)[1]
if fExt == '.MAT' or fExt == '.mat' and 'OSUDP2' in fName:
# load file
matFile = loadmat(fileName)
else:
AttributeError("SMOS BAD MAPPER")
# get geolocation
geolocArray = matFile['geolocation'][0]
srcProj4 = '+proj=stere +lon_0=%f +lat_0=%f +datum=WGS84 +ellps=WGS84 +units=km +no_defs' % (
geolocArray[0], geolocArray[1])
srcProjection = osr.SpatialReference()
srcProjection.ImportFromProj4(srcProj4)
srcProjection = srcProjection.ExportToWkt()
srcGeotransform = (geolocArray[2], geolocArray[4], 0, geolocArray[3],
0, geolocArray[5])
lon = matFile['longitude']
#lat = matFile['latitude']
srcRasterYSize, srcRasterXSize = lon.shape
# create VRT from lat/lon
#VRT.__init__(self, lon=lon, lat=lat)
VRT.__init__(self,
srcGeoTransform=srcGeotransform,
srcProjection=srcProjection,
srcRasterXSize=srcRasterXSize,
srcRasterYSize=srcRasterYSize,
**kwargs)
# add the following variables
varNames = [
'SSS1', 'SSS2', 'SSS3', 'SST', 'Sigma_SSS1', 'Sigma_SSS2',
'Sigma_SSS3', 'Control_Flags_1', 'Control_Flags_2',
'Control_Flags_3', 'Control_Flags_4', 'Science_Flags_1',
'Science_Flags_2', 'Science_Flags_3', 'Science_Flags_4'
]
self.varVRTs = {}
metaDict = []
for varName in varNames:
var = matFile[varName]
self.varVRTs[varName] = VRT(array=var)
metaDict.append({
'src': {
'SourceFilename': self.varVRTs[varName].fileName,
'sourceBand': 1
},
'dst': {
'name': varName
}
})
# create mask
cloudBits = [2, 3, 4, 5, 6]
maxSigma = 3.0
mask = np.zeros(lon.shape, 'uint16')
mask[:] = 128
mask[np.isnan(matFile['SSS1'])] = 0
mask[matFile['Sigma_SSS1'] > maxSigma] = 1
mask[matFile['Sigma_SSS2'] > maxSigma] = 1
mask[matFile['Sigma_SSS3'] > maxSigma] = 1
for cloudBit in cloudBits:
for cfi in range(1, 5):
bitMap = np.bitwise_and(matFile['Control_Flags_%d' % cfi],
np.power(2, cloudBit))
mask[bitMap > 0] = 1
self.varVRTs['mask'] = VRT(array=mask)
metaDict.append({
'src': {
'SourceFilename': self.varVRTs['mask'].fileName,
'sourceBand': 1
},
'dst': {
'name': 'mask'
}
})
self.logger.debug('metaDict: %s' % metaDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT '''
# number of GCPs along each dimention
kwDict = {'GCP_COUNT' : 10}
# init ADS parameters
obpgL2Kwargs = {}
for key in kwargs:
if key.startswith('obpg_l2_'):
keyName = key.replace('obpg_l2_', '')
obpgL2Kwargs[keyName] = kwargs[key]
# modify the default values using input values
kwDict = set_defaults(kwDict, obpgL2Kwargs)
"""
Title=
Title=HMODISA Level-2 Data
Title=MERIS Level-2 Data
"""
titles = ['HMODISA Level-2 Data', 'MODISA Level-2 Data', 'MERIS Level-2 Data']
# should raise error in case of not obpg_l2 file
title = gdalMetadata["Title"]
assert title in titles, 'obpg_l2 BAD MAPPER'
# get subdataset and parse to VRT.__init__() for retrieving geo-metadata
# but NOT from longitude or latitude because it can be smaller!
subDatasets = gdalDataset.GetSubDatasets()
for subDataset in subDatasets:
if 'longitude' not in subDataset[1] and 'latitude' not in subDataset[1]:
gdalSubDataset = gdal.Open(subDataset[0])
break
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalSubDataset, **kwargs)
# parts of dictionary for all Reflectances
#dictRrs = {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '412'} }
# dictionary for all possible bands
allBandsDict = {
'Rrs': {'src': {}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air'}},
'Kd': {'src': {}, 'dst': {'wkv': 'volume_attenuation_coefficient_of_downwelling_radiative_flux_in_sea_water'}},
'chlor_a': {'src': {}, 'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water', 'case': 'I'}},
'cdom_index': {'src': {}, 'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter', 'case': 'II'}},
'sst': {'src': {}, 'dst': {'wkv': 'sea_surface_temperature'}},
'l2_flags': {'src': {'SourceType': 'SimpleSource', 'DataType': 4},
'dst': {'wkv': 'quality_flags', 'dataType': 4}},
'qual_sst': {'src': {'SourceType': 'SimpleSource', 'DataType': 4},
'dst': {'wkv': 'quality_flags', 'name': 'qual_sst', 'dataType': 4}},
'latitude': {'src': {}, 'dst': {'wkv': 'latitude'}},
'longitude': {'src': {}, 'dst': {'wkv': 'longitude'}},
}
# loop through available bands and generate metaDict (non fixed)
metaDict = []
bandNo = 0
for subDataset in subDatasets:
# get sub dataset name
subDatasetName = subDataset[1].split(' ')[1]
self.logger.debug('Subdataset: %s' % subDataset[1])
self.logger.debug('Subdataset name: "%s"' % subDatasetName)
# get wavelength if applicable, get dataset name without wavelength
try:
wavelength = int(subDatasetName.split('_')[-1])
except:
wavelength = None
subBandName = subDatasetName
else:
subBandName = subDatasetName.split('_')[0]
self.logger.debug('subBandName, wavelength: %s %s' % (subBandName, str(wavelength)))
if subBandName in allBandsDict:
# get name, slope, intercept
self.logger.debug('name: %s' % subBandName)
tmpSubDataset = gdal.Open(subDataset[0])
tmpSubMetadata = tmpSubDataset.GetMetadata()
slope = tmpSubMetadata.get('slope', '1')
intercept = tmpSubMetadata.get('intercept', '0')
self.logger.debug('slope, intercept: %s %s ' % (slope, intercept))
# create meta entry
metaEntry = {'src': {'SourceFilename': subDataset[0],
'sourceBand': 1,
'ScaleRatio': slope,
'ScaleOffset': intercept},
'dst': {}}
# add more to src
for srcKey in allBandsDict[subBandName]['src']:
metaEntry['src'][srcKey] = allBandsDict[subBandName]['src'][srcKey]
# add dst from allBandsDict
for dstKey in allBandsDict[subBandName]['dst']:
metaEntry['dst'][dstKey] = allBandsDict[subBandName]['dst'][dstKey]
# add wavelength, band name to dst
if wavelength is not None:
metaEntry['dst']['suffix'] = str(wavelength)
metaEntry['dst']['wavelength'] = str(wavelength)
# append band metadata to metaDict
self.logger.debug('metaEntry: %d => %s' % (bandNo, str(metaEntry)))
metaDict.append(metaEntry)
bandNo += 1
if subBandName == 'Rrs':
metaEntryRrsw = {
'src': [{
'SourceFilename': subDataset[0],
'SourceBand': 1,
'ScaleRatio': slope,
'ScaleOffset': intercept,
'DataType': 6}],
'dst': {
'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix': str(wavelength),
'wavelength': str(wavelength),
'PixelFunctionType': 'NormReflectanceToRemSensReflectance',
}}
# append band metadata to metaDict
self.logger.debug('metaEntry: %d => %s' % (bandNo, str(metaEntryRrsw)))
metaDict.append(metaEntryRrsw)
bandNo += 1
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
geolocationMetadata = gdalSubDataset.GetMetadata('GEOLOCATION')
xDatasetSource = geolocationMetadata['X_DATASET']
xDatasetBand = geolocationMetadata['X_BAND']
xDataset = gdal.Open(xDatasetSource)
xVRT = VRT(vrtDataset=xDataset)
yDatasetSource = geolocationMetadata['Y_DATASET']
yDatasetBand = geolocationMetadata['Y_BAND']
yDataset = gdal.Open(yDatasetSource)
yVRT = VRT(vrtDataset=yDataset)
# estimate pixel/line step
pixelStep = int(ceil(float(gdalSubDataset.RasterXSize) / float(xDataset.RasterXSize)))
lineStep = int(ceil(float(gdalSubDataset.RasterYSize) / float(xDataset.RasterYSize)))
self.logger.debug('pixel/lineStep %f %f' % (pixelStep, lineStep))
# ==== ADD GCPs and Pojection ====
# get lat/lon matrices
longitude = xVRT.dataset.GetRasterBand(1).ReadAsArray()
latitude = yVRT.dataset.GetRasterBand(1).ReadAsArray()
# add geolocation array
if ( longitude.min() >= -180 and longitude.max() <= 180 and
latitude.min() >= -90 and latitude.max() <= 90):
self.add_geolocationArray(GeolocationArray(xDatasetSource,
yDatasetSource,
pixelStep=pixelStep,
lineStep=lineStep))
else:
self.remove_geolocationArray()
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / kwDict['GCP_COUNT']))
step1 = max(1, int(float(latitude.shape[1]) / kwDict['GCP_COUNT']))
self.logger.debug('gcpCount: %d %d %f %d %d',
latitude.shape[0], latitude.shape[1],
kwDict['GCP_COUNT'], step0, step1)
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep, i0 * lineStep)
self.logger.debug('%d %d %d %f %f',
k, gcp.GCPPixel, gcp.GCPLine,
gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, latlongSRS.ExportToWkt())
# set TIME
startYear = int(gdalMetadata['Start Year'])
startDay = int(gdalMetadata['Start Day'])
startMillisec = int(gdalMetadata['Start Millisec'])
startDate = datetime(startYear, 1, 1) + timedelta(startDay, 0, 0, startMillisec)
self._set_time(startDate)
def __init__(self, fileName, gdalDataset, gdalMetadata, latlonGrid=None,
mask='', **kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
#import pdb;pdb.set_trace()
# test if input files is GLOBCOLOUR L3B
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
#print 'idir:', iDir, iFile, iFileName[0:5], iFileExt[0:8]
assert iFileName[0:4] == 'L3b_' and iFileExt == '.nc'
# define shape of GLOBCOLOUR grid
GLOBCOLOR_ROWS = 180 * 24
GLOBCOLOR_COLS = 360 * 24
# define lon/lat grids for projected var
if latlonGrid is None:
latlonGrid = np.mgrid[90:-90:4320j,
-180:180:8640j].astype('float32')
#latlonGrid = np.mgrid[80:50:900j, -10:30:1200j].astype('float16')
#latlonGrid = np.mgrid[47:39:300j, 25:45:500j].astype('float32')
# create empty VRT dataset with geolocation only
VRT.__init__(self, lon=latlonGrid[1], lat=latlonGrid[0])
# get list of similar (same date) files in the directory
simFilesMask = os.path.join(iDir, iFileName[0:30] + '*' + mask)
simFiles = glob.glob(simFilesMask)
simFiles.sort()
metaDict = []
self.varVRTs = []
mask = None
for simFile in simFiles:
print 'sim: ', simFile
f = netcdf_file(simFile)
# get iBinned, index for converting from binned into GLOBCOLOR-grid
colBinned = f.variables['col'][:]
rowBinned = f.variables['row'][:]
iBinned = (colBinned.astype('uint32') +
(rowBinned.astype('uint32') - 1) * GLOBCOLOR_COLS)
colBinned = None
rowBinned = None
# get iRawPro, index for converting from GLOBCOLOR-grid to latlonGrid
yRawPro = np.rint(1 + (GLOBCOLOR_ROWS - 1) *
(latlonGrid[0] + 90) / 180)
lon_step_Mat = 1 / np.cos(np.pi * latlonGrid[0] / 180.) / 24.
xRawPro = np.rint(1 + (latlonGrid[1] + 180) / lon_step_Mat)
iRawPro = xRawPro + (yRawPro - 1) * GLOBCOLOR_COLS
iRawPro[iRawPro < 0] = 0
iRawPro = np.rint(iRawPro).astype('uint32')
yRawPro = None
xRawPro = None
for varName in f.variables:
# find variable with _mean, eg CHL1_mean
if '_mean' in varName:
var = f.variables[varName]
break
# skip variable if no WKV is give in Globcolour
if varName not in self.varname2wkv:
continue
# get WKV
varWKV = self.varname2wkv[varName]
# read binned data
varBinned = var[:]
# convert to GLOBCOLOR grid
varRawPro = np.zeros([GLOBCOLOR_ROWS, GLOBCOLOR_COLS], 'float32')
varRawPro.flat[iBinned] = varBinned
# convert to latlonGrid
varPro = varRawPro.flat[iRawPro.flat[:]].reshape(iRawPro.shape)
#plt.imshow(varPro);plt.colorbar();plt.show()
# add mask band
if mask is None:
mask = np.zeros(varPro.shape, 'uint8')
mask[:] = 1
mask[varPro > 0] = 64
# add VRT with array with data from projected variable
self.varVRTs.append(VRT(array=mask))
# add metadata to the dictionary
metaDict.append({
'src': {'SourceFilename': self.varVRTs[-1].fileName,
'SourceBand': 1},
'dst': {'name': 'mask'}})
# add VRT with array with data from projected variable
self.varVRTs.append(VRT(array=varPro))
# add metadata to the dictionary
metaEntry = {
'src': {'SourceFilename': self.varVRTs[-1].fileName,
'SourceBand': 1},
'dst': {'wkv': varWKV, 'original_name': varName}}
# add wavelength for nLw
longName = 'Fully normalised water leaving radiance'
if longName in f.variables[varName].long_name:
simWavelength = varName.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add all metadata from NC-file
for attr in var._attributes:
metaEntry['dst'][attr] = var._attributes[attr]
metaDict.append(metaEntry)
# add Rrsw band
metaEntry2 = self.make_rrsw_meta_entry(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# add time
startDate = datetime.datetime(int(iFileName[4:8]),
int(iFileName[8:10]),
int(iFileName[10:12]))
self._set_time(startDate)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create MODIS_L1 VRT '''
#get 1st subdataset and parse to VRT.__init__() for retrieving geo-metadata
gdalSubDataset = gdal.Open(gdalDataset.GetSubDatasets()[0][0])
#list of available modis names:resolutions
modisResolutions = {
'MYD02QKM': 250,
'MOD02QKM': 250,
'MYD02HKM': 500,
'MOD02HKM': 500,
'MYD021KM': 1000,
'MOD021KM': 1000
}
#should raise error in case of not MODIS_L1
mResolution = modisResolutions[gdalMetadata["SHORTNAME"]]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalSubDataset, **kwargs)
subDsString = 'HDF4_EOS:EOS_SWATH:"%s":MODIS_SWATH_Type_L1B:%s'
#provide all mappings
metaDict250SF = ['EV_250_RefSB']
metaDict250 = [{
'src': {
'SourceFilename': subDsString % (fileName, 'EV_250_RefSB'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '645'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_250_RefSB'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '858'
}
}]
metaDict500SF = ['EV_250_Aggr500_RefSB', 'EV_500_RefSB']
metaDict500 = [{
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_250_Aggr500_RefSB'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '645'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_250_Aggr500_RefSB'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '858'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_500_RefSB'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '469'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_500_RefSB'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '555'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_500_RefSB'),
'SourceBand': 3
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1240'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_500_RefSB'),
'SourceBand': 4
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1640'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_500_RefSB'),
'SourceBand': 5
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '2130'
}
}]
metaDict1000SF = [
'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB',
'EV_1KM_Emissive'
]
metaDict1000 = [{
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_250_Aggr1km_RefSB'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '645'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_250_Aggr1km_RefSB'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '858'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '469'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '555'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 3
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1240'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 4
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1640'
}
}, {
'src': {
'SourceFilename':
subDsString % (fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 5
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '2130'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '412'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '443'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 3
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '488'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 4
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '531'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 5
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '551'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 6
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '667'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 7
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '667'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 8
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '678'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 9
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '678'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 10
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '748'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 11
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '869'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 12
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '905'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 13
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '936'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 14
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '940'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_RefSB'),
'SourceBand': 15
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1375'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '3750'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '3959'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 3
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '3959'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 4
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '4050'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 5
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '4465'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 6
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '4515'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 7
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '6715'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 8
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '7325'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 9
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '8550'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 10
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '9730'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 11
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '11030'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 12
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '12020'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 13
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '13335'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 14
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '13635'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 15
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '13935'
}
}, {
'src': {
'SourceFilename': subDsString % (fileName, 'EV_1KM_Emissive'),
'SourceBand': 16
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '14235'
}
}]
# get proper mapping depending on resolution
metaDict = {
250: metaDict250,
500: metaDict500,
1000: metaDict1000,
}[mResolution]
# get proper mapping depending on resolution
metaDictSF = {
250: metaDict250SF,
500: metaDict500SF,
1000: metaDict1000SF,
}[mResolution]
# read all scales/offsets
rScales = {}
rOffsets = {}
for sf in metaDictSF:
dsName = subDsString % (fileName, sf)
ds = gdal.Open(dsName)
rScales[dsName] = map(
float,
ds.GetMetadataItem('radiance_scales').split(','))
rOffsets[dsName] = map(
float,
ds.GetMetadataItem('radiance_offsets').split(','))
self.logger.debug('radiance_scales: %s' % str(rScales))
# add 'band_name' to 'parameters'
for bandDict in metaDict:
SourceFilename = bandDict['src']['SourceFilename']
SourceBand = bandDict['src']['SourceBand']
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
scale = rScales[SourceFilename][SourceBand - 1]
offset = rOffsets[SourceFilename][SourceBand - 1]
self.logger.debug('band, scale, offset: %s_%d %s %s' %
(SourceFilename, SourceBand, scale, offset))
bandDict['src']['ScaleRatio'] = scale
bandDict['src']['ScaleOffset'] = offset
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
productDate = gdalMetadata["RANGEBEGINNINGDATE"]
productTime = gdalMetadata["RANGEENDINGTIME"]
self._set_time(parse(productDate + ' ' + productTime))
self.remove_geolocationArray()
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' GLOBCOLOR L3M VRT '''
print "=>%s<=" % gdalMetadata['NC_GLOBAL#title']
if 'GlobColour' not in gdalMetadata['NC_GLOBAL#title']:
raise AttributeError("GlobColour BAD MAPPER")
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
print 'idir:', iDir, iFile, iFileName[0:30], iFileExt[0:8]
simFilesMask = os.path.join(iDir, iFileName[0:30] + '*')
simFiles = glob.glob(simFilesMask)
print 'simFilesMask, simFiles', simFilesMask, simFiles
metaDict = []
for simFile in simFiles:
print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
simSubDatasets = simSupDataset.GetSubDatasets()
simWKV = None
for simSubDataset in simSubDatasets:
if '_mean' in simSubDataset[0]:
simValidSupDataset = simSupDataset
simGdalDataset = gdal.Open(simSubDataset[0])
simBandMetadata = simGdalDataset.GetRasterBand(
1).GetMetadata()
simVarname = simBandMetadata['NETCDF_VARNAME']
# get WKV
print ' simVarname', simVarname
if simVarname in self.varname2wkv:
simWKV = self.varname2wkv[simVarname]
break
# skipp adding this similar file if it is not valid
if simWKV is None:
continue
metaEntry = {
'src': {
'SourceFilename': simSubDataset[0],
'SourceBand': 1
},
'dst': {
'wkv': simWKV,
'original_name': simVarname
}
}
# add wavelength and name
longName = simBandMetadata['long_name']
if 'Fully normalised water leaving radiance' in longName:
simWavelength = simVarname.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add band with rrsw
metaEntry2 = None
if simWKV == 'surface_upwelling_spectral_radiance_in_air_emerging_from_sea_water':
solarIrradiance = simBandMetadata['solar_irradiance']
metaEntry2 = {'src': metaEntry['src']}
metaEntry2['dst'] = {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix':
simWavelength,
'wavelength':
simWavelength,
#'expression': 'self["nLw_%s"] / %s / (0.52 + 1.7 * self["nLw_%s"] / %s)' % (simWavelength, solarIrradiance, simWavelength, solarIrradiance),
'expression':
'self["nLw_%s"] / %s' % (simWavelength, solarIrradiance),
}
print ' metaEntry', metaEntry
metaDict.append(metaEntry)
if metaEntry2 is not None:
print ' metaEntry2', metaEntry2
metaDict.append(metaEntry2)
print 'simSubDatasets', simValidSupDataset.GetSubDatasets()
for simSubDataset in simValidSupDataset.GetSubDatasets():
print 'simSubDataset', simSubDataset
if '_flags ' in simSubDataset[1]:
print ' mask simSubDataset', simSubDataset[1]
flags = gdal.Open(simSubDataset[0]).ReadAsArray()
mask = np.ones(flags.shape) * 64
mask[np.bitwise_and(flags, np.power(2, 0)) > 0] = 1
mask[np.bitwise_and(flags, np.power(2, 3)) > 0] = 2
self.maskVRT = VRT(array=mask)
metaDict.append({
'src': {
'SourceFilename': self.maskVRT.fileName,
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# create empty VRT dataset with geolocation only
simGdalDataset.SetProjection(
'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
)
VRT.__init__(self, simGdalDataset, **kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' GLOBCOLOR L3M VRT '''
print "=>%s<=" % gdalMetadata['NC_GLOBAL#title']
if 'GlobColour' not in gdalMetadata['NC_GLOBAL#title']:
raise AttributeError("GlobColour BAD MAPPER")
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
print 'idir:', iDir, iFile, iFileName[0:30], iFileExt[0:8]
simFilesMask = os.path.join(iDir, iFileName[0:30]+'*')
simFiles = glob.glob(simFilesMask)
print 'simFilesMask, simFiles', simFilesMask, simFiles
metaDict = []
for simFile in simFiles:
print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
simSubDatasets = simSupDataset.GetSubDatasets()
simWKV = None
for simSubDataset in simSubDatasets:
if '_mean' in simSubDataset[0]:
simValidSupDataset = simSupDataset
simGdalDataset = gdal.Open(simSubDataset[0])
simBandMetadata = simGdalDataset.GetRasterBand(1).GetMetadata()
simVarname = simBandMetadata['NETCDF_VARNAME']
# get WKV
print ' simVarname', simVarname
if simVarname in self.varname2wkv:
simWKV = self.varname2wkv[simVarname]
break
# skipp adding this similar file if it is not valid
if simWKV is None:
continue
metaEntry = {
'src': {'SourceFilename': simSubDataset[0],
'SourceBand': 1},
'dst': {'wkv': simWKV, 'original_name': simVarname}}
# add wavelength and name
longName = simBandMetadata['long_name']
if 'Fully normalised water leaving radiance' in longName:
simWavelength = simVarname.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add band with rrsw
metaEntry2 = None
if simWKV == 'surface_upwelling_spectral_radiance_in_air_emerging_from_sea_water':
solarIrradiance = simBandMetadata['solar_irradiance']
metaEntry2 = {'src': metaEntry['src']}
metaEntry2['dst'] ={
'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix': simWavelength,
'wavelength': simWavelength,
#'expression': 'self["nLw_%s"] / %s / (0.52 + 1.7 * self["nLw_%s"] / %s)' % (simWavelength, solarIrradiance, simWavelength, solarIrradiance),
'expression': 'self["nLw_%s"] / %s' % (simWavelength, solarIrradiance),
}
print ' metaEntry', metaEntry
metaDict.append(metaEntry)
if metaEntry2 is not None:
print ' metaEntry2', metaEntry2
metaDict.append(metaEntry2)
print 'simSubDatasets', simValidSupDataset.GetSubDatasets()
for simSubDataset in simValidSupDataset.GetSubDatasets():
print 'simSubDataset', simSubDataset
if '_flags ' in simSubDataset[1]:
print ' mask simSubDataset', simSubDataset[1]
flags = gdal.Open(simSubDataset[0]).ReadAsArray()
mask = np.ones(flags.shape) * 64
mask[np.bitwise_and(flags, np.power(2, 0)) > 0] = 1
mask[np.bitwise_and(flags, np.power(2, 3)) > 0] = 2
self.maskVRT = VRT(array=mask)
metaDict.append(
{'src': {'SourceFilename': self.maskVRT.fileName, 'SourceBand': 1},
'dst': {'name': 'mask'}})
# create empty VRT dataset with geolocation only
simGdalDataset.SetProjection('GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]')
VRT.__init__(self, simGdalDataset, **kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create NCEP VRT '''
if gdalDataset.GetGeoTransform() != (
-0.25, 0.5, 0.0, 90.25, 0.0,
-0.5) or gdalDataset.RasterCount != 9: # Not water proof
raise AttributeError("NCEP BAD MAPPER")
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 8
},
'dst': {
'wkv': 'eastward_wind',
'name': 'east_wind',
'height': '10 m'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 9
},
'dst': {
'wkv': 'northward_wind',
'name': 'north_wind',
'height': '10 m'
}
}, {
'src': [{
'SourceFilename': fileName,
'SourceBand': 8
}, {
'SourceFilename': fileName,
'SourceBand': 9
}],
'dst': {
'wkv': 'wind_speed',
'PixelFunctionType': 'UVToMagnitude',
'name': 'windspeed',
'height': '2 m'
}
}, {
'src': [{
'SourceFilename': fileName,
'SourceBand': 8
}, {
'SourceFilename': fileName,
'SourceBand': 9
}],
'dst': {
'wkv': 'wind_from_direction',
'PixelFunctionType': 'UVToDirectionFrom',
'name': 'winddirection',
'height': '2 m'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 6
},
'dst': {
'wkv': 'air_temperature',
'name': 'air_t',
'height': '2 m'
}
}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time from the GRIB file to dataset
validTime = gdalDataset.GetRasterBand(
8).GetMetadata()['GRIB_VALID_TIME']
self._set_time(
datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0])))
return
def __init__(self,
srs=None,
ext=None,
ds=None,
lon=None,
lat=None,
name='',
logLevel=None):
'''Create Domain from GDALDataset or string options or lat/lon grids
d = Domain(srs, ext)
Size, extent and spatial reference is given by strings
d = Domain(ds=GDALDataset):
Size, extent and spatial reference is copied from input
GDAL dataset
d = Domain(srs, ds=GDALDataset):
Spatial reference is given by srs, but size and extent is
determined
from input GDAL dataset
d = Domain(lon=lonGrid, lat=latGrid)
Size, extent and spatial reference is given by two grids
Parameters
----------
srs : PROJ4 or EPSG or WKT
Specifies spatial reference system (SRS)
PROJ4:
string with proj4 options [http://trac.osgeo.org/proj/] e.g.:
'+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs'
'+proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=75 +lon_0=10
+no_defs'
EPSG:
integer with EPSG number, [http://spatialreference.org/],
e.g. 4326
WKT:
string with Well Know Text of SRS. E.g.:
'GEOGCS["WGS 84",
DATUM["WGS_1984",
SPHEROID["WGS 84",6378137,298.257223563,
AUTHORITY["EPSG","7030"]],
TOWGS84[0,0,0,0,0,0,0],
AUTHORITY["EPSG","6326"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.0174532925199433,
AUTHORITY["EPSG","9108"]],
AUTHORITY["EPSG","4326"]]'
ext : string
some gdalwarp options + additional options
[http://www.gdal.org/gdalwarp.html]
Specifies extent, resolution / size
Available options: (('-te' or '-lle') and ('-tr' or '-ts'))
(e.g. '-lle -10 30 55 60 -ts 1000 1000' or
'-te 100 2000 300 10000 -tr 300 200')
-tr resolutionx resolutiony
-ts sizex sizey
-te xmin ymin xmax ymax
-lle lonmin latmin lonmax latmax
ds : GDAL dataset
lat : Numpy array
Grid with latitudes
lon : Numpy array
Grid with longitudes
name : string, optional
Name to be added to the Domain object
logLevel : int, optional, default=30
level of logging
Raises
-------
ProjectionError : occurs when Projection() is empty
despite it is required for creating extentDic.
OptionError : occures when the arguments are not proper.
Modifies
---------
self.vrt.datasetset : dataset in memory
dataset is created based on the input arguments
See Also
---------
Nansat.reproject()
[http://www.gdal.org/gdalwarp.html]
[http://trac.osgeo.org/proj/]
[http://spatialreference.org/]
[http://www.gdal.org/ogr/osr_tutorial.html]
'''
# set default attributes
self.logger = add_logger('Nansat', logLevel)
self.name = name
self.logger.debug('ds: %s' % str(ds))
self.logger.debug('srs: %s' % srs)
self.logger.debug('ext: %s' % ext)
# If too much information is given raise error
if ds is not None and srs is not None and ext is not None:
raise OptionError('Ambiguous specification of both '
'dataset, srs- and ext-strings.')
# if srs is given, convert it to WKT
if srs is not None:
# if XML-file and domain name is given - read that file
if isinstance(srs, str) and os.path.isfile(srs):
srs, ext, self.name = self._from_xml(srs, ext)
# import srs from srsString and get the projection
sr = osr.SpatialReference()
# try to use different import methods
# import from proj4 string
try:
status = sr.ImportFromProj4(srs)
except:
status = 1
# import from EPSG number
if status > 0:
try:
status = sr.ImportFromEPSG(srs)
except:
status = 1
# import from WKT text
if status > 0:
try:
status = sr.ImportFromWkt(srs)
except:
status = 1
# create WKT
dstWKT = sr.ExportToWkt()
# test success of WKT
if status > 0 or dstWKT == '':
raise ProjectionError('srs (%s) is wrong' % (srs))
# choose between input opitons:
# ds
# ds and srs
# srs and ext
# lon and lat
# if only a dataset is given:
# copy geo-reference from the dataset
if ds is not None and srs is None:
self.vrt = VRT(gdalDataset=ds)
# If dataset and srs are given (but not ext):
# use AutoCreateWarpedVRT to determine bounds and resolution
elif ds is not None and srs is not None:
tmpVRT = gdal.AutoCreateWarpedVRT(ds, None, dstWKT)
if tmpVRT is None:
raise ProjectionError('Could not warp the given dataset'
'to the given SRS.')
else:
self.vrt = VRT(gdalDataset=tmpVRT)
# If proj4 and extent string are given (but not dataset)
elif srs is not None and ext is not None:
# create full dictionary of parameters
extentDic = self._create_extentDic(ext)
# convert -lle to -te
if 'lle' in extentDic.keys():
extentDic = self._convert_extentDic(dstWKT, extentDic)
# get size/extent from the created extet dictionary
[geoTransform, rasterXSize,
rasterYSize] = self._get_geotransform(extentDic)
# create VRT object with given geo-reference parameters
self.vrt = VRT(srcGeoTransform=geoTransform,
srcProjection=dstWKT,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
self.extentDic = extentDic
elif lat is not None and lon is not None:
# create self.vrt from given lat/lon
self.vrt = VRT(lat=lat, lon=lon)
else:
raise OptionError('"dataset" or "srsString and extentString" '
'or "dataset and srsString" are required')
self.logger.debug('vrt.dataset: %s' % str(self.vrt.dataset))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
product = gdalMetadata["MPH_PRODUCT"]
if product[0:9] != "MER_FRS_2" and product[0:9] != "MER_RR__2":
raise AttributeError("MERIS_L2 BAD MAPPER")
kwDict = {'geolocation' : True}
# choose kwargs for envisat and asar and change keyname
for key in kwargs:
if key.startswith('envisat') or key.startswith('meris'):
keyName = key.replace('envisat_', '').replace('meris_', '')
kwDict[keyName] = kwargs[key]
else:
kwDict[key] = kwargs[key]
Envisat.__init__(self, fileName, product[0:4], **kwDict)
metaDict = [
{'src': {'SourceFilename': fileName, 'SourceBand': 1}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '412'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 2}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '443'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 3}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '490'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 4}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '510'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 5}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '560'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 6}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '620'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 7}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '665'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 8}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '680'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 9}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '708'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 10}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '753'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 11}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '761'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 12}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '778'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 13}, 'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '864'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 15}, 'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water', 'suffix': '1_log', 'case': 'I'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 16}, 'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter', 'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 17}, 'dst': {'wkv': 'mass_concentration_of_suspended_matter_in_sea_water', 'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 18}, 'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water', 'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 22}, 'dst': {'wkv': 'quality_flags', 'suffix': 'l2'}},
]
# add 'name' to 'parameters'
for bandDict in metaDict:
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
#get GADS from header
scales = self.read_scaling_gads(range(7, 20) + [20, 21, 22, 20])
offsets = self.read_scaling_gads(range(33, 46) + [46, 47, 48, 46])
# set scale/offset to the band metadata (only reflectance)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
bandDict['src']['ScaleOffset'] = str(offsets[i])
# add log10-scaled variables
metaDict += [
{'src': {'SourceFilename': fileName, 'SourceBand': 1}, 'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water', 'suffix': '1', 'case': 'I', 'expression': 'np.power(10., self["chlor_a_1_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1}, 'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water', 'suffix': '2', 'case': 'II', 'expression': 'np.power(10., self["chlor_a_2_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1}, 'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter', 'suffix': '2', 'case': 'II', 'expression': 'np.power(10., self["cdom_a_2_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1}, 'dst': {'wkv': 'mass_concentration_of_suspended_matter_in_sea_water', 'suffix': '2', 'case': 'II', 'expression': 'np.power(10., self["tsm_2_log"])'}},
]
# get list with resized VRTs from ADS
self.adsVRTs = []
self.adsVRTs = self.get_ads_vrts(gdalDataset, ['sun zenith angles', "sun azimuth angles", "zonal winds", "meridional winds"])
# add bands from the ADS VRTs
for adsVRT in self.adsVRTs:
metaDict.append({'src': {'SourceFilename': adsVRT.fileName,
'SourceBand': 1},
'dst': {'name': adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name'),
'units': adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units')}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if self.d['geolocation']:
self.add_geolocation_from_ads(gdalDataset)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VIIRS VRT '''
assert 'GMTCO_npp_' in fileName, 'viirs_l1 BAD MAPPER'
ifiledir = os.path.split(fileName)[0]
ifiles = glob.glob(ifiledir + 'SVM??_npp_d*_obpg_ops.h5')
ifiles.sort()
viirsWavelengths = [None, 412, 445, 488, 555, 672, 746, 865, 1240, 1378, 1610, 2250, 3700, 4050, 8550, 10736, 12013]
# create dictionary of viirsL1 parameters
kwDict = { 'GCP_COUNT0' : 5,
'GCP_COUNT1' : 20,
'pixelStep' : 1,
'lineStep' : 1}
# set kwargs
viirsL1Kwargs = {}
for key in kwargs:
if key.startswith('viirs_l1'):
keyName = key.replace('viirs_l1_', '')
viirsL1Kwargs[keyName] = kwargs[key]
# modify the default values using input values
kwDict = set_defaults(kwDict, viirsL1Kwargs)
# create empty VRT dataset with geolocation only
xDatasetSource = 'HDF5:"%s"://All_Data/VIIRS-MOD-GEO-TC_All/Longitude' % fileName
xDatasetBand = 1
xDataset = gdal.Open(xDatasetSource)
VRT.__init__(self, xDataset, **kwargs)
metaDict = []
for ifile in ifiles:
ifilename = os.path.split(ifile)[1]
print ifilename
bNumber = int(ifilename[3:5])
print bNumber
bWavelength = viirsWavelengths[bNumber]
print bWavelength
SourceFilename = 'HDF5:"%s"://All_Data/VIIRS-M%d-SDR_All/Radiance' % (ifile, bNumber)
print SourceFilename
metaEntry = {
'src': {'SourceFilename': SourceFilename, 'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance', 'wavelength': str(bWavelength),
'suffix': str(bWavelength)},
}
metaDict.append(metaEntry)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
xVRTArray = xDataset.ReadAsArray()
xVRTArray = gaussian_filter(xVRTArray, 5).astype('float32')
xVRT = VRT(array=xVRTArray)
yDatasetSource = 'HDF5:"%s"://All_Data/VIIRS-MOD-GEO-TC_All/Latitude' % fileName
yDatasetBand = 1
yDataset = gdal.Open(yDatasetSource)
yVRTArray = yDataset.ReadAsArray()
yVRTArray = gaussian_filter(yVRTArray, 5).astype('float32')
yVRT = VRT(array=yVRTArray)
#self.add_geolocationArray(GeolocationArray(xDatasetSource, yDatasetSource))
#"""
# estimate pixel/line step
self.logger.debug('pixel/lineStep %f %f' % (kwDict['pixelStep'], kwDict['lineStep']))
# ==== ADD GCPs and Pojection ====
# get lat/lon matrices
longitude = xVRT.dataset.GetRasterBand(1).ReadAsArray()
latitude = yVRT.dataset.GetRasterBand(1).ReadAsArray()
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / kwDict['GCP_COUNT0']))
step1 = max(1, int(float(latitude.shape[1]) / kwDict['GCP_COUNT1']))
self.logger.debug('gcpCount: %d %d %d %d, %d %d',
latitude.shape[0], latitude.shape[1],
kwDict['GCP_COUNT0'], kwDict['GCP_COUNT1'],
step0, step1)
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0,
i1 * kwDict['pixelStep'],
i0 * kwDict['lineStep'])
self.logger.debug('%d %d %d %f %f',
k, gcp.GCPPixel, gcp.GCPLine,
gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, latlongSRS.ExportToWkt())
# remove geolocation array
self.remove_geolocationArray()
def __init__(self, fileName, gdalDataset, gdalMetadata , **kwargs):
''' Create CSKS VRT '''
if fileName[0:4] != "CSKS":
raise AttributeError("COSMO-SKYMED BAD MAPPER")
# Get coordinates
bottom_left_lon = float(gdalMetadata['Estimated_Bottom_Left_Geodetic_Coordinates'].split(' ')[1])
bottom_left_lat = float(gdalMetadata['Estimated_Bottom_Left_Geodetic_Coordinates'].split(' ')[0])
bottom_right_lon = float(gdalMetadata['Estimated_Bottom_Right_Geodetic_Coordinates'].split(' ')[1])
bottom_right_lat = float(gdalMetadata['Estimated_Bottom_Right_Geodetic_Coordinates'].split(' ')[0])
top_left_lon = float(gdalMetadata['Estimated_Top_Left_Geodetic_Coordinates'].split(' ')[1])
top_left_lat = float(gdalMetadata['Estimated_Top_Left_Geodetic_Coordinates'].split(' ')[0])
top_right_lon = float(gdalMetadata['Estimated_Top_Right_Geodetic_Coordinates'].split(' ')[1])
top_right_lat = float(gdalMetadata['Estimated_Top_Right_Geodetic_Coordinates'].split(' ')[0])
center_lon = float(gdalMetadata['Scene_Centre_Geodetic_Coordinates'].split(' ')[1])
center_lat = float(gdalMetadata['Scene_Centre_Geodetic_Coordinates'].split(' ')[0])
# Get sub-datasets
subDatasets = gdalDataset.GetSubDatasets()
# Get file names from dataset or subdataset
if subDatasets.__len__()==1:
fileNames = [fileName]
else:
fileNames = [f[0] for f in subDatasets]
for i,elem in enumerate(fileNames):
if fileNames[i][-3:]=='QLK':
fileNames.pop(i)
#print fileNames
subDataset = gdal.Open(fileNames[0])
# generate list of GCPs
gcps = []
# create GCP with X,Y,Z(?),pixel,line from lat/lon matrices
gcp = gdal.GCP(float(bottom_left_lon), float(bottom_left_lat), 0, 0, 0)
gcps.append( gcp )
#self.logger.debug('%d %d %d %f %f', 0, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(bottom_right_lon), float(bottom_right_lat), 0, subDataset.RasterXSize, 0)
gcps.append( gcp )
#self.logger.debug('%d %d %d %f %f', 1, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_left_lon), float(top_left_lat), 0, 0, subDataset.RasterYSize)
gcps.append( gcp )
#self.logger.debug('%d %d %d %f %f', 2, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_right_lon), float(top_right_lat),
0, subDataset.RasterXSize, subDataset.RasterYSize)
gcps.append( gcp )
#self.logger.debug('%d %d %d %f %f', 3, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(center_lon), float(center_lat),
0, int(np.round(subDataset.RasterXSize/2.)),
int(round(subDataset.RasterYSize/2.)))
gcps.append( gcp )
#self.logger.debug('%d %d %d %f %f', 4, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY)
# append GCPs and lat/lon projection to the vsiDataset
latlongSRS = osr.SpatialReference()
latlongSRS.ImportFromProj4("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
latlongSRSWKT = latlongSRS.ExportToWkt()
# create empty VRT dataset with geolocation only
VRT.__init__(self, srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
srcGCPs=gcps,
srcGCPProjection=latlongSRSWKT)
#print self.fileName
# Read all bands later
#band='S01'
#res='SBI'
# Use only full size "original" datasets
for i,elem in enumerate(fileNames):
band_number = i
if fileNames[i][-3:]=='SBI':
# Add real and imaginary raw counts as bands
src = {'SourceFilename': fileNames[i], 'SourceBand': 1, 'DataType': gdal.GDT_Int16}
dst = {'dataType': gdal.GDT_Float32, 'name': 'RawCounts_%s_real' %
gdalMetadata[fileNames[i][-7:-4]+'_Polarisation']}
self._create_band(src,dst)
src = {'SourceFilename': fileNames[i], 'SourceBand': 2, 'DataType': gdal.GDT_Int16}
dst = {'dataType': gdal.GDT_Float32, 'name': 'RawCounts_%s_imaginary' %
gdalMetadata[fileNames[i][-7:-4]+'_Polarisation'] }
self._create_band(src,dst)
self.dataset.FlushCache()
for i,elem in enumerate(fileNames):
band_number = i
if fileNames[i][-3:]=='SBI':
# Calculate sigma0 scaling factor
Rref = float(gdalMetadata['Reference_Slant_Range'])
Rexp = float(gdalMetadata['Reference_Slant_Range_Exponent'])
alphaRef = float(gdalMetadata['Reference_Incidence_Angle'])
F=float(gdalMetadata['Rescaling_Factor'])
K=float(gdalMetadata[fileNames[i][-7:-4]+'_Calibration_Constant'])
Ftot = Rref**(2.*Rexp)
Ftot *=np.sin(alphaRef*np.pi/180.0)
Ftot /=F**2.
Ftot /=K
#print Ftot
src = [{'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2*i+1, 'ScaleRatio': np.sqrt(Ftot)},
{ 'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2*i+2, 'ScaleRatio': np.sqrt(Ftot)}]
dst = {'wkv':
'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType': 'RawcountsToSigma0_CosmoSkymed_SBI',
'polarisation': gdalMetadata[fileNames[i][-7:-4]+'_Polarisation'],
'name': 'sigma0_%s' % gdalMetadata[fileNames[i][-7:-4]+'_Polarisation'],
'SatelliteID': gdalMetadata['Satellite_ID'],
'dataType': gdal.GDT_Float32}
#'pass': gdalMetadata[''] - I can't find this in the metadata...
self._create_band(src,dst)
self.dataset.FlushCache()
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create MODIS_L1 VRT '''
#get 1st subdataset and parse to VRT.__init__() for retrieving geo-metadata
gdalSubDataset = gdal.Open(gdalDataset.GetSubDatasets()[0][0])
#list of available modis names:resolutions
modisResolutions = {'MYD02QKM': 250, 'MOD02QKM': 250,
'MYD02HKM': 500, 'MOD02HKM': 500,
'MYD021KM': 1000, 'MOD021KM': 1000}
#should raise error in case of not MODIS_L1
mResolution = modisResolutions[gdalMetadata["SHORTNAME"]]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalSubDataset)
subDsString = 'HDF4_EOS:EOS_SWATH:"%s":MODIS_SWATH_Type_L1B:%s'
#provide all mappings
metaDict250SF = ['EV_250_RefSB']
metaDict250 = [{'src': {'SourceFilename': subDsString %
(fileName, 'EV_250_RefSB'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '645'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_250_RefSB'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '858'}}]
metaDict500SF = ['EV_250_Aggr500_RefSB', 'EV_500_RefSB']
metaDict500 = [{'src': {'SourceFilename': subDsString %
(fileName, 'EV_250_Aggr500_RefSB'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '645'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_250_Aggr500_RefSB'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '858'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_RefSB'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '469'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_RefSB'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '555'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_RefSB'),
'SourceBand': 3},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1240'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_RefSB'),
'SourceBand': 4},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1640'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_RefSB'),
'SourceBand': 5},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '2130'}}]
metaDict1000SF = ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB', 'EV_1KM_Emissive']
metaDict1000 = [{'src': {'SourceFilename': subDsString %
(fileName, 'EV_250_Aggr1km_RefSB'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '645'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_250_Aggr1km_RefSB'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '858'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '469'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '555'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 3},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1240'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 4},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1640'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_500_Aggr1km_RefSB'),
'SourceBand': 5},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '2130'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '412'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '443'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 3},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '488'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 4},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '531'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 5},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '551'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 6},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '667'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 7},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '667'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 8},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '678'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 9},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '678'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 10},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '748'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 11},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '869'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 12},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '905'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 13},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '936'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 14},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '940'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_RefSB'),
'SourceBand': 15},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '1375'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '3750'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '3959'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 3},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '3959'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 4},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '4050'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 5},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '4465'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 6},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '4515'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 7},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '6715'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 8},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '7325'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 9},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '8550'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 10},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '9730'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 11},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '11030'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 12},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '12020'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 13},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '13335'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 14},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '13635'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 15},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '13935'}},
{'src': {'SourceFilename': subDsString %
(fileName, 'EV_1KM_Emissive'),
'SourceBand': 16},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '14235'}}]
# get proper mapping depending on resolution
metaDict = {250: metaDict250,
500: metaDict500,
1000: metaDict1000,
}[mResolution]
# get proper mapping depending on resolution
metaDictSF = {250: metaDict250SF,
500: metaDict500SF,
1000: metaDict1000SF,
}[mResolution]
# read all scales/offsets
rScales = {}
rOffsets = {}
for sf in metaDictSF:
dsName = subDsString % (fileName, sf)
ds = gdal.Open(dsName)
rScales[dsName] = map(float,
ds.GetMetadataItem('radiance_scales').
split(','))
rOffsets[dsName] = map(float,
ds.GetMetadataItem('radiance_offsets').
split(','))
self.logger.debug('radiance_scales: %s' % str(rScales))
# add 'band_name' to 'parameters'
for bandDict in metaDict:
SourceFilename = bandDict['src']['SourceFilename']
SourceBand = bandDict['src']['SourceBand']
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
scale = rScales[SourceFilename][SourceBand-1]
offset = rOffsets[SourceFilename][SourceBand-1]
self.logger.debug('band, scale, offset: %s_%d %s %s' %
(SourceFilename, SourceBand, scale, offset))
bandDict['src']['ScaleRatio'] = scale
bandDict['src']['ScaleOffset'] = offset
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
productDate = gdalMetadata["RANGEBEGINNINGDATE"]
productTime = gdalMetadata["RANGEENDINGTIME"]
self._set_time(parse(productDate+' '+productTime))
self.remove_geolocationArray()
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
'''
Parameters (**kwargs)
---------------------
ASA_full_incAng : bool (default False)
if True, use full-size incidence angle band.
if False, use one-line incidence angle band.
'''
product = gdalMetadata.get("MPH_PRODUCT")
if product[0:4] != "ASA_":
raise AttributeError("ASAR_L1 BAD MAPPER")
kwDict = {'geolocation': False}
# choose kwargs for envisat and asar and change keyname
for key in kwargs:
if key.startswith('envisat') or key.startswith('asar'):
keyName = key.replace('envisat_', '').replace('asar_', '')
kwDict[keyName] = kwargs[key]
else:
kwDict[key] = kwargs[key]
Envisat.__init__(self, fileName, product[0:4], **kwDict)
# get polarization string (remove '/', since NetCDF doesnt support that in metadata)
polarization = gdalMetadata['SPH_MDS1_TX_RX_POLAR'].replace("/", "")
# Create VRTdataset with small VRTRawRasterbands
self.adsVRTs = self.get_ads_vrts(gdalDataset,
["first_line_incidenceAngle"])
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwDict)
# get calibration constant
gotCalibration = True
try:
calibrationConst = float(
gdalDataset.GetMetadataItem(
"MAIN_PROCESSING_PARAMS_ADS_1_CALIBRATION_FACTORS.1.EXT_CAL_FACT",
"records"))
except:
self.logger.warning('Cannot get calibrationConst')
gotCalibration = False
# add dictionary for raw counts
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'name': 'RawCounts_%s' % polarization
}
}]
if gotCalibration:
# add dicrtionary for sigma0, ice and water
names = [
'sigma0', 'sigma0_normalized_ice', 'sigma0_normalized_water'
]
wkt = [
'surface_backwards_scattering_coefficient_of_radar_wave',
'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_ice',
'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_water'
]
sphPass = [gdalMetadata['SPH_PASS'], '', '']
sourceFileNames = [fileName, self.adsVRTs[0].fileName]
pixelFunctionTypes = [
'RawcountsIncidenceToSigma0', 'Sigma0NormalizedIce'
]
if polarization == 'HH':
pixelFunctionTypes.append('Sigma0HHNormalizedWater')
elif polarization == 'VV':
pixelFunctionTypes.append('Sigma0VVNormalizedWater')
# add pixelfunction bands to metaDict
for iPixFunc in range(len(pixelFunctionTypes)):
srcFiles = []
for iFileName in sourceFileNames:
sourceFile = {'SourceFilename': iFileName, 'SourceBand': 1}
# if ASA_full_incAng, set 'ScaleRatio' into source file dict
if iFileName == fileName:
sourceFile['ScaleRatio'] = np.sqrt(1.0 /
calibrationConst)
srcFiles.append(sourceFile)
metaDict.append({
'src': srcFiles,
'dst': {
'name': names[iPixFunc],
'wkv': wkt[iPixFunc],
'PixelFunctionType': pixelFunctionTypes[iPixFunc],
'polarization': polarization,
'suffix': polarization,
'pass': sphPass[iPixFunc],
'dataType': 6
}
})
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if self.d['geolocation']:
self.add_geolocation_from_ads(gdalDataset)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
product = gdalMetadata["MPH_PRODUCT"]
if product[0:9] != "MER_FRS_2" and product[0:9] != "MER_RR__2":
raise AttributeError("MERIS_L2 BAD MAPPER")
kwDict = {'geolocation': True}
# choose kwargs for envisat and asar and change keyname
for key in kwargs:
if key.startswith('envisat') or key.startswith('meris'):
keyName = key.replace('envisat_', '').replace('meris_', '')
kwDict[keyName] = kwargs[key]
else:
kwDict[key] = kwargs[key]
Envisat.__init__(self, fileName, product[0:4], **kwDict)
metaDict = [
{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '412'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 2
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '443'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 3
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '490'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 4
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '510'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 5
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '560'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 6
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '620'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 7
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '665'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 8
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '680'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 9
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '708'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 10
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '753'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 11
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '761'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 12
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '778'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 13
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '864'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 15
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1_log',
'case': 'I'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 16
},
'dst': {
'wkv':
'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2_log',
'case': 'II'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 17
},
'dst': {
'wkv':
'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2_log',
'case': 'II'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 18
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2_log',
'case': 'II'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 22
},
'dst': {
'wkv': 'quality_flags',
'suffix': 'l2'
}
},
]
# add 'name' to 'parameters'
for bandDict in metaDict:
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
#get GADS from header
scales = self.read_scaling_gads(range(7, 20) + [20, 21, 22, 20])
offsets = self.read_scaling_gads(range(33, 46) + [46, 47, 48, 46])
# set scale/offset to the band metadata (only reflectance)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
bandDict['src']['ScaleOffset'] = str(offsets[i])
# add log10-scaled variables
metaDict += [
{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1',
'case': 'I',
'expression': 'np.power(10., self["chlor_a_1_log"])'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2',
'case': 'II',
'expression': 'np.power(10., self["chlor_a_2_log"])'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv':
'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2',
'case': 'II',
'expression': 'np.power(10., self["cdom_a_2_log"])'
}
},
{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv':
'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2',
'case': 'II',
'expression': 'np.power(10., self["tsm_2_log"])'
}
},
]
# get list with resized VRTs from ADS
self.adsVRTs = []
self.adsVRTs = self.get_ads_vrts(gdalDataset, [
'sun zenith angles', "sun azimuth angles", "zonal winds",
"meridional winds"
])
# add bands from the ADS VRTs
for adsVRT in self.adsVRTs:
metaDict.append({
'src': {
'SourceFilename': adsVRT.fileName,
'SourceBand': 1
},
'dst': {
'name':
adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name'),
'units':
adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units')
}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset, **kwDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if self.d['geolocation']:
self.add_geolocation_from_ads(gdalDataset)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Ocean Productivity website VRT '''
if ('IDL' not in gdalMetadata['Projection Category'] and 'Source' not in gdalMetadata and '-9999' not in gdalMetadata['Hole Value']):
raise AttributeError("BAD MAPPER")
print 'Ocean Productivity website data'
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
simFilesMask = os.path.join(iDir, '*' + iFileName[4:11] + iFileExt)
#print 'simFilesMask', simFilesMask
simFiles = glob.glob(simFilesMask)
#print 'simFiles', simFiles
metaDict = []
for simFile in simFiles:
#print 'simFile',simFile
# open subdataset with GDAL
tmpSourceFilename = simFile
tmpGdalDataset = gdal.Open(tmpSourceFilename)
# get metadata, get 'Parameter'
tmpGdalMetadata = tmpGdalDataset.GetMetadata()
iDir, ifileName = os.path.split(tmpSourceFilename)
#print 'ifileName',ifileName
simParameter = ifileName[0:3]
# set params of the similar file
simSourceFilename = tmpSourceFilename
simGdalDataset = tmpGdalDataset
simGdalMetadata = tmpGdalMetadata
# get WKV from the similar file
for param in self.param2wkv:
#print 'param', param
if param in simParameter:
simWKV = self.param2wkv[param]
break
#print 'simWKV', simWKV
# generate entry to metaDict
metaEntry = {
'src': {'SourceFilename': simSourceFilename,
'SourceBand': 1,
'ScaleRatio': float(simGdalMetadata['Slope']),
'ScaleOffset': float(simGdalMetadata['Intercept'])},
'dst': {'wkv': simWKV,
'name': self.bandNames[simWKV],
'Parameter': simParameter}}
#print 'metaEntry', metaEntry
# append entry to metaDict
metaDict.append(metaEntry)
#get array with data and make 'mask'
a = simGdalDataset.ReadAsArray()
mask = np.zeros(a.shape, 'uint8') + 128
mask[a < -9990] = 1
self.maskVRT = VRT(array=mask)
metaDict.append(
{'src': {'SourceFilename': self.maskVRT.fileName, 'SourceBand': 1},
'dst': {'name': 'mask'}})
# create empty VRT dataset with geolocation only
# print 'simGdalMetadata', simGdalMetadata
latitudeStep = 0.08333334
longitudeStep = 0.08333334
numberOfColumns = 4320
numberOfLines = 2160
#longitudeStep = float(simGdalMetadata['Longitude Step'])
VRT.__init__(self, srcGeoTransform=(-180.0, longitudeStep, 0.0, 90.0, 0.0, -longitudeStep),
srcProjection='GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
srcRasterXSize=numberOfColumns,
srcRasterYSize=numberOfLines,
**kwargs
)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add valid time
startYear = int(iFile[4:8])
startDay = int(iFile[8:11])
self._set_time(datetime.datetime(startYear, 1, 1) + datetime.timedelta(startDay))
def __init__(self, fileName, gdalDataset, gdalMetadata,
full_incAng=True, geolocation=False, zoomSize=500,
step=1, **kwargs):
'''
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
full_incAng : bool (default is True)
if True, add full size incedence angle
geolocation : bool (default is False)
if True, add gdal geolocation
zoomSize: int (used in envisat.py)
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int (used in envisat.py)
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
'''
product = gdalMetadata.get("MPH_PRODUCT")
if product[0:4] != "ASA_":
raise AttributeError("ASAR_L1 BAD MAPPER")
Envisat.__init__(self, fileName, product[0:4])
# get polarization string (remove '/', since NetCDF doesnt support that in metadata)
polarization = gdalMetadata['SPH_MDS1_TX_RX_POLAR'].replace("/", "")
# Create VRTdataset with small VRTRawRasterbands
self.adsVRTs = self.get_ads_vrts(gdalDataset,
["first_line_incidence_angle"],
zoomSize=zoomSize, step=step, **kwargs)
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# get calibration constant
gotCalibration = True
try:
calibrationConst = float(gdalDataset.GetMetadataItem(
"MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS.1.EXT_CAL_FACT", "records"))
except:
try:
# Apparently some ASAR files have calibration constant stored in another place
calibrationConst = float(gdalDataset.GetMetadataItem(
"MAIN_PROCESSING_PARAMS_ADS_1_CALIBRATION_FACTORS.1.EXT_CAL_FACT", "records"))
except:
self.logger.warning('Cannot get calibrationConst')
gotCalibration = False
# add dictionary for raw counts
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'short_name': 'RawCounts'}}]
if full_incAng:
for adsVRT in self.adsVRTs:
metaDict.append({'src': {'SourceFilename': adsVRT.fileName,
'SourceBand': 1},
'dst': {'name': adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name').replace('last_line_', ''),
'units': adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units')}})
if gotCalibration:
# add dicrtionary for sigma0, ice and water
short_names = ['sigma0', 'sigma0_normalized_ice',
'sigma0_normalized_water']
wkt = ['surface_backwards_scattering_coefficient_of_radar_wave',
'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_ice',
'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_water']
sphPass = [gdalMetadata['SPH_PASS'], '', '']
sourceFileNames = [fileName,
self.adsVRTs[0].fileName]
pixelFunctionTypes = ['RawcountsIncidenceToSigma0',
'Sigma0NormalizedIce']
if polarization == 'HH':
pixelFunctionTypes.append('Sigma0HHNormalizedWater')
elif polarization == 'VV':
pixelFunctionTypes.append('Sigma0VVNormalizedWater')
# add pixelfunction bands to metaDict
for iPixFunc in range(len(pixelFunctionTypes)):
srcFiles = []
for iFileName in sourceFileNames:
sourceFile = {'SourceFilename': iFileName,
'SourceBand': 1}
# if ASA_full_incAng, set 'ScaleRatio' into source file dict
if iFileName == fileName:
sourceFile['ScaleRatio'] = np.sqrt(1.0/calibrationConst)
srcFiles.append(sourceFile)
metaDict.append({'src': srcFiles,
'dst': {'short_name': short_names[iPixFunc],
'wkv': wkt[iPixFunc],
'PixelFunctionType': pixelFunctionTypes[iPixFunc],
'polarization': polarization,
'suffix': polarization,
'pass': sphPass[iPixFunc],
'dataType': 6}})
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if geolocation:
self.add_geolocation_from_ads(gdalDataset,
zoomSize=zoomSize, step=step)
# Add SAR look direction to metadata domain
# Note that this is the look direction in the center of the domain. For
# longer domains, especially at high latitudes, the azimuth direction
# may vary a lot over the domain, and using the center angle will be a
# coarse approximation.
self.dataset.SetMetadataItem('SAR_center_look_direction',
str(np.mod(Domain(ds=gdalDataset).
upwards_azimuth_direction() + 90,
360)))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' OBPG L3 VRT '''
if 'Level-3 Standard Mapped Image' not in gdalMetadata['Title']:
raise AttributeError("OBPG L3 Standard Mapped Image BAD MAPPER")
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
simFilesMask = os.path.join(iDir, iFileName)
simFiles = glob.glob(simFilesMask + iFileExt[0:6] + '*')
#print 'simFilesMask, simFiles', simFilesMask, simFiles
metaDict = []
for simFile in simFiles:
#print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
if simSupDataset is None:
# skip this similar file
#print 'No dataset: %s not a supported SMI file' % simFile
continue
# get subdatasets from the similar file
simSubDatasets = simSupDataset.GetSubDatasets()
if len(simSubDatasets) > 0:
for simSubDataset in simSubDatasets:
#print 'simSubDataset', simSubDataset
if 'l3m_data' in simSubDataset[1]:
# get SourceFilename from subdataset
tmpSourceFilename = simSubDataset[0]
break
else:
# get SourceFilename from dataset
tmpSourceFilename = simFile
# open subdataset with GDAL
#print 'tmpSourceFilename', tmpSourceFilename
tmpGdalDataset = gdal.Open(tmpSourceFilename)
try:
# get metadata, get 'Parameter'
tmpGdalMetadata = tmpGdalDataset.GetMetadata()
simParameter = tmpGdalMetadata['Parameter']
except:
print 'No parameter: %s not a supported SMI file' % simFile
continue
else:
# set params of the similar file
simSourceFilename = tmpSourceFilename
simGdalDataset = tmpGdalDataset
simGdalMetadata = tmpGdalMetadata
# get WKV from the similar file
#print 'simParameter', simParameter
for param in self.param2wkv:
#print 'param', param
if param in simParameter:
simWKV = self.param2wkv[param]
break
# generate entry to metaDict
metaEntry = {
'src': {'SourceFilename': simSourceFilename,
'SourceBand': 1,
'ScaleRatio': float(simGdalMetadata['Slope']),
'ScaleOffset': float(simGdalMetadata['Intercept'])},
'dst': {'wkv': simWKV}}
# add wavelength and BandName
if ' at ' in simParameter and ' nm' in simParameter:
simWavelength = simParameter.split(' at ')[1].split(' nm')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add band with Rrsw
metaEntry2 = None
if simWKV == 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air':
metaEntry2 = {'src': [metaEntry['src']]}
metaEntry2['dst'] ={
'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix': simWavelength,
'wavelength': simWavelength,
'PixelFunctionType': 'NormReflectanceToRemSensReflectance',
}
# append entry to metaDict
metaDict.append(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
#get array with data and make 'mask'
a = simGdalDataset.ReadAsArray()
mask = np.zeros(a.shape, 'uint8') + 64
mask[a < -32000] = 1
self.maskVRT = VRT(array=mask)
metaDict.append(
{'src': {'SourceFilename': self.maskVRT.fileName, 'SourceBand': 1},
'dst': {'name': 'mask'}})
# create empty VRT dataset with geolocation only
# print 'simGdalMetadata', simGdalMetadata
latitudeStep = float(simGdalMetadata.get('Latitude Step', simGdalMetadata.get('Latitude_Step', 1)))
longitudeStep = float(simGdalMetadata.get('Longitude Step', simGdalMetadata.get('Longitude_Step', 1)))
numberOfColumns = int(simGdalMetadata.get('Number of Columns', simGdalMetadata.get('Number_of_Columns', 1)))
numberOfLines = int(simGdalMetadata.get('Number of Lines', simGdalMetadata.get('Number_of_Lines', 1)))
#longitudeStep = float(simGdalMetadata['Longitude Step'])
VRT.__init__(self, srcGeoTransform=(-180.0, longitudeStep, 0.0, 90.0, 0.0, -longitudeStep),
srcProjection=gdalDataset.GetProjection(),
srcRasterXSize=numberOfColumns,
srcRasterYSize=numberOfLines,
**kwargs
)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add valid time
startYear = int(simGdalMetadata.get('Start Year', simGdalMetadata.get('Start_Year', 1)))
startDay = int(simGdalMetadata.get('Start Day', simGdalMetadata.get('Start)Day', 1)))
self._set_time(datetime.datetime(startYear, 1, 1) + datetime.timedelta(startDay))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT '''
# check if it is ASTER L1A
assert 'AST_L1A_' in fileName
shortName = gdalMetadata['INSTRUMENTSHORTNAME']
assert shortName == 'ASTER'
subDatasets = gdalDataset.GetSubDatasets()
kwDict = {'GCP_COUNT' : 10, # number of GCPs along each dimention
'bandNames' : ['VNIR_Band1', 'VNIR_Band2', 'VNIR_Band3N'],
'bandWaves' : [560, 660, 820]}
'''
'VNIR_Band3B' : 820, 'SWIR_Band4' : 1650, 'SWIR_Band5' : 2165,
'SWIR_Band6' : 2205, 'SWIR_Band7' : 2260, 'SWIR_Band8' : 2330,
'SWIR_Band9' : 2395, 'TIR_Band10' : 8300, 'TIR_Band11' : 8650,
'TIR_Band12' : 9100, 'TIR_Band13' : 10600, 'TIR_Band14' : 11300
'''
# set kwargs
asterL1aKwargs = {}
for key in kwargs:
if key.startswith('aster_l1a'):
keyName = key.replace('aster_l1a_', '')
asterL1aKwargs[keyName] = kwargs[key]
# modify the default values using input values
kwDict = set_defaults(kwDict, asterL1aKwargs)
# find datasets for each band and generate metaDict
metaDict = []
bandDatasetMask = 'HDF4_EOS:EOS_SWATH:"%s":%s:ImageData'
for bandName, bandWave in zip(kwDict['bandNames'],
kwDict['bandWaves']):
metaEntry = {
'src': {
'SourceFilename': bandDatasetMask % (fileName, bandName),
'SourceBand': 1,
'DataType': 6,
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': str(bandWave),
'suffix': str(bandWave),
},
}
metaDict.append(metaEntry)
# create empty VRT dataset with geolocation only
gdalSubDataset = gdal.Open(metaDict[0]['src']['SourceFilename'])
VRT.__init__(self, gdalSubDataset, **kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# find largest lon/lat subdatasets
latShape0 = 0
for subDataset in subDatasets:
if 'Latitude' in subDataset[1]:
ls = int(subDataset[1].strip().split('[')[1].split('x')[0])
if ls >= latShape0:
latShape0 = ls
latSubDS = subDataset[0]
if 'Longitude' in subDataset[1]:
ls = int(subDataset[1].strip().split('[')[1].split('x')[0])
if ls >= latShape0:
latShape0 = ls
lonSubDS = subDataset[0]
self.logger.debug(latSubDS)
self.logger.debug(lonSubDS)
# get lat/lon matrices
xDataset = gdal.Open(lonSubDS)
yDataset = gdal.Open(latSubDS)
longitude = xDataset.ReadAsArray()
latitude = yDataset.ReadAsArray()
step0 = longitude.shape[0] / kwDict['GCP_COUNT']
step1 = longitude.shape[1] / kwDict['GCP_COUNT']
# estimate pixel/line step
pixelStep = int(ceil(float(gdalSubDataset.RasterXSize) / float(xDataset.RasterXSize)))
lineStep = int(ceil(float(gdalSubDataset.RasterYSize) / float(xDataset.RasterYSize)))
self.logger.debug('steps: %d %d %d %d' % (step0, step1, pixelStep, lineStep))
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep, i0 * lineStep)
self.logger.debug('%d %d %d %f %f' % (k, gcp.GCPPixel, gcp.GCPLine, gcp.GCPX, gcp.GCPY))
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, latlongSRS.ExportToWkt())
self._set_time(parse(gdalMetadata['FIRSTPACKETTIME']))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Ocean Productivity website VRT '''
if ('IDL' not in gdalMetadata['Projection Category']
and 'Source' not in gdalMetadata
and '-9999' not in gdalMetadata['Hole Value']):
raise AttributeError("BAD MAPPER")
print 'Ocean Productivity website data'
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
simFilesMask = os.path.join(iDir, '*' + iFileName[4:11] + iFileExt)
#print 'simFilesMask', simFilesMask
simFiles = glob.glob(simFilesMask)
#print 'simFiles', simFiles
metaDict = []
for simFile in simFiles:
#print 'simFile',simFile
# open subdataset with GDAL
tmpSourceFilename = simFile
tmpGdalDataset = gdal.Open(tmpSourceFilename)
# get metadata, get 'Parameter'
tmpGdalMetadata = tmpGdalDataset.GetMetadata()
iDir, ifileName = os.path.split(tmpSourceFilename)
#print 'ifileName',ifileName
simParameter = ifileName[0:3]
# set params of the similar file
simSourceFilename = tmpSourceFilename
simGdalDataset = tmpGdalDataset
simGdalMetadata = tmpGdalMetadata
# get WKV from the similar file
for param in self.param2wkv:
#print 'param', param
if param in simParameter:
simWKV = self.param2wkv[param]
break
#print 'simWKV', simWKV
# generate entry to metaDict
metaEntry = {
'src': {
'SourceFilename': simSourceFilename,
'SourceBand': 1,
'ScaleRatio': float(simGdalMetadata['Slope']),
'ScaleOffset': float(simGdalMetadata['Intercept'])
},
'dst': {
'wkv': simWKV,
'name': self.bandNames[simWKV],
'Parameter': simParameter
}
}
#print 'metaEntry', metaEntry
# append entry to metaDict
metaDict.append(metaEntry)
#get array with data and make 'mask'
a = simGdalDataset.ReadAsArray()
mask = np.zeros(a.shape, 'uint8') + 128
mask[a < -9990] = 1
self.maskVRT = VRT(array=mask)
metaDict.append({
'src': {
'SourceFilename': self.maskVRT.fileName,
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# create empty VRT dataset with geolocation only
# print 'simGdalMetadata', simGdalMetadata
latitudeStep = 0.08333334
longitudeStep = 0.08333334
numberOfColumns = 4320
numberOfLines = 2160
#longitudeStep = float(simGdalMetadata['Longitude Step'])
VRT.__init__(
self,
srcGeoTransform=(-180.0, longitudeStep, 0.0, 90.0, 0.0,
-longitudeStep),
srcProjection=
'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
srcRasterXSize=numberOfColumns,
srcRasterYSize=numberOfLines,
**kwargs)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add valid time
startYear = int(iFile[4:8])
startDay = int(iFile[8:11])
self._set_time(
datetime.datetime(startYear, 1, 1) + datetime.timedelta(startDay))