def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT '''
##############
# Get time
##############
if fileName[0:len(keywordBase)] != keywordBase:
raise AttributeError("Wrong mapper")
timestr = fileName[len(keywordBase)+1::]
time = datetime.strptime(timestr, '%Y%m%d%H%M')
######################################################
# Find windFileName corresponding to a Nansat-readable
# file in your local (or remote) file archive
######################################################
windFileName = localFolder + <.......>
######################################################
# Open file with any other Nansat mapper
######################################################
w = Nansat(windFileName)
VRT.__init__(self, vrtDataset=w.vrt.dataset)
return
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
'''
Mapping for the global 30 arc-second elevation (see
https://lta.cr.usgs.gov/GTOPO30).
Parameters:
-----------
fileName : string
Either the name of a gtopo30 DEM file, or <path>/gtopo30.vrt. The
latter is an aggregation of the DEM-files available with gtopo30
except the Antarctic one, which is in polarstereographic
projection. You can create your own gtopo30.vrt file with gdal:
> gdalbuildvrt gtopo30.vrt [E,W]*.DEM
'''
bn = os.path.basename(fileName)
if not bn=='gtopo30.vrt' and not os.path.splitext(bn)[1]=='.DEM':
raise WrongMapperError
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'height_above_reference_ellipsoid'}}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create NCEP VRT '''
if not gdalDataset:
raise WrongMapperError
geotransform = gdalDataset.GetGeoTransform()
if (geotransform != (-0.25, 0.5, 0.0, 90.25, 0.0, -0.5) or
gdalDataset.RasterCount != 2): # Not water proof
raise WrongMapperError
metaDict = [{'src': {'SourceFilename': fileName,
'SourceBand': 1},
'dst': {'wkv': 'eastward_wind',
'height': '10 m'}},
{'src': {'SourceFilename': fileName,
'SourceBand': 2},
'dst': {'wkv': 'northward_wind',
'height': '10 m'}},
{'src': [{'SourceFilename': fileName,
'SourceBand': 1,
'DataType': gdalDataset.GetRasterBand(1).DataType
},
{'SourceFilename': fileName,
'SourceBand': 2,
'DataType': gdalDataset.GetRasterBand(2).DataType
}],
'dst': {'wkv': 'wind_speed',
'PixelFunctionType': 'UVToMagnitude',
'name': 'windspeed',
'height': '2 m'
}},
{'src': [{'SourceFilename': fileName,
'SourceBand': 1,
'DataType': gdalDataset.GetRasterBand(1).DataType
},
{'SourceFilename': fileName,
'SourceBand': 2,
'DataType': gdalDataset.GetRasterBand(2).DataType
}],
'dst': {'wkv': 'wind_from_direction',
'PixelFunctionType': 'UVToDirectionFrom',
'name': 'winddirection',
'height': '2 m'
}
}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time from the GRIB file to dataset
validTime = gdalDataset.GetRasterBand(1).\
GetMetadata()['GRIB_VALID_TIME']
self._set_time(datetime.datetime.
utcfromtimestamp(int(validTime.strip().split(' ')[0])))
return
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
title_correct = False
if not gdalMetadata:
raise WrongMapperError
for key, val in gdalMetadata.iteritems():
if "title" in key:
if not val == "Daily AMSR-E Arctic lead area fraction [in percent]":
raise WrongMapperError
else:
title_correct = True
if not title_correct:
raise WrongMapperError
# initiate VRT for the NSIDC 10 km grid
VRT.__init__(
self,
srcGeoTransform=(-3850000, 6250, 0.0, 5850000, 0.0, -6250),
srcProjection=NSR(3411).wkt,
srcRasterXSize=1216,
srcRasterYSize=1792,
)
src = {"SourceFilename": 'NETCDF:"%s":lf' % fileName, "SourceBand": 1}
dst = {"name": "leadFraction", "long_name": "AMSRE sea ice lead fraction"}
self._create_band(src, dst)
self.dataset.FlushCache()
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create LANDSAT VRT '''
# try to open .tar or .tar.gz or .tgz file with tar
try:
tarFile = tarfile.open(fileName)
except:
raise WrongMapperError
tarNames = tarFile.getnames()
#print tarNames
metaDict = []
for tarName in tarNames:
if ((tarName[0] == 'L' or tarName[0] == 'M') and
(tarName[-4:] == '.TIF' or tarName[-4:] == '.tif')):
#print tarName
bandNo = tarName[-6:-4]
metaDict.append({
'src': {'SourceFilename': '/vsitar/%s/%s' % (fileName,
tarName),
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'suffix': bandNo}})
if not metaDict:
raise WrongMapperError
#print metaDict
sizeDiffBands = []
for iFile in range(len(metaDict)):
tmpName = metaDict[iFile]['src']['SourceFilename']
gdalDatasetTmp = gdal.Open(tmpName)
if iFile == 0:
gdalDatasetTmp0 = gdalDatasetTmp
xSize = gdalDatasetTmp.RasterXSize
ySize = gdalDatasetTmp.RasterYSize
elif (xSize != gdalDatasetTmp.RasterXSize or
ySize != gdalDatasetTmp.RasterYSize):
sizeDiffBands.append(iFile)
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDatasetTmp0)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# 8th band of LANDSAT8 is a double size band.
# Reduce the size to same as the 1st band.
if len(sizeDiffBands) != 0:
vrtXML = self.read_xml()
node0 = Node.create(vrtXML)
for iBand in sizeDiffBands:
iBandNode = node0.nodeList('VRTRasterBand')[iBand]
iNodeDstRect = iBandNode.node('DstRect')
iNodeDstRect.replaceAttribute('xSize', str(xSize))
iNodeDstRect.replaceAttribute('ySize', str(ySize))
self.write_xml(node0.rawxml())
def __init__(self, inputFileName, gdalDataset, gdalMetadata, logLevel=30,
**kwargs):
# check if mapper fits
if not gdalMetadata:
raise WrongMapperError
if not os.path.splitext(inputFileName)[1] == '.mnt':
raise WrongMapperError
try:
mbNorthLatitude = float(gdalMetadata['NC_GLOBAL#mbNorthLatitude'])
mbSouthLatitude = float(gdalMetadata['NC_GLOBAL#mbSouthLatitude'])
mbEastLongitude = float(gdalMetadata['NC_GLOBAL#mbEastLongitude'])
mbWestLongitude = float(gdalMetadata['NC_GLOBAL#mbWestLongitude'])
mbProj4String = gdalMetadata['NC_GLOBAL#mbProj4String']
Number_lines = int(gdalMetadata['NC_GLOBAL#Number_lines'])
Number_columns = int(gdalMetadata['NC_GLOBAL#Number_columns'])
Element_x_size = float(gdalMetadata['NC_GLOBAL#Element_x_size'])
Element_y_size = float(gdalMetadata['NC_GLOBAL#Element_y_size'])
except:
raise WrongMapperError
# find subdataset with DEPTH
subDatasets = gdalDataset.GetSubDatasets()
dSourceFile = None
for subDataset in subDatasets:
if subDataset[0].endswith('.mnt":DEPTH'):
dSourceFile = subDataset[0]
if dSourceFile is None:
raise WrongMapperError
dSubDataset = gdal.Open(dSourceFile)
dMetadata = dSubDataset.GetMetadata()
try:
scale_factor = dMetadata['DEPTH#scale_factor']
add_offset = dMetadata['DEPTH#add_offset']
except:
raise WrongMapperError
geoTransform = [mbWestLongitude, Element_x_size, 0,
mbNorthLatitude, 0, -Element_y_size]
# create empty VRT dataset with geolocation only
VRT.__init__(self, srcGeoTransform=geoTransform,
srcMetadata=gdalMetadata,
srcProjection=NSR(mbProj4String).wkt,
srcRasterXSize=Number_columns,
srcRasterYSize=Number_lines)
metaDict = [{'src': {'SourceFilename': dSourceFile,
'SourceBand': 1,
'ScaleRatio' : scale_factor,
'ScaleOffset' : add_offset},
'dst': {'wkv': 'depth'}}]
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create MODIS_L1 VRT '''
# raise error in case of not GOCI L1B
try:
title = gdalMetadata['HDFEOS_POINTS_Scene_Header_Scene_Title']
except (TypeError, KeyError):
raise WrongMapperError
if not title == 'GOCI Level-1B Data':
raise WrongMapperError
# set GOCI projection parameters
lat_0 = gdalMetadata['HDFEOS_POINTS_Map_Projection_Central_Latitude_(parallel)']
lon_0 = gdalMetadata['HDFEOS_POINTS_Map_Projection_Central_Longitude_(meridian)']
rasterXSize = int(gdalMetadata['HDFEOS_POINTS_Scene_Header_number_of_columns'].split(' ')[0])
rasterYSize = int(gdalMetadata['HDFEOS_POINTS_Scene_Header_number_of_rows'].split(' ')[0])
proj4 = '+proj=ortho +lat_0=%s +lon_0=%s units=m +ellps=WGS84 +datum=WGS84 +no_defs' % (lat_0, lon_0)
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4)
projection = srs.ExportToWkt()
geoTransform = (-1391500.0, 500.0, 0.0, 1349500.0, 0.0, -500.0)
# create empty VRT dataset with georeference only
VRT.__init__(self, srcGeoTransform=geoTransform,
srcProjection=projection,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
# add bands from subdatasets
subDatasets = gdalDataset.GetSubDatasets()
metaDict = []
wavelengths = ['412', '443', '490', '555', '660', '680', '745', '865']
for subDSI in range(8):
metaEntry = {'src': {'SourceFilename': subDatasets[subDSI][0],
'SourceBand': 1,
'ScaleRatio': 1e-6},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': wavelengths[subDSI],
'suffix': wavelengths[subDSI]}}
metaDict.append(metaEntry)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT '''
fileBaseName = os.path.basename(fileName)
if not fileBaseName == 'MOD44W.vrt':
raise WrongMapperError
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'land_binary_mask'}}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
mm = pti.get_gcmd_instrument('MODIS')
ee = pti.get_gcmd_platform('TERRA')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT '''
if (os.path.split(fileName)[1][0:4] != '101_' or
os.path.split(fileName)[1][0:4] != '102_'):
raise WrongMapperError
try:
product = gdalDataset.GetDriver().LongName
except:
raise WrongMapperError
if (product != 'GeoTIFF' or fileName[-3:] != 'tif' or
gdalDataset.RasterCount != 3):
raise WrongMapperError
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '555'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 2},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '655'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 3},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '800'}}
]
# from https://gsics.nesdis.noaa.gov/wiki/Development/StandardVariableNames
# add DataType into 'src' and name into 'dst'
for bandDict in metaDict:
if 'DataType' not in bandDict['src']:
bandDict['src']['DataType'] = 2
if 'wavelength' in bandDict['dst']:
bandDict['dst']['name'] = ('toa_radiance_' +
bandDict['dst']['wavelength'])
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def init_from_xml(self, productXml):
''' Fast init from metada in XML only '''
numberOfLines = int(productXml
.node('imageAttributes')
.node('rasterAttributes')
.node('numberOfLines')
.value)
numberOfSamples = int(productXml
.node('imageAttributes')
.node('rasterAttributes')
.node('numberOfSamplesPerLine')
.value)
VRT.__init__(self, srcRasterXSize=numberOfSamples, srcRasterYSize=numberOfLines)
gcps = []
geogrid = productXml.node('imageAttributes').node('geographicInformation').node('geolocationGrid')
for child in geogrid.children:
pix = float(child.node('imageCoordinate').node('pixel').value)
lin = float(child.node('imageCoordinate').node('line').value)
lon = float(child.node('geodeticCoordinate').node('longitude').value)
lat = float(child.node('geodeticCoordinate').node('latitude').value)
gcps.append(gdal.GCP(lon, lat, 0, pix, lin))
self.dataset.SetGCPs(gcps, NSR().wkt)
dates = list(map(parse, [child.node('timeStamp').value for child in
(productXml.node('sourceAttributes')
.node('orbitAndAttitude')
.node('orbitInformation')
.nodeList('stateVector'))]))
self.dataset.SetMetadataItem('time_coverage_start', min(dates).isoformat())
self.dataset.SetMetadataItem('time_coverage_end', max(dates).isoformat())
self.dataset.SetMetadataItem('platform', json.dumps(pti.get_gcmd_platform('radarsat-2')))
self.dataset.SetMetadataItem('instrument', json.dumps(pti.get_gcmd_instrument('SAR')))
self.dataset.SetMetadataItem('Entry Title', 'Radarsat-2 SAR')
self.dataset.SetMetadataItem('Data Center', 'CSA')
self.dataset.SetMetadataItem('ISO Topic Category', 'Oceans')
self.dataset.SetMetadataItem('Summary', 'Radarsat-2 SAR data')
def init_from_manifest_only(self, manifestXML, annotXML, missionName):
''' Create fake VRT and add metadata only from the manifest.safe '''
X, Y, lon, lat, inc, ele, numberOfSamples, numberOfLines = self.read_geolocation_lut(annotXML)
VRT.__init__(self, srcRasterXSize=numberOfSamples, srcRasterYSize=numberOfLines)
doc = ET.fromstring(manifestXML)
gcps = []
for i in range(len(X)):
gcps.append(gdal.GCP(lon[i], lat[i], 0, X[i], Y[i]))
self.dataset.SetGCPs(gcps, NSR().wkt)
self.dataset.SetMetadataItem('time_coverage_start',
doc.findall(".//*[{http://www.esa.int/safe/sentinel-1.0}startTime]")[0][0].text)
self.dataset.SetMetadataItem('time_coverage_end',
doc.findall(".//*[{http://www.esa.int/safe/sentinel-1.0}stopTime]")[0][0].text)
self.dataset.SetMetadataItem('platform', json.dumps(pti.get_gcmd_platform(missionName)))
self.dataset.SetMetadataItem('instrument', json.dumps(pti.get_gcmd_instrument('SAR')))
self.dataset.SetMetadataItem('Entry Title', missionName + ' SAR')
self.dataset.SetMetadataItem('Data Center', 'ESA/EO')
self.dataset.SetMetadataItem('ISO Topic Category', 'Oceans')
self.dataset.SetMetadataItem('Summary', missionName + ' SAR data')
def create_vrt(self, fileName, gdalDataset, gdalMetadata, date, ds, bands, cachedir):
''' Create VRT '''
if date is None:
warnings.warn('''
Date is not specified! Will return the first layer.
Please add date="YYYY-MM-DD"''')
self.fileName = fileName
self.cachedir = cachedir
self.ds = self.get_dataset(ds)
dsTime = self.get_dataset_time()
dsDatetimes = self.convert_dstime_datetimes(dsTime)
dsLayerNo, dsLayerDate = self.get_layer_datetime(date, dsDatetimes)
if bands is None:
dsVarNames = self.get_geospatial_variable_names()
else:
dsVarNames = bands
# create VRT with correct lon/lat (geotransform)
srcRasterXSize, srcRasterYSize = self.get_shape()
srcGeoTransform = self.get_geotransform()
VRT.__init__(self, srcProjection=self.srcDSProjection,
srcRasterXSize=srcRasterXSize,
srcRasterYSize=srcRasterYSize,
srcGeoTransform=srcGeoTransform)
metaDict = [self.get_metaitem(fileName, dsVarName, dsLayerNo)
for dsVarName in dsVarNames]
self._create_bands(metaDict)
# set time
timeResSecs = self.get_time_coverage_resolution()
self.dataset.SetMetadataItem('time_coverage_start', str(dsLayerDate))
self.dataset.SetMetadataItem('time_coverage_end', str(dsLayerDate + timeResSecs))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
########################################
# Read metadata from binary file
########################################
try:
fp = open(fileName, 'rb')
except IOError:
raise WrongMapperError
fp.seek(72)
try:
satNum = int(struct.unpack('<H', fp.read(2))[0])
except:
raise WrongMapperError
if satNum >= 11:
isMetop = True
else:
isMetop = False
if satNum in satIDs.keys():
satID = satIDs[satNum]
else:
raise WrongMapperError
fp.seek(76)
dataFormatNum = int(struct.unpack('<H', fp.read(2))[0])
if dataFormatNum in dataFormats.keys():
dataFormat = dataFormats[dataFormatNum]
else:
raise WrongMapperError
fp.seek(dataSetQualityIndicatorOffset + 14)
numScanLines = int(struct.unpack('<H', fp.read(2))[0])
numCalibratedScanLines = int(struct.unpack('<H', fp.read(2))[0])
missingScanLines = int(struct.unpack('<H', fp.read(2))[0])
if missingScanLines != 0:
print('WARNING: Missing scanlines: ' + str(missingScanLines))
##################
# Read time
##################
fp.seek(84)
year = int(struct.unpack('<H', fp.read(2))[0])
dayofyear = int(struct.unpack('<H', fp.read(2))[0])
millisecondsOfDay = int(struct.unpack('<l', fp.read(4))[0])
time = (
datetime.datetime(year, 1, 1) +
datetime.timedelta(dayofyear - 1, milliseconds=millisecondsOfDay))
##################################
# Read calibration information
##################################
#IRcalibration = {}
#fp.seek(202)
#avh_h_irttcoef=[]
#for i in range(24):
# avh_h_irttcoef.append(int(struct.unpack('<H', fp.read(2))[0]))
##print avh_h_irttcoef
#avh_h_albcnv=[]
#for i in range(6):
# avh_h_albcnv.append(int(struct.unpack('<l', fp.read(4))[0]))
##print avh_h_albcnv
#fp.seek(280)
#avh_h_radtempcnv = np.zeros((3,3))
#for IRchannelNo in range(3):
# for coeffNo in range(3):
# avh_h_radtempcnv[IRchannelNo, coeffNo] = \
# int(struct.unpack('<l', fp.read(4))[0])
#print avh_h_radtempcnv
#IRcalibration['centralWavenumber'] = (avh_h_radtempcnv[:,0] /
# [1E2, 1E3, 1E3])
#IRcalibration['c1'] = avh_h_radtempcnv[:,1] / 1E5
#IRcalibration['c2'] = avh_h_radtempcnv[:,2] / 1E6
########################################################
# Read visible calibration coefficients per scanline
# - for channels 1, 2, 3A
########################################################
#for scanline in range(1):
# avh_calvis=np.zeros((3,3,5))
# fp.seek(headerLength + recordLength*scanline + 48)
# for VISchannel in range(3):
# for sets in range(3):
# for coeff in range(5):
# avh_calvis[sets, VISchannel, coeff] = \
# int(struct.unpack('<l', fp.read(4))[0])
# print avh_calvis
# print '----'
########################################################
# Read IR calibration coefficients per scanline
# - for channels 3B, 4, 5
########################################################
#for scanline in range(1):
# avh_calir=np.zeros((2,3,3))
# fp.seek(headerLength + recordLength*scanline + 228)
# for IRchannelNo in range(3):
# for setNo in range(2):
# for coeffNo in range(3):
# avh_calir[setNo, IRchannelNo, coeffNo] = \
# int(struct.unpack('<l', fp.read(4))[0])
#avh_filler2 = np.zeros(3)
#for fillerNo in range(3):
# avh_filler2[fillerNo] = int(struct.unpack('<l', fp.read(4))[0])
#setNo = 0 # Use operational set (the only available)
#a = np.zeros((3,3))
#for IRchannelNo in range(3):
# for coeffNo in range(3):
# # NB: apparently stored "backwards", therefore 2-coeffNo
# a[IRchannelNo, 2-coeffNo] = (avh_calir[setNo, IRchannelNo,
# coeffNo]
# / np.power(10,
# avh_filler2[coeffNo]))
#
###########################
# Apply calibration
###########################
#C = 410
#for IRchannelNo in range(3):
# Ne = a[IRchannelNo,0] + a[IRchannelNo,1]*C + a[IRchannelNo,2]*C*C
# vC = IRcalibration['centralWavenumber'][IRchannelNo]
# c1 = -IRcalibration['c1'][IRchannelNo] # Note minus
# c2 = IRcalibration['c2'][IRchannelNo]
#print '-----'
#print a[IRchannelNo,:]
#print vC, c1, c2
#TeStar = c2*vC/np.log(1 + (c1*vC*vC*vC)/Ne)
#Te = c1 + c2*TeStar
#print Ne, TeStar, Te
#print '-----'
#sys.exit('stop')
###########################
# Make Geolocation Arrays
###########################
srcRasterYSize = numCalibratedScanLines
# Making VRT with raw (unscaled) lon and lat
# (smaller bands than full dataset)
self.bandVRTs = {
'RawGeolocVRT': VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
}
RawGeolocMetaDict = []
for lonlatNo in range(1, 3):
RawGeolocMetaDict.append({
'src': {
'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'DataType': gdal.GDT_Int32,
'ImageOffset': (headerLength + 640 + (lonlatNo - 1) * 4),
'PixelOffset': 8,
'LineOffset': recordLength,
'ByteOrder': 'LSB'
},
'dst': {}
})
self.bandVRTs['RawGeolocVRT']._create_bands(RawGeolocMetaDict)
# Make derived GeolocVRT with scaled lon and lat
self.bandVRTs['GeolocVRT'] = VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
GeolocMetaDict = []
for lonlatNo in range(1, 3):
GeolocMetaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['RawGeolocVRT'].fileName),
'SourceBand': lonlatNo,
'ScaleRatio': 0.0001,
'ScaleOffset': 0,
'DataType': gdal.GDT_Int32
},
'dst': {}
})
self.bandVRTs['GeolocVRT']._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.bandVRTs['GeolocVRT'],
yVRT=self.bandVRTs['GeolocVRT'],
xBand=2,
yBand=1, # x = lon, y = lat
lineOffset=0,
pixelOffset=25,
lineStep=1,
pixelStep=40)
#######################
# Initialize dataset
#######################
# create empty VRT dataset with geolocation only
# (from Geolocation Array)
VRT.__init__(self,
srcRasterXSize=2048,
srcRasterYSize=numCalibratedScanLines,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Since warping quality is horrible using geolocation arrays
# which are much smaller than raster bands (due to a bug in GDAL:
# http://trac.osgeo.org/gdal/ticket/4907), the geolocation arrays
# are here converted to GCPs. Only a subset of GCPs is added,
# significantly increasing speed when using -tps warping
reductionFactor = 2
self.convert_GeolocationArray2GPCs(1 * reductionFactor,
40 * reductionFactor)
##################
# Create bands
##################
metaDict = []
ch = ({}, {}, {}, {}, {}, {})
ch[1]['wavelength'] = 0.63
ch[2]['wavelength'] = 0.86
ch[3]['wavelength'] = '1.6 or 3.7 mum'
ch[4]['wavelength'] = 10.8
ch[5]['wavelength'] = 12.0
ch[1]['minmax'] = '0 700'
ch[2]['minmax'] = '0 700'
ch[3]['minmax'] = '0 800'
ch[4]['minmax'] = '400 1000'
ch[5]['minmax'] = '400 1000'
for bandNo in range(1, 6):
metaDict.append({
'src': {
'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'dataType': gdal.GDT_UInt16,
'ImageOffset': imageOffset + (bandNo - 1) * 2,
'PixelOffset': 10,
'LineOffset': recordLength,
'ByteOrder': 'LSB'
},
'dst': {
'dataType': gdal.GDT_UInt16,
'wkv': 'raw_counts',
'colormap': 'gray',
'wavelength': ch[bandNo]['wavelength'],
'minmax': ch[bandNo]['minmax'],
'unit': "1"
}
})
self._create_bands(metaDict)
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start', time.isoformat())
self.dataset.SetMetadataItem('time_coverage_end', time.isoformat())
return
def __init__(self, inputFileName, gdalDataset, gdalMetadata, logLevel=30,
rmMetadatas=['NETCDF_VARNAME', '_Unsigned',
'ScaleRatio', 'ScaleOffset', 'dods_variable'],
**kwargs):
# Remove 'NC_GLOBAL#' and 'GDAL_' and 'NANSAT_'
# from keys in gdalDataset
tmpGdalMetadata = {}
geoMetadata = {}
origin_is_nansat = False
if not gdalMetadata:
raise WrongMapperError
for key in gdalMetadata.keys():
newKey = key.replace('NC_GLOBAL#', '').replace('GDAL_', '')
if 'NANSAT_' in newKey:
geoMetadata[newKey.replace('NANSAT_', '')] = gdalMetadata[key]
origin_is_nansat = True
else:
tmpGdalMetadata[newKey] = gdalMetadata[key]
gdalMetadata = tmpGdalMetadata
fileExt = os.path.splitext(inputFileName)[1]
# Get file names from dataset or subdataset
subDatasets = gdalDataset.GetSubDatasets()
if len(subDatasets) == 0:
fileNames = [inputFileName]
else:
fileNames = [f[0] for f in subDatasets]
# add bands with metadata and corresponding values to the empty VRT
metaDict = []
xDatasetSource = ''
yDatasetSource = ''
firstXSize = 0
firstYSize = 0
for _, fileName in enumerate(fileNames):
subDataset = gdal.Open(fileName)
# choose the first dataset whith grid
if (firstXSize == 0 and firstYSize == 0 and
subDataset.RasterXSize > 1 and subDataset.RasterYSize > 1):
firstXSize = subDataset.RasterXSize
firstYSize = subDataset.RasterYSize
firstSubDataset = subDataset
# get projection from the first subDataset
projection = firstSubDataset.GetProjection()
# take bands whose sizes are same as the first band.
if (subDataset.RasterXSize == firstXSize and
subDataset.RasterYSize == firstYSize):
if projection == '':
projection = subDataset.GetProjection()
if ('GEOLOCATION_X_DATASET' in fileName or
'longitude' in fileName):
xDatasetSource = fileName
elif ('GEOLOCATION_Y_DATASET' in fileName or
'latitude' in fileName):
yDatasetSource = fileName
else:
for iBand in range(subDataset.RasterCount):
subBand = subDataset.GetRasterBand(iBand+1)
bandMetadata = subBand.GetMetadata_Dict()
if 'PixelFunctionType' in bandMetadata:
bandMetadata.pop('PixelFunctionType')
sourceBands = iBand + 1
# sourceBands = i*subDataset.RasterCount + iBand + 1
# generate src metadata
src = {'SourceFilename': fileName,
'SourceBand': sourceBands}
# set scale ratio and scale offset
scaleRatio = bandMetadata.get(
'ScaleRatio',
bandMetadata.get(
'scale',
bandMetadata.get('scale_factor', '')))
if len(scaleRatio) > 0:
src['ScaleRatio'] = scaleRatio
scaleOffset = bandMetadata.get(
'ScaleOffset',
bandMetadata.get(
'offset',
bandMetadata.get(
'add_offset', '')))
if len(scaleOffset) > 0:
src['ScaleOffset'] = scaleOffset
# sate DataType
src['DataType'] = subBand.DataType
# generate dst metadata
# get all metadata from input band
dst = bandMetadata
# set wkv and bandname
dst['wkv'] = bandMetadata.get('standard_name', '')
# first, try the name metadata
if 'name' in bandMetadata:
bandName = bandMetadata['name']
else:
# if it doesn't exist get name from NETCDF_VARNAME
bandName = bandMetadata.get('NETCDF_VARNAME', '')
if len(bandName) == 0:
bandName = bandMetadata.get(
'dods_variable', ''
)
# remove digits added by gdal in
# exporting to netcdf...
if (len(bandName) > 0 and origin_is_nansat and
fileExt == '.nc'):
if bandName[-1:].isdigit():
bandName = bandName[:-1]
if bandName[-1:].isdigit():
bandName = bandName[:-1]
# if still no bandname, create one
if len(bandName) == 0:
bandName = 'band_%03d' % iBand
dst['name'] = bandName
# remove non-necessary metadata from dst
for rmMetadata in rmMetadatas:
if rmMetadata in dst:
dst.pop(rmMetadata)
# append band with src and dst dictionaries
metaDict.append({'src': src, 'dst': dst})
# create empty VRT dataset with geolocation only
VRT.__init__(self, firstSubDataset, srcMetadata=gdalMetadata)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Create complex data bands from 'xxx_real' and 'xxx_imag' bands
# using pixelfunctions
rmBands = []
for iBandNo in range(self.dataset.RasterCount):
iBand = self.dataset.GetRasterBand(iBandNo + 1)
iBandName = iBand.GetMetadataItem('name')
# find real data band
if iBandName.find("_real") != -1:
realBandNo = iBandNo
realBand = self.dataset.GetRasterBand(realBandNo + 1)
realDtype = realBand.GetMetadataItem('DataType')
bandName = iBandName.replace(iBandName.split('_')[-1],
'')[0:-1]
for jBandNo in range(self.dataset.RasterCount):
jBand = self.dataset.GetRasterBand(jBandNo + 1)
jBandName = jBand.GetMetadataItem('name')
# find an imaginary data band corresponding to the real
# data band and create complex data band from the bands
if jBandName.find(bandName+'_imag') != -1:
imagBandNo = jBandNo
imagBand = self.dataset.GetRasterBand(imagBandNo + 1)
imagDtype = imagBand.GetMetadataItem('DataType')
dst = imagBand.GetMetadata()
dst['name'] = bandName
dst['PixelFunctionType'] = 'ComplexData'
dst['dataType'] = 10
src = [{'SourceFilename': fileNames[realBandNo],
'SourceBand': 1,
'DataType': realDtype},
{'SourceFilename': fileNames[imagBandNo],
'SourceBand': 1,
'DataType': imagDtype}]
self._create_band(src, dst)
self.dataset.FlushCache()
rmBands.append(realBandNo + 1)
rmBands.append(imagBandNo + 1)
# Delete real and imaginary bands
if len(rmBands) != 0:
self.delete_bands(rmBands)
if len(projection) == 0:
# projection was not set automatically
# get projection from GCPProjection
projection = geoMetadata.get('GCPProjection', '')
if len(projection) == 0:
# no projection was found in dataset or metadata:
# generate WGS84 by default
projection = NSR().wkt
# fix problem with MET.NO files where a, b given in m and XC/YC in km
if ('UNIT["kilometre"' in projection and
',SPHEROID["Spheroid",6378273,7.331926543631893e-12]' in
projection):
projection = projection.replace(
',SPHEROID["Spheroid",6378273,7.331926543631893e-12]',
'')
# set projection
self.dataset.SetProjection(self.repare_projection(projection))
# check if GCPs were added from input dataset
gcps = firstSubDataset.GetGCPs()
gcpProjection = firstSubDataset.GetGCPProjection()
# if no GCPs in input dataset: try to add GCPs from metadata
if not gcps:
gcps = self.add_gcps_from_metadata(geoMetadata)
# if yet no GCPs: try to add GCPs from variables
if not gcps:
gcps = self.add_gcps_from_variables(inputFileName)
if gcps:
if len(gcpProjection) == 0:
# get GCP projection and repare
gcpProjection = self.repare_projection(geoMetadata.
get('GCPProjection', ''))
# add GCPs to dataset
self.dataset.SetGCPs(gcps, gcpProjection)
self.dataset.SetProjection('')
self._remove_geotransform()
# Find proper bands and insert GEOLOCATION ARRAY into dataset
if len(xDatasetSource) > 0 and len(yDatasetSource) > 0:
self.add_geolocationArray(GeolocationArray(xDatasetSource,
yDatasetSource))
elif not gcps:
# if no GCPs found and not GEOLOCATION ARRAY set:
# Set Nansat Geotransform if it is not set automatically
geoTransform = self.dataset.GetGeoTransform()
if len(geoTransform) == 0:
geoTransformStr = geoMetadata.get('GeoTransform',
'(0|1|0|0|0|0|1)')
geoTransform = eval(geoTransformStr.replace('|', ','))
self.dataset.SetGeoTransform(geoTransform)
subMetadata = firstSubDataset.GetMetadata()
### GET START TIME from METADATA
time_coverage_start = None
if 'start_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['start_time'])
elif 'start_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['start_date'])
elif 'time_coverage_start' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_start'])
### GET END TIME from METADATA
time_coverage_end = None
if 'stop_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['stop_time'])
elif 'stop_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['stop_date'])
elif 'time_coverage_stop' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_stop'])
elif 'end_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['end_time'])
elif 'end_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['end_date'])
elif 'time_coverage_end' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_end'])
### GET start time from time variable
if (time_coverage_start is None and cfunitsInstalled and
'time#standard_name' in subMetadata and
subMetadata['time#standard_name'] == 'time' and
'time#units' in subMetadata and
'time#calendar' in subMetadata):
# get data from netcdf data
ncFile = netcdf_file(inputFileName, 'r')
timeLength = ncFile.variables['time'].shape[0]
timeValueStart = ncFile.variables['time'][0]
timeValueEnd = ncFile.variables['time'][-1]
ncFile.close()
try:
timeDeltaStart = Units.conform(timeValueStart,
Units(subMetadata['time#units'],
calendar=subMetadata['time#calendar']),
Units('days since 1950-01-01'))
except ValueError:
self.logger.error('calendar units are wrong: %s' %
subMetadata['time#calendar'])
else:
time_coverage_start = (datetime.datetime(1950,1,1) +
datetime.timedelta(float(timeDeltaStart)))
if timeLength > 1:
timeDeltaEnd = Units.conform(timeValueStart,
Units(subMetadata['time#units'],
calendar=subMetadata['time#calendar']),
Units('days since 1950-01-01'))
else:
timeDeltaEnd = timeDeltaStart + 1
time_coverage_end = (datetime.datetime(1950,1,1) +
datetime.timedelta(float(timeDeltaEnd)))
## finally set values of time_coverage start and end if available
if time_coverage_start is not None:
self.dataset.SetMetadataItem('time_coverage_start',
time_coverage_start.isoformat())
if time_coverage_end is not None:
self.dataset.SetMetadataItem('time_coverage_end',
time_coverage_end.isoformat())
if 'sensor' not in gdalMetadata:
self.dataset.SetMetadataItem('sensor', 'unknown')
if 'satellite' not in gdalMetadata:
self.dataset.SetMetadataItem('satellite', 'unknown')
if 'source_type' not in gdalMetadata:
self.dataset.SetMetadataItem('source_type', 'unknown')
if 'platform' not in gdalMetadata:
self.dataset.SetMetadataItem('platform', 'unknown')
if 'instrument' not in gdalMetadata:
self.dataset.SetMetadataItem('instrument', 'unknown')
self.logger.info('Use generic mapper - OK!')
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' OBPG L3 VRT '''
# test the product
try:
assert gdalMetadata['PlatformShortName'] == 'GCOM-W1'
assert gdalMetadata['SensorShortName'] == 'AMSR2'
assert gdalMetadata['ProductName'] == 'AMSR2-L3'
except:
raise WrongMapperError
# get list of similar (same date, A/D orbit) files in the directory
iDir, iFile = os.path.split(fileName)
iFileMask = iFile[:30] + '%02d' + iFile[32:]
simFiles = []
for freq in self.freqs:
simFile = os.path.join(iDir, iFileMask % freq)
#print simFile
if os.path.exists(simFile):
simFiles.append(simFile)
metaDict = []
for freq in self.freqs:
simFile = os.path.join(iDir, iFileMask % freq)
if simFile not in simFiles:
continue
#print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
if simSupDataset is None:
# skip this similar file
#print 'No dataset: %s not a supported SMI file' % simFile
continue
# get subdatasets from the similar file
simSubDatasets = simSupDataset.GetSubDatasets()
for simSubDataset in simSubDatasets:
#print 'simSubDataset', simSubDataset
if 'Brightness_Temperature' in simSubDataset[0]:
# get SourceFilename from subdataset
metaEntry = {
'src': {
'SourceFilename': simSubDataset[0],
'SourceBand': 1,
'ScaleRatio': 0.0099999998,
'ScaleOffset': 0
},
'dst': {
'wkv': 'brightness_temperature',
'frequency': '%02d' % freq,
'polarisation': simSubDataset[0][-2:-1],
'suffix':
('%02d%s' % (freq, simSubDataset[0][-2:-1]))
}
}
metaDict.append(metaEntry)
# initiate VRT for the NSIDC 10 km grid
VRT.__init__(self,
srcGeoTransform=(-3850000, 10000, 0.0, 5850000, 0.0,
-10000),
srcProjection=NSR(3411).wkt,
srcRasterXSize=760,
srcRasterYSize=1120)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time to dataset
self.dataset.SetMetadataItem(
'time_coverage_start',
parse(gdalMetadata['ObservationStartDateTime']).isoformat())
self.dataset.SetMetadataItem(
'time_coverage_end',
parse(gdalMetadata['ObservationStartDateTime']).isoformat())
mm = pti.get_gcmd_instrument('AMSR2')
ee = pti.get_gcmd_platform('GCOM-W1')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
geolocation=False,
zoomSize=500,
step=1,
**kwargs):
''' Create MER1 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
geolocation : bool (default is False)
if True, add gdal geolocation
zoomSize: int (used in envisat.py)
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int (used in envisat.py)
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
'''
self.setup_ads_parameters(fileName, gdalMetadata)
if (self.product[0:9] != "MER_FRS_1"
and self.product[0:9] != "MER_RR__1"):
raise WrongMapperError
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '413'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 2
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '443'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 3
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '490'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 4
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '510'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 5
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '560'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 6
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '620'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 7
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '665'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 8
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '681'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 9
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '709'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 10
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '753'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 11
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '761'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 12
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '778'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 13
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '864'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 14
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '849'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 15
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': '900'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 16,
'DataType': 1
},
'dst': {
'wkv': 'quality_flags',
'suffix': 'l1'
}
}]
# add DataType into 'src' and suffix into 'dst'
for bandDict in metaDict:
if 'DataType' not in bandDict['src']:
# default for meris L1 DataType UInt16
bandDict['src']['DataType'] = 2
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
# get scaling GADS from header
scales = self.read_scaling_gads(range(7, 22))
# set scale factor to the band metadata (only radiances)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
# get list with resized VRTs from ADS
self.bandVRTs = {
'adsVRTs':
self.get_ads_vrts(gdalDataset, [
'sun zenith angles', 'sun azimuth angles', 'zonal winds',
'meridional winds'
],
zoomSize=zoomSize,
step=step)
}
# add bands from the ADS VRTs
for adsVRT in self.bandVRTs['adsVRTs']:
metaDict.append({
'src': {
'SourceFilename': adsVRT.fileName,
'SourceBand': 1
},
'dst': {
'name':
(adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name')),
'units':
(adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units'))
}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('meris')
ee = pti.get_gcmd_platform('envisat')
# TODO: Validate that the found instrument and platform are indeed what we
# want....
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
# add geolocation arrays
if geolocation:
self.add_geolocation_from_ads(gdalDataset,
zoomSize=zoomSize,
step=step)
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_COUNT0=5, GCP_COUNT1=20, pixelStep=1,
lineStep=1, **kwargs):
''' Create VIIRS VRT '''
if not 'GMTCO_npp_' in fileName:
raise WrongMapperError
ifiledir = os.path.split(fileName)[0]
ifiles = glob.glob(ifiledir + 'SVM??_npp_d*_obpg_ops.h5')
ifiles.sort()
viirsWavelengths = [None, 412, 445, 488, 555, 672, 746, 865, 1240,
1378, 1610, 2250, 3700, 4050, 8550, 10736, 12013]
# create empty VRT dataset with geolocation only
xDatasetSource = ('HDF5:"%s"://All_Data/VIIRS-MOD-GEO-TC_All/Longitude'
% fileName)
xDatasetBand = 1
xDataset = gdal.Open(xDatasetSource)
VRT.__init__(self, xDataset)
metaDict = []
for ifile in ifiles:
ifilename = os.path.split(ifile)[1]
print ifilename
bNumber = int(ifilename[3:5])
print bNumber
bWavelength = viirsWavelengths[bNumber]
print bWavelength
SourceFilename = ('HDF5:"%s"://All_Data/VIIRS-M%d-SDR_All/Radiance'
% (ifile, bNumber))
print SourceFilename
metaEntry = {'src': {'SourceFilename': SourceFilename,
'SourceBand': 1},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': str(bWavelength),
'suffix': str(bWavelength)}
}
metaDict.append(metaEntry)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
xVRTArray = xDataset.ReadAsArray()
xVRTArray = gaussian_filter(xVRTArray, 5).astype('float32')
xVRT = VRT(array=xVRTArray)
yDatasetSource = ('HDF5:"%s"://All_Data/VIIRS-MOD-GEO-TC_All/Latitude'
% fileName)
yDatasetBand = 1
yDataset = gdal.Open(yDatasetSource)
yVRTArray = yDataset.ReadAsArray()
yVRTArray = gaussian_filter(yVRTArray, 5).astype('float32')
yVRT = VRT(array=yVRTArray)
#self.add_geolocationArray(GeolocationArray(xDatasetSource,
# yDatasetSource))
#"""
# estimate pixel/line step
self.logger.debug('pixel/lineStep %f %f' % (pixelStep, lineStep))
# ==== ADD GCPs and Pojection ====
# get lat/lon matrices
longitude = xVRT.dataset.GetRasterBand(1).ReadAsArray()
latitude = yVRT.dataset.GetRasterBand(1).ReadAsArray()
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT0))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT1))
self.logger.debug('gcpCount: %d %d %d %d, %d %d',
latitude.shape[0], latitude.shape[1],
GCP_COUNT0, GCP_COUNT1, step0, step1)
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0,
i1 * pixelStep, i0 * lineStep)
self.logger.debug('%d %d %d %f %f',
k, gcp.GCPPixel, gcp.GCPLine,
gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
# remove geolocation array
self.remove_geolocationArray()
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' GLOBCOLOR L3M VRT '''
try:
print "=>%s<=" % gdalMetadata['NC_GLOBAL#title']
except (TypeError, KeyError):
raise WrongMapperError
if 'GlobColour' not in gdalMetadata['NC_GLOBAL#title']:
raise WrongMapperError
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
print 'idir:', iDir, iFile, iFileName[0:30], iFileExt[0:8]
simFilesMask = os.path.join(iDir, iFileName[0:30] + '*.nc')
simFiles = glob.glob(simFilesMask)
print 'simFilesMask, simFiles', simFilesMask, simFiles
metaDict = []
for simFile in simFiles:
print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
simSubDatasets = simSupDataset.GetSubDatasets()
simWKV = None
for simSubDataset in simSubDatasets:
if '_mean' in simSubDataset[0]:
simValidSupDataset = simSupDataset
simGdalDataset = gdal.Open(simSubDataset[0])
simBand = simGdalDataset.GetRasterBand(1)
simBandMetadata = simBand.GetMetadata()
simVarname = simBandMetadata['NETCDF_VARNAME']
# get WKV
print ' simVarname', simVarname
if simVarname in self.varname2wkv:
simWKV = self.varname2wkv[simVarname]
break
# skipp adding this similar file if it is not valid
if simWKV is None:
continue
metaEntry = {
'src': {
'SourceFilename': simSubDataset[0],
'SourceBand': 1
},
'dst': {
'wkv': simWKV,
'original_name': simVarname
}
}
# add wavelength and name
longName = simBandMetadata['long_name']
if 'Fully normalised water leaving radiance' in longName:
simWavelength = simVarname.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add band with rrsw
metaEntry2 = None
if simWKV == 'surface_upwelling_spectral_radiance_in_air_emerging_from_sea_water':
solarIrradiance = simBandMetadata['solar_irradiance']
metaEntry2 = {'src': metaEntry['src']}
metaEntry2['dst'] = {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix':
simWavelength,
'wavelength':
simWavelength,
#'expression': 'self["nLw_%s"] / %s / (0.52 + 1.7 * self["nLw_%s"] / %s)' % (simWavelength, solarIrradiance, simWavelength, solarIrradiance),
'expression':
'self["nLw_%s"] / %s' % (simWavelength, solarIrradiance)
}
print ' metaEntry', metaEntry
metaDict.append(metaEntry)
if metaEntry2 is not None:
print ' metaEntry2', metaEntry2
metaDict.append(metaEntry2)
print 'simSubDatasets', simValidSupDataset.GetSubDatasets()
for simSubDataset in simValidSupDataset.GetSubDatasets():
print 'simSubDataset', simSubDataset
if '_flags ' in simSubDataset[1]:
print ' mask simSubDataset', simSubDataset[1]
flags = gdal.Open(simSubDataset[0]).ReadAsArray()
mask = np.ones(flags.shape) * 64
mask[np.bitwise_and(flags, np.power(2, 0)) > 0] = 1
mask[np.bitwise_and(flags, np.power(2, 3)) > 0] = 2
self.bandVRTs = {'maskVRT': VRT(array=mask)}
metaDict.append({
'src': {
'SourceFilename': self.bandVRTs['maskVRT'].fileName,
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# create empty VRT dataset with geolocation only
simGdalDataset.SetProjection(
'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'
)
VRT.__init__(self, simGdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata,
outFolder=downloads, **kwargs):
''' Create NCEP VRT '''
if not os.path.exists(outFolder):
os.mkdir(outFolder)
##############
# Get time
##############
keywordBase = 'ncep_wind_online'
if fileName[0:len(keywordBase)] != keywordBase:
raise WrongMapperError
timestr = fileName[len(keywordBase)+1::]
time = datetime.strptime(timestr, '%Y%m%d%H%M')
print time
########################################
# Find and download online grib file
########################################
# Find closest 6 hourly modelrun and forecast hour
modelRunHour = round((time.hour + time.minute/60.)/6)*6
nearestModelRun = (datetime(time.year, time.month, time.day)
+ timedelta(hours=modelRunHour))
if sys.version_info < (2, 7):
td = (time - nearestModelRun)
forecastHour = (td.microseconds +
(td.seconds + td.days * 24 * 3600)
* 10**6) / 10**6 / 3600.
else:
forecastHour = (time - nearestModelRun).total_seconds()/3600.
if modelRunHour == 24:
modelRunHour = 0
if forecastHour < 1.5:
forecastHour = 0
else:
forecastHour = 3
#########################################################
# Try first to get NRT data from
# ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/
# - avaliable approximately the latest month
#########################################################
url = ('ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/' +
'gfs.' + nearestModelRun.strftime('%Y%m%d') +
'%.2d' % modelRunHour +
'/gfs.t' + '%.2d' % modelRunHour + 'z.master.grbf' +
'%.3d' % forecastHour + '.10m.uv.grib2')
outFileName = os.path.join(outFolder,
('ncep_gfs_' +
nearestModelRun.strftime('%Y%m%d_%HH_') +
'%.2d' % forecastHour +
'.10m.uv.grib2'))
if os.path.exists(outFileName):
print 'NCEP wind is already downloaded: ' + outFileName
else:
os.system('curl -so ' + outFileName + ' ' + url)
if os.path.exists(outFileName):
print 'Downloaded ' + outFileName
else:
print 'NRT GRIB file not available: ' + url
#########################################################
# If NRT file not available, search in long term archive
#########################################################
url = ('http://nomads.ncdc.noaa.gov/data/gfs4/' +
nearestModelRun.strftime('%Y%m/%Y%m%d/'))
baseName = ('gfs_4_' + nearestModelRun.strftime('%Y%m%d_') +
nearestModelRun.strftime('%H%M_') +
'%.3d' % forecastHour)
fileName = baseName + '.grb2'
outFileName = os.path.join(outFolder, fileName)
print 'Downloading ' + url + fileName
# Download subset of grib file
mapperDir = os.path.dirname(os.path.abspath(__file__))
get_inv = os.path.join(mapperDir, 'get_inv.pl')
if not os.path.isfile(get_inv):
raise IOError('%s: File not found' % get_inv)
get_grib = os.path.join(mapperDir, 'get_grib.pl')
if not os.path.isfile(get_grib):
raise IOError('%s: File not found' % get_grib)
if not os.path.isfile(outFileName):
command = (get_inv + ' ' + url + baseName +
'.inv | egrep "(:UGRD:10 m |:VGRD:10 m )" | ' +
get_grib + ' ' + url + fileName +
' ' + outFileName)
os.system(command)
if os.path.isfile(outFileName):
print 'Downloaded ' + fileName + ' to ' + outFolder
else:
print 'Already downloaded %s' % outFileName
if not os.path.isfile(outFileName):
sys.exit('No NCEP wind files found for requested time')
######################################################
# Open downloaded grib file with a(ny) Nansat mapper
######################################################
w = Nansat(outFileName)
VRT.__init__(self, vrtDataset=w.vrt.dataset)
return
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
geolocation=False,
zoomSize=500,
step=1,
**kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
geolocation : bool (default is False)
if True, add gdal geolocation
zoomSize: int (used in envisat.py)
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int (used in envisat.py)
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
'''
self.setup_ads_parameters(fileName, gdalMetadata)
if self.product[0:9] != "MER_FRS_2" and self.product[
0:9] != "MER_RR__2":
raise WrongMapperError(fileName)
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '412'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 2
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '443'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 3
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '490'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 4
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '510'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 5
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '560'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 6
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '620'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 7
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '665'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 8
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '680'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 9
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '708'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 10
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '753'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 11
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '761'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 12
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '778'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 13
},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '864'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 15
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1_log',
'case': 'I'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 16
},
'dst': {
'wkv':
'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2_log',
'case': 'II'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 17
},
'dst': {
'wkv': 'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2_log',
'case': 'II'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 18
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2_log',
'case': 'II'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 22
},
'dst': {
'wkv': 'quality_flags',
'suffix': 'l2'
}
}]
# add 'name' to 'parameters'
for bandDict in metaDict:
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
#get GADS from header
scales = self.read_scaling_gads(range(7, 20) + [20, 21, 22, 20])
offsets = self.read_scaling_gads(range(33, 46) + [46, 47, 48, 46])
# set scale/offset to the band metadata (only reflectance)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
bandDict['src']['ScaleOffset'] = str(offsets[i])
# add log10-scaled variables
metaDict += [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1',
'case': 'I',
'expression': 'np.power(10., self["chlor_a_1_log"])'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2',
'case': 'II',
'expression': 'np.power(10., self["chlor_a_2_log"])'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv':
'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2',
'case': 'II',
'expression': 'np.power(10., self["cdom_a_2_log"])'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2',
'case': 'II',
'expression': 'np.power(10., self["tsm_2_log"])'
}
}]
# get list with resized VRTs from ADS
self.bandVRTs = {
'adsVRTs':
self.get_ads_vrts(gdalDataset, [
'sun zenith angles', 'sun azimuth angles', 'zonal winds',
'meridional winds'
],
zoomSize=zoomSize,
step=step)
}
# add bands from the ADS VRTs
for adsVRT in self.bandVRTs['adsVRTs']:
metaDict.append({
'src': {
'SourceFilename': adsVRT.fileName,
'SourceBand': 1
},
'dst': {
'name':
(adsVRT.dataset.GetRasterBand(1).GetMetadataItem('name')),
'units':
(adsVRT.dataset.GetRasterBand(1).GetMetadataItem('units'))
}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if geolocation:
self.add_geolocation_from_ads(gdalDataset,
zoomSize=zoomSize,
step=step)
# set time
self._set_envisat_time(gdalMetadata)
self.dataset.SetMetadataItem('sensor', 'MERIS')
self.dataset.SetMetadataItem('satellite', 'ENVISAT')
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
'''
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
'''
self.setup_ads_parameters(fileName, gdalMetadata)
if self.product[0:4] != "ASA_":
raise WrongMapperError
# get channel string (remove '/', since NetCDF
# does not support that in metadata)
polarization = [{'channel': gdalMetadata['SPH_MDS1_TX_RX_POLAR']
.replace("/", ""), 'bandNum': 1}]
# if there is the 2nd band, get channel string
if 'SPH_MDS2_TX_RX_POLAR' in gdalMetadata.keys():
channel = gdalMetadata['SPH_MDS2_TX_RX_POLAR'].replace("/", "")
if not(channel.isspace()):
polarization.append({'channel': channel,
'bandNum': 2})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# get calibration constant
gotCalibration = True
try:
for iPolarization in polarization:
metaKey = ('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS.%d.EXT_CAL_FACT'
% (iPolarization['bandNum']))
iPolarization['calibrationConst'] = float(
gdalDataset.GetMetadataItem(metaKey, 'records'))
except:
try:
for iPolarization in polarization:
# Apparently some ASAR files have calibration
# constant stored in another place
metaKey = ('MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS.%d.EXT_CAL_FACT'
% (iPolarization['bandNum']))
iPolarization['calibrationConst'] = float(
gdalDataset.GetMetadataItem(metaKey, 'records'))
except:
self.logger.warning('Cannot get calibrationConst')
gotCalibration = False
# add dictionary for raw counts
metaDict = []
for iPolarization in polarization:
metaDict.append({'src': {'SourceFilename': fileName,
'SourceBand': iPolarization['bandNum']},
'dst': {'name': 'raw_counts_%s'
% iPolarization['channel']}})
#####################################################################
# Add incidence angle and look direction through small VRT objects
#####################################################################
lon = self.get_array_from_ADS('first_line_longs')
lat = self.get_array_from_ADS('first_line_lats')
inc = self.get_array_from_ADS('first_line_incidence_angle')
# Calculate SAR look direction (ASAR is always right-looking)
SAR_look_direction = initial_bearing(lon[:, :-1], lat[:, :-1],
lon[:, 1:], lat[:, 1:])
# Interpolate to regain lost row
SAR_look_direction = scipy.ndimage.interpolation.zoom(
SAR_look_direction, (1, 11./10.))
# Decompose, to avoid interpolation errors around 0 <-> 360
SAR_look_direction_u = np.sin(np.deg2rad(SAR_look_direction))
SAR_look_direction_v = np.cos(np.deg2rad(SAR_look_direction))
look_u_VRT = VRT(array=SAR_look_direction_u, lat=lat, lon=lon)
look_v_VRT = VRT(array=SAR_look_direction_v, lat=lat, lon=lon)
# Note: If incidence angle and look direction are stored in
# same VRT, access time is about twice as large
incVRT = VRT(array=inc, lat=lat, lon=lon)
lookVRT = VRT(lat=lat, lon=lon)
lookVRT._create_band([{'SourceFilename': look_u_VRT.fileName,
'SourceBand': 1},
{'SourceFilename': look_v_VRT.fileName,
'SourceBand': 1}],
{'PixelFunctionType': 'UVToDirectionTo'})
# Blow up bands to full size
incVRT = incVRT.get_resized_vrt(gdalDataset.RasterXSize,
gdalDataset.RasterYSize)
lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize,
gdalDataset.RasterYSize)
# Store VRTs so that they are accessible later
self.bandVRTs = {'incVRT': incVRT,
'look_u_VRT': look_u_VRT,
'look_v_VRT': look_v_VRT,
'lookVRT': lookVRT}
# Add band to full sized VRT
incFileName = self.bandVRTs['incVRT'].fileName
lookFileName = self.bandVRTs['lookVRT'].fileName
metaDict.append({'src': {'SourceFilename': incFileName,
'SourceBand': 1},
'dst': {'wkv': 'angle_of_incidence',
'name': 'incidence_angle'}})
metaDict.append({'src': {'SourceFilename': lookFileName,
'SourceBand': 1},
'dst': {'wkv': 'sensor_azimuth_angle',
'name': 'SAR_look_direction'}})
####################
# Add Sigma0-bands
####################
if gotCalibration:
for iPolarization in polarization:
# add dictionary for sigma0, ice and water
short_names = ['sigma0', 'sigma0_normalized_ice',
'sigma0_normalized_water']
wkt = [
'surface_backwards_scattering_coefficient_of_radar_wave',
'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_ice',
'surface_backwards_scattering_coefficient_of_radar_wave_normalized_over_water']
sphPass = [gdalMetadata['SPH_PASS'], '', '']
sourceFileNames = [fileName, incFileName]
pixelFunctionTypes = ['RawcountsIncidenceToSigma0',
'Sigma0NormalizedIce']
if iPolarization['channel'] == 'HH':
pixelFunctionTypes.append('Sigma0HHNormalizedWater')
elif iPolarization['channel'] == 'VV':
pixelFunctionTypes.append('Sigma0VVNormalizedWater')
# add pixelfunction bands to metaDict
for iPixFunc in range(len(pixelFunctionTypes)):
srcFiles = []
for j, jFileName in enumerate(sourceFileNames):
sourceFile = {'SourceFilename': jFileName}
if j == 0:
sourceFile['SourceBand'] = iPolarization['bandNum']
# if ASA_full_incAng,
# set 'ScaleRatio' into source file dict
sourceFile['ScaleRatio'] = np.sqrt(
1.0 / iPolarization['calibrationConst'])
else:
sourceFile['SourceBand'] = 1
srcFiles.append(sourceFile)
metaDict.append({
'src': srcFiles,
'dst': {'short_name': short_names[iPixFunc],
'wkv': wkt[iPixFunc],
'PixelFunctionType': (
pixelFunctionTypes[iPixFunc]),
'polarization': iPolarization['channel'],
'suffix': iPolarization['channel'],
'pass': sphPass[iPixFunc],
'dataType': 6}})
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add oribit and look information to metadata domain
# ASAR is always right-looking
self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT')
self.dataset.SetMetadataItem('ORBIT_DIRECTION',
gdalMetadata['SPH_PASS'].upper())
###################################################################
# Add sigma0_VV
###################################################################
polarizations = []
for pp in polarization:
polarizations.append(pp['channel'])
if 'VV' not in polarizations and 'HH' in polarizations:
srcFiles = []
for j, jFileName in enumerate(sourceFileNames):
sourceFile = {'SourceFilename': jFileName}
if j == 0:
sourceFile['SourceBand'] = iPolarization['bandNum']
# if ASA_full_incAng,
# set 'ScaleRatio' into source file dict
sourceFile['ScaleRatio'] = np.sqrt(
1.0 / iPolarization['calibrationConst'])
else:
sourceFile['SourceBand'] = 1
srcFiles.append(sourceFile)
dst = {'wkv': (
'surface_backwards_scattering_coefficient_of_radar_wave'),
'PixelFunctionType': 'Sigma0HHToSigma0VV',
'polarization': 'VV',
'suffix': 'VV'}
self._create_band(srcFiles, dst)
self.dataset.FlushCache()
# set time
self._set_envisat_time(gdalMetadata)
# When using TPS for reprojection, use only every 3rd GCP
# to improve performance (tradeoff vs accuracy)
self.dataset.SetMetadataItem('skip_gcps', '3')
# set SADCAT specific metadata
self.dataset.SetMetadataItem('start_date',
(parse(gdalMetadata['MPH_SENSING_START']).
isoformat()))
self.dataset.SetMetadataItem('stop_date',
(parse(gdalMetadata['MPH_SENSING_STOP']).
isoformat()))
self.dataset.SetMetadataItem('sensor', 'ASAR')
self.dataset.SetMetadataItem('satellite', 'Envisat')
self.dataset.SetMetadataItem('mapper', 'asar')
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_COUNT=10,
bandNames=['VNIR_Band1', 'VNIR_Band2', 'VNIR_Band3N'],
bandWaves=[560, 660, 820], **kwargs):
''' Create VRT
Parameters
-----------
GCP_COUNT : int
number of GCPs along each dimention
bandNames : list of string (band name)
bandWaves : list of integer (waves corresponding to band name)
Band name and waves
--------------------
'VNIR_Band3B' : 820, 'SWIR_Band4' : 1650, 'SWIR_Band5' : 2165,
'SWIR_Band6' : 2205, 'SWIR_Band7' : 2260, 'SWIR_Band8' : 2330,
'SWIR_Band9' : 2395, 'TIR_Band10' : 8300, 'TIR_Band11' : 8650,
'TIR_Band12' : 9100, 'TIR_Band13' : 10600, 'TIR_Band14' : 11300
'''
# check if it is ASTER L1A
try:
assert 'AST_L1A_' in fileName
shortName = gdalMetadata['INSTRUMENTSHORTNAME']
assert shortName == 'ASTER'
except:
raise WrongMapperError
subDatasets = gdalDataset.GetSubDatasets()
# find datasets for each band and generate metaDict
metaDict = []
bandDatasetMask = 'HDF4_EOS:EOS_SWATH:"%s":%s:ImageData'
for bandName, bandWave in zip(bandNames, bandWaves):
metaEntry = {'src': {'SourceFilename': (bandDatasetMask
% (fileName, bandName)),
'SourceBand': 1,
'DataType': 6,
},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': str(bandWave),
'suffix': str(bandWave),
}}
metaDict.append(metaEntry)
# create empty VRT dataset with geolocation only
gdalSubDataset = gdal.Open(metaDict[0]['src']['SourceFilename'])
VRT.__init__(self, gdalSubDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# find largest lon/lat subdatasets
latShape0 = 0
for subDataset in subDatasets:
if 'Latitude' in subDataset[1]:
ls = int(subDataset[1].strip().split('[')[1].split('x')[0])
if ls >= latShape0:
latShape0 = ls
latSubDS = subDataset[0]
if 'Longitude' in subDataset[1]:
ls = int(subDataset[1].strip().split('[')[1].split('x')[0])
if ls >= latShape0:
latShape0 = ls
lonSubDS = subDataset[0]
self.logger.debug(latSubDS)
self.logger.debug(lonSubDS)
# get lat/lon matrices
xDataset = gdal.Open(lonSubDS)
yDataset = gdal.Open(latSubDS)
longitude = xDataset.ReadAsArray()
latitude = yDataset.ReadAsArray()
step0 = longitude.shape[0] / GCP_COUNT
step1 = longitude.shape[1] / GCP_COUNT
# estimate pixel/line step
pixelStep = int(ceil(float(gdalSubDataset.RasterXSize) /
float(xDataset.RasterXSize)))
lineStep = int(ceil(float(gdalSubDataset.RasterYSize) /
float(xDataset.RasterYSize)))
self.logger.debug('steps: %d %d %d %d' % (step0, step1,
pixelStep, lineStep))
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep, i0 * lineStep)
self.logger.debug('%d %d %d %f %f' % (k, gcp.GCPPixel,
gcp.GCPLine,
gcp.GCPX, gcp.GCPY))
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
self._set_time(parse(gdalMetadata['FIRSTPACKETTIME']))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' OBPG L3 VRT '''
try:
assert 'Level-3 Standard Mapped Image' in gdalMetadata['Title']
except:
raise WrongMapperError
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
simFilesMask = os.path.join(iDir, iFileName)
simFiles = glob.glob(simFilesMask + iFileExt[0:6] + '*')
#print 'simFilesMask, simFiles', simFilesMask, simFiles
metaDict = []
for simFile in simFiles:
#print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
if simSupDataset is None:
# skip this similar file
#print 'No dataset: %s not a supported SMI file' % simFile
continue
# get subdatasets from the similar file
simSubDatasets = simSupDataset.GetSubDatasets()
if len(simSubDatasets) > 0:
for simSubDataset in simSubDatasets:
#print 'simSubDataset', simSubDataset
if 'l3m_data' in simSubDataset[1]:
# get SourceFilename from subdataset
tmpSourceFilename = simSubDataset[0]
break
else:
# get SourceFilename from dataset
tmpSourceFilename = simFile
# open subdataset with GDAL
#print 'tmpSourceFilename', tmpSourceFilename
tmpGdalDataset = gdal.Open(tmpSourceFilename)
try:
# get metadata, get 'Parameter'
tmpGdalMetadata = tmpGdalDataset.GetMetadata()
simParameter = tmpGdalMetadata['Parameter']
except:
print 'No parameter: %s not a supported SMI file' % simFile
continue
else:
# set params of the similar file
simSourceFilename = tmpSourceFilename
simGdalDataset = tmpGdalDataset
simGdalMetadata = tmpGdalMetadata
# get WKV from the similar file
#print 'simParameter', simParameter
for param in self.param2wkv:
#print 'param', param
if param in simParameter:
simWKV = self.param2wkv[param]
break
# generate entry to metaDict
metaEntry = {
'src': {
'SourceFilename': simSourceFilename,
'SourceBand': 1,
'ScaleRatio': float(simGdalMetadata['Slope']),
'ScaleOffset': float(simGdalMetadata['Intercept'])
},
'dst': {
'wkv': simWKV
}
}
# add wavelength and BandName
if ' at ' in simParameter and ' nm' in simParameter:
simWavelength = simParameter.split(' at ')[1].split(' nm')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add band with Rrsw
metaEntry2 = None
if simWKV == 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air':
metaEntry2 = {'src': [metaEntry['src']]}
metaEntry2['dst'] = {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix': simWavelength,
'wavelength': simWavelength,
'PixelFunctionType': 'NormReflectanceToRemSensReflectance',
}
# append entry to metaDict
metaDict.append(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
#get array with data and make 'mask'
a = simGdalDataset.ReadAsArray()
mask = np.zeros(a.shape, 'uint8') + 64
mask[a < -32000] = 1
self.bandVRTs = {'mask': VRT(array=mask)}
metaDict.append({
'src': {
'SourceFilename': self.bandVRTs['mask'].fileName,
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# create empty VRT dataset with geolocation only
# print 'simGdalMetadata', simGdalMetadata
latitudeStep = float(
simGdalMetadata.get('Latitude Step',
simGdalMetadata.get('Latitude_Step', 1)))
longitudeStep = float(
simGdalMetadata.get('Longitude Step',
simGdalMetadata.get('Longitude_Step', 1)))
numberOfColumns = int(
simGdalMetadata.get('Number of Columns',
simGdalMetadata.get('Number_of_Columns', 1)))
numberOfLines = int(
simGdalMetadata.get('Number of Lines',
simGdalMetadata.get('Number_of_Lines', 1)))
#longitudeStep = float(simGdalMetadata['Longitude Step'])
VRT.__init__(self,
srcGeoTransform=(-180.0, longitudeStep, 0.0, 90.0, 0.0,
-longitudeStep),
srcProjection=NSR().wkt,
srcRasterXSize=numberOfColumns,
srcRasterYSize=numberOfLines)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add valid time
startYear = int(
simGdalMetadata.get('Start Year',
simGdalMetadata.get('Start_Year', 1)))
startDay = int(
simGdalMetadata.get('Start Day',
simGdalMetadata.get('Start)Day', 1)))
self.dataset.SetMetadataItem(
'time_coverage_start', (datetime.datetime(startYear, 1, 1) +
datetime.timedelta(startDay)).isoformat())
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create NCEP VRT '''
if not gdalDataset:
raise WrongMapperError
geotransform = gdalDataset.GetGeoTransform()
if (geotransform != (-0.25, 0.5, 0.0, 90.25, 0.0, -0.5)
or gdalDataset.RasterCount != 2): # Not water proof
raise WrongMapperError
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'eastward_wind',
'height': '10 m'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 2
},
'dst': {
'wkv': 'northward_wind',
'height': '10 m'
}
}, {
'src': [{
'SourceFilename': fileName,
'SourceBand': 1,
'DataType': gdalDataset.GetRasterBand(1).DataType
}, {
'SourceFilename': fileName,
'SourceBand': 2,
'DataType': gdalDataset.GetRasterBand(2).DataType
}],
'dst': {
'wkv': 'wind_speed',
'PixelFunctionType': 'UVToMagnitude',
'name': 'windspeed',
'height': '2 m'
}
}, {
'src': [{
'SourceFilename': fileName,
'SourceBand': 1,
'DataType': gdalDataset.GetRasterBand(1).DataType
}, {
'SourceFilename': fileName,
'SourceBand': 2,
'DataType': gdalDataset.GetRasterBand(2).DataType
}],
'dst': {
'wkv': 'wind_from_direction',
'PixelFunctionType': 'UVToDirectionFrom',
'name': 'winddirection',
'height': '2 m'
}
}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time from the GRIB file to dataset
validTime = gdalDataset.GetRasterBand(1).\
GetMetadata()['GRIB_VALID_TIME']
self._set_time(
datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0])))
return
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
latlonGrid=None,
mask='',
**kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is GLOBCOLOUR L3B
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
#print 'idir:', iDir, iFile, iFileName[0:5], iFileExt[0:8]
if (iFileName[0:4] != 'L3b_' or iFileExt != '.nc'
or not os.path.exists(fileName) or
(gdalDataset is not None and (len(gdalDataset.GetSubDatasets()) > 0
or gdalDataset.RasterCount > 0))):
raise WrongMapperError
# define shape of GLOBCOLOUR grid
GLOBCOLOR_ROWS = 180 * 24
GLOBCOLOR_COLS = 360 * 24
# define lon/lat grids for projected var
if latlonGrid is None:
latlonGrid = np.mgrid[90:-90:4320j,
-180:180:8640j].astype('float32')
#latlonGrid = np.mgrid[80:50:900j, -10:30:1200j].astype('float16')
#latlonGrid = np.mgrid[47:39:300j, 25:45:500j].astype('float32')
# create empty VRT dataset with geolocation only
VRT.__init__(self, lon=latlonGrid[1], lat=latlonGrid[0])
# get list of similar (same date) files in the directory
simFilesMask = os.path.join(iDir, iFileName[0:30] + '*' + mask + '.nc')
simFiles = glob.glob(simFilesMask)
simFiles.sort()
metaDict = []
self.bandVRTs = {'mask': [], 'lonlat': []}
mask = None
for simFile in simFiles:
print 'sim: ', simFile
# copy simFile to a temporary file
tmpf = tempfile.mkstemp()
shutil.copyfile(simFile, tmpf[1])
f = Dataset(tmpf[1])
# get iBinned, index for converting from binned into GLOBCOLOR-grid
colBinned = f.variables['col'][:]
rowBinned = f.variables['row'][:]
iBinned = (colBinned.astype('uint32') +
(rowBinned.astype('uint32') - 1) * GLOBCOLOR_COLS)
colBinned = None
rowBinned = None
# get iRawPro, index for converting
# from GLOBCOLOR-grid to latlonGrid
yRawPro = np.rint(1 + (GLOBCOLOR_ROWS - 1) *
(latlonGrid[0] + 90) / 180.)
lon_step_Mat = 24. * np.cos(np.pi * latlonGrid[0] / 180.)
xRawPro = np.rint(1 + (latlonGrid[1] + 180) * lon_step_Mat)
iRawPro = xRawPro.astype('uint32') + (yRawPro.astype('uint32') -
1) * GLOBCOLOR_COLS
yRawPro = None
xRawPro = None
for varName in f.variables:
# find variable with _mean, eg CHL1_mean
if '_mean' in varName:
var = f.variables[varName]
break
# skip variable if no WKV is give in Globcolour
if varName not in self.varname2wkv:
continue
# get WKV
varWKV = self.varname2wkv[varName]
# read binned data
varBinned = var[:]
# convert to GLOBCOLOR grid
varRawPro = np.zeros([GLOBCOLOR_ROWS, GLOBCOLOR_COLS], 'float32')
varRawPro.flat[iBinned] = varBinned
# convert to latlonGrid
varPro = varRawPro.flat[iRawPro.flat[:]].reshape(iRawPro.shape)
#plt.imshow(varPro);plt.colorbar();plt.show()
# add mask band
if mask is None:
mask = np.zeros(varPro.shape, 'uint8')
mask[:] = 1
mask[varPro > 0] = 64
# add VRT with array with data from projected variable
self.bandVRTs['mask'].append(VRT(array=mask))
# add metadata to the dictionary
metaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['mask'][-1].fileName),
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# add VRT with array with data from projected variable
self.bandVRTs['lonlat'].append(VRT(array=varPro))
# add metadata to the dictionary
metaEntry = {
'src': {
'SourceFilename': self.bandVRTs['lonlat'][-1].fileName,
'SourceBand': 1
},
'dst': {
'wkv': varWKV,
'original_name': varName
}
}
# add wavelength for nLw
longName = 'Fully normalised water leaving radiance'
if longName in f.variables[varName].long_name:
simWavelength = varName.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add all metadata from NC-file
for attr in var.ncattrs():
metaEntry['dst'][attr] = var.getncattr(attr)
metaDict.append(metaEntry)
# add Rrsw band
metaEntry2 = self.make_rrsw_meta_entry(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
os.remove(tmpf[1])
instrument = f.title.strip().split(' ')[-2].split('/')[0]
mm = pti.get_gcmd_instrument(instrument)
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
platform = {
'MODIS': 'AQUA',
'MERIS': 'ENVISAT',
'SEAWIFS': 'QUICKBIRD',
'VIIRS': 'SUOMI-NPP'
}[instrument.upper()]
pp = pti.get_gcmd_platform(platform)
self.dataset.SetMetadataItem('platform', json.dumps(pp))
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# add time
startDate = datetime.datetime(int(iFileName[4:8]),
int(iFileName[8:10]),
int(iFileName[10:12]))
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start',
startDate.isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
startDate.isoformat())
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
logLevel=30,
**kwargs):
if fileName[0:len(keywordBase)] != keywordBase:
raise WrongMapperError(
__file__, "Not Nora10 data converted from felt to netCDF")
requestedTime = datetime.strptime(fileName[len(keywordBase) + 1:],
'%Y%m%d%H%M')
# For correct rounding
fileTime = requestedTime + timedelta(minutes=30)
fileTime = fileTime - timedelta(minutes=fileTime.minute)
nc_file = (baseFolder + 'windspeed_10m' +
fileTime.strftime('/%Y/%m/') + 'windspeed_' +
fileTime.strftime('%Y%m%d%H.nc'))
nc_file_winddir = (baseFolder + 'winddir_10m' +
fileTime.strftime('/%Y/%m/') + 'winddir_' +
fileTime.strftime('%Y%m%d%H.nc'))
# Would prefer to use geotransform, but ob_tran
# (General Oblique Transformation) is not supported by GDAL
# Keeping lines below for potential future use:
#proj4 = gdalDataset.GetMetadataItem('projection_rotated_ll#proj4')
#proj4 = '+proj=ob_tran +o_proj=longlat +lon_0=-40 +o_lat_p=22 +a=6367470 +e=0'
#rlatmin = -13.25; rlatmax = 26.65; deltarlat = 0.1
#rlonmin = 5.75; rlonmax = 30.45; deltarlon = 0.1
# Needed due to precence of time dimension in netCDF file
gdal.SetConfigOption('GDAL_NETCDF_BOTTOMUP', 'No')
# Read relevant arrays into memory
g = gdal.Open('NETCDF:"' + nc_file + '":' + 'windspeed_10m')
ws_10m = np.flipud(g.GetRasterBand(1).ReadAsArray())
g = gdal.Open(nc_file_winddir)
wd_10m = np.flipud(g.GetRasterBand(1).ReadAsArray())
g = gdal.Open('NETCDF:"' + nc_file + '":' + 'latitude')
lat = np.flipud(g.GetRasterBand(1).ReadAsArray())
g = gdal.Open('NETCDF:"' + nc_file + '":' + 'longitude')
lon = np.flipud(g.GetRasterBand(1).ReadAsArray())
u10 = ws_10m * np.cos(np.deg2rad(wd_10m))
v10 = ws_10m * np.sin(np.deg2rad(wd_10m))
VRT_u10 = VRT(array=u10, lat=lat, lon=lon)
VRT_v10 = VRT(array=v10, lat=lat, lon=lon)
# Store bandVRTs so that they are available after reprojection etc
self.bandVRTs = {'u_VRT': VRT_u10, 'v_VRT': VRT_v10}
metaDict = []
metaDict.append({
'src': {
'SourceFilename': VRT_u10.fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'eastward_wind',
'name': 'eastward_wind'
}
})
metaDict.append({
'src': {
'SourceFilename': VRT_v10.fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'northward_wind',
'name': 'northward_wind'
}
})
# Add pixel function with wind speed
metaDict.append({
'src': [{
'SourceFilename': self.bandVRTs['u_VRT'].fileName,
'SourceBand': 1,
'DataType': 6
}, {
'SourceFilename': self.bandVRTs['v_VRT'].fileName,
'SourceBand': 1,
'DataType': 6
}],
'dst': {
'wkv': 'wind_speed',
'name': 'windspeed',
'height': '10 m',
'PixelFunctionType': 'UVToMagnitude'
}
})
# Add pixel function with wind direction
metaDict.append({
'src': [{
'SourceFilename': self.bandVRTs['u_VRT'].fileName,
'SourceBand': 1,
'DataType': 6
}, {
'SourceFilename': self.bandVRTs['v_VRT'].fileName,
'SourceBand': 1,
'DataType': 6
}],
'dst': {
'wkv': 'wind_to_direction',
'name': 'winddir',
'height': '10 m',
'PixelFunctionType': 'UVToDirectionTo'
}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, lat=lat, lon=lon)
# add bands with metadata and corresponding values
# to the empty VRT
self._create_bands(metaDict)
# Add time
self._set_time(fileTime)
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
GCP_STEP=20,
MAX_LAT=90,
MIN_LAT=50,
resolution='low',
**kwargs):
''' Create VRT
Parameters
----------
GCP_COUNT : int
number of GCPs along each dimention
'''
ifile = os.path.split(fileName)[1]
if not ifile.startswith('GW1AM2_') or not ifile.endswith('.h5'):
raise WrongMapperError
try:
ProductName = gdalMetadata['ProductName']
PlatformShortName = gdalMetadata['PlatformShortName']
SensorShortName = gdalMetadata['SensorShortName']
except:
raise WrongMapperError
if (not ProductName == 'AMSR2-L1R'
or not PlatformShortName == 'GCOM-W1'
or not SensorShortName == 'AMSR2'):
raise WrongMapperError
if resolution == 'low':
subDatasetWidth = 243
else:
subDatasetWidth = 486
# get GCPs from lon/lat grids
latGrid = gdal.Open(
'HDF5:"%s"://Latitude_of_Observation_Point_for_89A' %
fileName).ReadAsArray()
lonGrid = gdal.Open(
'HDF5:"%s"://Longitude_of_Observation_Point_for_89A' %
fileName).ReadAsArray()
if subDatasetWidth == 243:
latGrid = latGrid[:, ::2]
lonGrid = lonGrid[:, ::2]
dx = .5
dy = .5
gcps = []
k = 0
maxY = 0
minY = latGrid.shape[0]
for i0 in range(0, latGrid.shape[0], GCP_STEP):
for i1 in range(0, latGrid.shape[1], GCP_STEP):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(lonGrid[i0, i1])
lat = float(latGrid[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= MIN_LAT
and lat <= MAX_LAT):
gcp = gdal.GCP(lon, lat, 0, i1 + dx, i0 + dy)
gcps.append(gcp)
k += 1
maxY = max(maxY, i0)
minY = min(minY, i0)
yOff = minY
ySize = maxY - minY
# remove Y-offset from gcps
for gcp in gcps:
gcp.GCPLine -= yOff
metaDict = []
subDatasets = gdalDataset.GetSubDatasets()
metadata = gdalDataset.GetMetadata()
for subDataset in subDatasets:
# select subdatasets fro that resolution (width)
if (subDatasetWidth == int(
subDataset[1].split(']')[0].split('x')[-1])
and 'Latitude' not in subDataset[0]
and 'Longitude' not in subDataset[0]):
name = subDataset[0].split('/')[-1]
# find scale
scale = 1
for meta in metadata:
if name + '_SCALE' in meta:
scale = float(metadata[meta])
# create meta entry
metaEntry = {
'src': {
'SourceFilename': subDataset[0],
'sourceBand': 1,
'ScaleRatio': scale,
'ScaleOffset': 0,
'yOff': yOff,
'ySize': ySize,
},
'dst': {
'name': name
}
}
metaDict.append(metaEntry)
# create VRT from one of the subdatasets
gdalSubDataset = gdal.Open(metaEntry['src']['SourceFilename'])
VRT.__init__(self,
srcRasterXSize=subDatasetWidth,
srcRasterYSize=ySize)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
self.dataset.SetMetadataItem(
'time_coverage_start',
parse_time(gdalMetadata['ObservationStartDateTime']).isoformat())
self.dataset.SetMetadataItem(
'time_coverage_end',
parse_time(gdalMetadata['ObservationEndDateTime']).isoformat())
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
self.reproject_GCPs(
'+proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=90 +lon_0=0 +no_defs'
)
self.tps = True
mm = pti.get_gcmd_instrument('AMSR2')
ee = pti.get_gcmd_platform('GCOM-W1')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata,
manifestonly=False, **kwargs):
if zipfile.is_zipfile(fileName):
zz = zipfile.PyZipFile(fileName)
# Assuming the file names are consistent, the polarization
# dependent data should be sorted equally such that we can use the
# same indices consistently for all the following lists
# THIS IS NOT THE CASE...
mdsFiles = ['/vsizip/%s/%s' % (fileName, fn)
for fn in zz.namelist() if 'measurement/s1a' in fn]
calFiles = ['/vsizip/%s/%s' % (fileName, fn)
for fn in zz.namelist()
if 'annotation/calibration/calibration-s1a' in fn]
noiseFiles = ['/vsizip/%s/%s' % (fileName, fn)
for fn in zz.namelist()
if 'annotation/calibration/noise-s1a' in fn]
annotationFiles = ['/vsizip/%s/%s' % (fileName, fn)
for fn in zz.namelist()
if 'annotation/s1a' in fn]
manifestFile = ['/vsizip/%s/%s' % (fileName, fn)
for fn in zz.namelist()
if 'manifest.safe' in fn]
zz.close()
else:
mdsFiles = glob.glob('%s/measurement/s1a*' % fileName)
calFiles = glob.glob('%s/annotation/calibration/calibration-s1a*'
% fileName)
noiseFiles = glob.glob('%s/annotation/calibration/noise-s1a*'
% fileName)
annotationFiles = glob.glob('%s/annotation/s1a*'
% fileName)
manifestFile = glob.glob('%s/manifest.safe' % fileName)
if (not mdsFiles or not calFiles or not noiseFiles or
not annotationFiles or not manifestFile):
raise WrongMapperError
mdsDict = {}
for ff in mdsFiles:
mdsDict[
os.path.splitext(os.path.basename(ff))[0].split('-')[3]] = ff
self.calXMLDict = {}
for ff in calFiles:
self.calXMLDict[
os.path.splitext(
os.path.basename(ff))[0].split('-')[4]] = self.read_xml(ff)
self.noiseXMLDict = {}
for ff in noiseFiles:
self.noiseXMLDict[
os.path.splitext(
os.path.basename(ff))[0].split('-')[4]] = self.read_xml(ff)
self.annotationXMLDict = {}
for ff in annotationFiles:
self.annotationXMLDict[
os.path.splitext(
os.path.basename(ff))[0].split('-')[3]] = self.read_xml(ff)
self.manifestXML = self.read_xml(manifestFile[0])
# very fast constructor without any bands
if manifestonly:
self.init_from_manifest_only(self.manifestXML,
self.annotationXMLDict[
self.annotationXMLDict.keys()[0]])
return
gdalDatasets = {}
for key in mdsDict.keys():
# Open data files
gdalDatasets[key] = gdal.Open(mdsDict[key])
if not gdalDatasets:
raise WrongMapperError('No Sentinel-1 datasets found')
# Check metadata to confirm it is Sentinel-1 L1
for key in gdalDatasets:
metadata = gdalDatasets[key].GetMetadata()
break
if not 'TIFFTAG_IMAGEDESCRIPTION' in metadata.keys():
raise WrongMapperError
if (not 'Sentinel-1' in metadata['TIFFTAG_IMAGEDESCRIPTION']
and not 'L1' in metadata['TIFFTAG_IMAGEDESCRIPTION']):
raise WrongMapperError
warnings.warn('Sentinel-1 level-1 mapper is not yet adapted to '
'complex data. In addition, the band names should be '
'updated for multi-swath data - '
'and there might be other issues.')
# create empty VRT dataset with geolocation only
for key in gdalDatasets:
VRT.__init__(self, gdalDatasets[key])
break
# Read annotation, noise and calibration xml-files
pol = {}
it = 0
for key in self.annotationXMLDict:
xml = Node.create(self.annotationXMLDict[key])
pol[key] = (xml.node('product').
node('adsHeader')['polarisation'].upper())
it += 1
if it == 1:
# Get incidence angle
pi = xml.node('generalAnnotation').node('productInformation')
self.dataset.SetMetadataItem('ORBIT_DIRECTION',
str(pi['pass']))
(X, Y, lon, lat, inc, ele, numberOfSamples,
numberOfLines) = self.read_geolocation_lut(
self.annotationXMLDict[key])
X = np.unique(X)
Y = np.unique(Y)
lon = np.array(lon).reshape(len(Y), len(X))
lat = np.array(lat).reshape(len(Y), len(X))
inc = np.array(inc).reshape(len(Y), len(X))
ele = np.array(ele).reshape(len(Y), len(X))
incVRT = VRT(array=inc, lat=lat, lon=lon)
eleVRT = VRT(array=ele, lat=lat, lon=lon)
incVRT = incVRT.get_resized_vrt(self.dataset.RasterXSize,
self.dataset.RasterYSize,
eResampleAlg=2)
eleVRT = eleVRT.get_resized_vrt(self.dataset.RasterXSize,
self.dataset.RasterYSize,
eResampleAlg=2)
self.bandVRTs['incVRT'] = incVRT
self.bandVRTs['eleVRT'] = eleVRT
for key in self.calXMLDict:
calibration_LUT_VRTs, longitude, latitude = (
self.get_LUT_VRTs(self.calXMLDict[key],
'calibrationVectorList',
['sigmaNought', 'betaNought',
'gamma', 'dn']
))
self.bandVRTs['LUT_sigmaNought_VRT_'+pol[key]] = (
calibration_LUT_VRTs['sigmaNought'].
get_resized_vrt(self.dataset.RasterXSize,
self.dataset.RasterYSize,
eResampleAlg=1))
self.bandVRTs['LUT_betaNought_VRT_'+pol[key]] = (
calibration_LUT_VRTs['betaNought'].
get_resized_vrt(self.dataset.RasterXSize,
self.dataset.RasterYSize,
eResampleAlg=1))
self.bandVRTs['LUT_gamma_VRT'] = calibration_LUT_VRTs['gamma']
self.bandVRTs['LUT_dn_VRT'] = calibration_LUT_VRTs['dn']
for key in self.noiseXMLDict:
noise_LUT_VRT = self.get_LUT_VRTs(self.noiseXMLDict[key],
'noiseVectorList',
['noiseLut'])[0]
self.bandVRTs['LUT_noise_VRT_'+pol[key]] = (
noise_LUT_VRT['noiseLut'].get_resized_vrt(
self.dataset.RasterXSize,
self.dataset.RasterYSize,
eResampleAlg=1))
metaDict = []
bandNumberDict = {}
bnmax = 0
for key in gdalDatasets.keys():
dsPath, dsName = os.path.split(mdsDict[key])
name = 'DN_%s' % pol[key]
# A dictionary of band numbers is needed for the pixel function
# bands further down. This is not the best solution. It would be
# better to have a function in VRT that returns the number given a
# band name. This function exists in Nansat but could perhaps be
# moved to VRT? The existing nansat function could just call the
# VRT one...
bandNumberDict[name] = bnmax + 1
bnmax = bandNumberDict[name]
band = gdalDatasets[key].GetRasterBand(1)
dtype = band.DataType
metaDict.append({
'src': {
'SourceFilename': mdsDict[key],
'SourceBand': 1,
'DataType': dtype,
},
'dst': {
'name': name,
#'SourceTransferType': gdal.GetDataTypeName(dtype),
#'dataType': 6,
},
})
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
'''
Calibration should be performed as
s0 = DN^2/sigmaNought^2,
where sigmaNought is from e.g.
annotation/calibration/calibration-s1a-iw-grd-hh-20140811t151231-20140811t151301-001894-001cc7-001.xml,
and DN is the Digital Numbers in the tiff files.
Also the noise should be subtracted.
See
https://sentinel.esa.int/web/sentinel/sentinel-1-sar-wiki/-/wiki/Sentinel%20One/Application+of+Radiometric+Calibration+LUT
'''
# Get look direction
sat_heading = initial_bearing(longitude[:-1, :],
latitude[:-1, :],
longitude[1:, :],
latitude[1:, :])
look_direction = scipy.ndimage.interpolation.zoom(
np.mod(sat_heading + 90, 360),
(np.shape(longitude)[0] / (np.shape(longitude)[0]-1.), 1))
# Decompose, to avoid interpolation errors around 0 <-> 360
look_direction_u = np.sin(np.deg2rad(look_direction))
look_direction_v = np.cos(np.deg2rad(look_direction))
look_u_VRT = VRT(array=look_direction_u,
lat=latitude, lon=longitude)
look_v_VRT = VRT(array=look_direction_v,
lat=latitude, lon=longitude)
lookVRT = VRT(lat=latitude, lon=longitude)
lookVRT._create_band([{'SourceFilename': look_u_VRT.fileName,
'SourceBand': 1},
{'SourceFilename': look_v_VRT.fileName,
'SourceBand': 1}],
{'PixelFunctionType': 'UVToDirectionTo'}
)
# Blow up to full size
lookVRT = lookVRT.get_resized_vrt(self.dataset.RasterXSize,
self.dataset.RasterYSize,
eResampleAlg=1)
# Store VRTs so that they are accessible later
self.bandVRTs['look_u_VRT'] = look_u_VRT
self.bandVRTs['look_v_VRT'] = look_v_VRT
self.bandVRTs['lookVRT'] = lookVRT
metaDict = []
# Add bands to full size VRT
for key in pol:
name = 'LUT_sigmaNought_%s' % pol[key]
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
metaDict.append(
{'src': {'SourceFilename':
(self.bandVRTs['LUT_sigmaNought_VRT_' +
pol[key]].fileName),
'SourceBand': 1
},
'dst': {'name': name
}
})
name = 'LUT_noise_%s' % pol[key]
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
metaDict.append({
'src': {
'SourceFilename': self.bandVRTs['LUT_noise_VRT_' +
pol[key]].fileName,
'SourceBand': 1
},
'dst': {
'name': name
}
})
name = 'look_direction'
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
metaDict.append({
'src': {
'SourceFilename': self.bandVRTs['lookVRT'].fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'sensor_azimuth_angle',
'name': name
}
})
for key in gdalDatasets.keys():
dsPath, dsName = os.path.split(mdsDict[key])
name = 'sigma0_%s' % pol[key]
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
metaDict.append(
{'src': [{'SourceFilename': self.fileName,
'SourceBand': bandNumberDict['DN_%s' % pol[key]],
},
{'SourceFilename':
(self.bandVRTs['LUT_sigmaNought_VRT_%s'
% pol[key]].fileName),
'SourceBand': 1
}
],
'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType': 'Sentinel1Calibration',
'polarization': pol[key],
'suffix': pol[key],
},
})
name = 'beta0_%s' % pol[key]
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
metaDict.append(
{'src': [{'SourceFilename': self.fileName,
'SourceBand': bandNumberDict['DN_%s' % pol[key]]
},
{'SourceFilename':
(self.bandVRTs['LUT_betaNought_VRT_%s'
% pol[key]].fileName),
'SourceBand': 1
}
],
'dst': {'wkv': 'surface_backwards_brightness_coefficient_of_radar_wave',
'PixelFunctionType': 'Sentinel1Calibration',
'polarization': pol[key],
'suffix': pol[key],
},
})
self._create_bands(metaDict)
# Add incidence angle as band
name = 'incidence_angle'
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
src = {'SourceFilename': self.bandVRTs['incVRT'].fileName,
'SourceBand': 1}
dst = {'wkv': 'angle_of_incidence',
'name': name}
self._create_band(src, dst)
self.dataset.FlushCache()
# Add elevation angle as band
name = 'elevation_angle'
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
src = {'SourceFilename': self.bandVRTs['eleVRT'].fileName,
'SourceBand': 1}
dst = {'wkv': 'angle_of_elevation',
'name': name}
self._create_band(src, dst)
self.dataset.FlushCache()
# Add sigma0_VV
pp = [pol[key] for key in pol]
if 'VV' not in pp and 'HH' in pp:
name = 'sigma0_VV'
bandNumberDict[name] = bnmax+1
bnmax = bandNumberDict[name]
src = [{'SourceFilename': self.fileName,
'SourceBand': bandNumberDict['DN_HH'],
},
{'SourceFilename': (self.bandVRTs['LUT_noise_VRT_HH'].
fileName),
'SourceBand': 1
},
{'SourceFilename': (self.bandVRTs['LUT_sigmaNought_VRT_HH'].
fileName),
'SourceBand': 1,
},
{'SourceFilename': self.bandVRTs['incVRT'].fileName,
'SourceBand': 1}
]
dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType': 'Sentinel1Sigma0HHToSigma0VV',
'polarization': 'VV',
'suffix': 'VV'}
self._create_band(src, dst)
self.dataset.FlushCache()
# set time as acquisition start time
n = Node.create(self.manifestXML)
meta = n.node('metadataSection')
for nn in meta.children:
if nn.getAttribute('ID') == u'acquisitionPeriod':
# set valid time
self.dataset.SetMetadataItem(
'time_coverage_start',
parse((nn.node('metadataWrap').
node('xmlData').
node('safe:acquisitionPeriod')['safe:startTime'])
).isoformat())
self.dataset.SetMetadataItem(
'time_coverage_end',
parse((nn.node('metadataWrap').
node('xmlData').
node('safe:acquisitionPeriod')['safe:stopTime'])
).isoformat())
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('sar')
ee = pti.get_gcmd_platform('sentinel-1a')
# TODO: Validate that the found instrument and platform are indeed what we
# want....
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
latlonGrid=None,
mask='',
**kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is GLOBCOLOUR L3B
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
#print 'idir:', iDir, iFile, iFileName[0:5], iFileExt[0:8]
if iFileName[0:4] != 'L3b_' or iFileExt != '.nc':
raise WrongMapperError
# define shape of GLOBCOLOUR grid
GLOBCOLOR_ROWS = 180 * 24
GLOBCOLOR_COLS = 360 * 24
# define lon/lat grids for projected var
if latlonGrid is None:
latlonGrid = np.mgrid[90:-90:4320j,
-180:180:8640j].astype('float32')
#latlonGrid = np.mgrid[80:50:900j, -10:30:1200j].astype('float16')
#latlonGrid = np.mgrid[47:39:300j, 25:45:500j].astype('float32')
# create empty VRT dataset with geolocation only
VRT.__init__(self, lon=latlonGrid[1], lat=latlonGrid[0])
# get list of similar (same date) files in the directory
simFilesMask = os.path.join(iDir, iFileName[0:30] + '*' + mask + '.nc')
simFiles = glob.glob(simFilesMask)
simFiles.sort()
metaDict = []
self.bandVRTs = {'mask': [], 'lonlat': []}
mask = None
for simFile in simFiles:
print 'sim: ', simFile
f = netcdf_file(simFile)
# get iBinned, index for converting from binned into GLOBCOLOR-grid
colBinned = f.variables['col'][:]
rowBinned = f.variables['row'][:]
iBinned = (colBinned.astype('uint32') +
(rowBinned.astype('uint32') - 1) * GLOBCOLOR_COLS)
colBinned = None
rowBinned = None
# get iRawPro, index for converting
# from GLOBCOLOR-grid to latlonGrid
yRawPro = np.rint(1 + (GLOBCOLOR_ROWS - 1) *
(latlonGrid[0] + 90) / 180)
lon_step_Mat = 1 / np.cos(np.pi * latlonGrid[0] / 180.) / 24.
xRawPro = np.rint(1 + (latlonGrid[1] + 180) / lon_step_Mat)
iRawPro = xRawPro + (yRawPro - 1) * GLOBCOLOR_COLS
iRawPro[iRawPro < 0] = 0
iRawPro = np.rint(iRawPro).astype('uint32')
yRawPro = None
xRawPro = None
for varName in f.variables:
# find variable with _mean, eg CHL1_mean
if '_mean' in varName:
var = f.variables[varName]
break
# skip variable if no WKV is give in Globcolour
if varName not in self.varname2wkv:
continue
# get WKV
varWKV = self.varname2wkv[varName]
# read binned data
varBinned = var[:]
# convert to GLOBCOLOR grid
varRawPro = np.zeros([GLOBCOLOR_ROWS, GLOBCOLOR_COLS], 'float32')
varRawPro.flat[iBinned] = varBinned
# convert to latlonGrid
varPro = varRawPro.flat[iRawPro.flat[:]].reshape(iRawPro.shape)
#plt.imshow(varPro);plt.colorbar();plt.show()
# add mask band
if mask is None:
mask = np.zeros(varPro.shape, 'uint8')
mask[:] = 1
mask[varPro > 0] = 64
# add VRT with array with data from projected variable
self.bandVRTs['mask'].append(VRT(array=mask))
# add metadata to the dictionary
metaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['mask'][-1].fileName),
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# add VRT with array with data from projected variable
self.bandVRTs['lonlat'].append(VRT(array=varPro))
# add metadata to the dictionary
metaEntry = {
'src': {
'SourceFilename': self.bandVRTs['lonlat'][-1].fileName,
'SourceBand': 1
},
'dst': {
'wkv': varWKV,
'original_name': varName
}
}
# add wavelength for nLw
longName = 'Fully normalised water leaving radiance'
if longName in f.variables[varName].long_name:
simWavelength = varName.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add all metadata from NC-file
for attr in var._attributes:
metaEntry['dst'][attr] = var._attributes[attr]
metaDict.append(metaEntry)
# add Rrsw band
metaEntry2 = self.make_rrsw_meta_entry(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# add time
startDate = datetime.datetime(int(iFileName[4:8]),
int(iFileName[8:10]),
int(iFileName[10:12]))
self._set_time(startDate)
def __init__(self, fileName, gdalDataset, gdalMetadata, logLevel=30,
**kwargs):
if fileName[0:len(keywordBase)] != keywordBase:
raise WrongMapperError(__file__,
"Not Nora10 data converted from felt to netCDF")
requestedTime = datetime.strptime(fileName[len(keywordBase)+1:],
'%Y%m%d%H%M')
# For correct rounding
fileTime = requestedTime + timedelta(minutes=30)
fileTime = fileTime - timedelta(minutes=fileTime.minute)
nc_file = (baseFolder + 'windspeed_10m' +
fileTime.strftime('/%Y/%m/') + 'windspeed_' +
fileTime.strftime('%Y%m%d%H.nc'))
nc_file_winddir = (baseFolder + 'winddir_10m' +
fileTime.strftime('/%Y/%m/') +
'winddir_' + fileTime.strftime('%Y%m%d%H.nc'))
# Would prefer to use geotransform, but ob_tran
# (General Oblique Transformation) is not supported by GDAL
# Keeping lines below for potential future use:
#proj4 = gdalDataset.GetMetadataItem('projection_rotated_ll#proj4')
#proj4 = '+proj=ob_tran +o_proj=longlat +lon_0=-40 +o_lat_p=22 +a=6367470 +e=0'
#rlatmin = -13.25; rlatmax = 26.65; deltarlat = 0.1
#rlonmin = 5.75; rlonmax = 30.45; deltarlon = 0.1
# Needed due to precence of time dimension in netCDF file
gdal.SetConfigOption('GDAL_NETCDF_BOTTOMUP', 'No')
# Read relevant arrays into memory
g = gdal.Open('NETCDF:"' + nc_file + '":' + 'windspeed_10m')
ws_10m = np.flipud(g.GetRasterBand(1).ReadAsArray())
g = gdal.Open(nc_file_winddir)
wd_10m = np.flipud(g.GetRasterBand(1).ReadAsArray())
g = gdal.Open('NETCDF:"' + nc_file + '":' + 'latitude')
lat = np.flipud(g.GetRasterBand(1).ReadAsArray())
g = gdal.Open('NETCDF:"' + nc_file + '":' + 'longitude')
lon = np.flipud(g.GetRasterBand(1).ReadAsArray())
u10 = ws_10m*np.cos(np.deg2rad(wd_10m))
v10 = ws_10m*np.sin(np.deg2rad(wd_10m))
VRT_u10 = VRT(array=u10, lat=lat, lon=lon)
VRT_v10 = VRT(array=v10, lat=lat, lon=lon)
# Store bandVRTs so that they are available after reprojection etc
self.bandVRTs = {'u_VRT': VRT_u10,
'v_VRT': VRT_v10}
metaDict = []
metaDict.append({'src': {'SourceFilename': VRT_u10.fileName,
'SourceBand': 1},
'dst': {'wkv': 'eastward_wind',
'name': 'eastward_wind'}})
metaDict.append({'src': {'SourceFilename': VRT_v10.fileName,
'SourceBand': 1},
'dst': {'wkv': 'northward_wind',
'name': 'northward_wind'}})
# Add pixel function with wind speed
metaDict.append({
'src': [{'SourceFilename': self.bandVRTs['u_VRT'].fileName,
'SourceBand': 1,
'DataType': 6},
{'SourceFilename': self.bandVRTs['v_VRT'].fileName,
'SourceBand': 1,
'DataType': 6}],
'dst': {'wkv': 'wind_speed',
'name': 'windspeed',
'height': '10 m',
'PixelFunctionType': 'UVToMagnitude'}})
# Add pixel function with wind direction
metaDict.append({
'src': [{'SourceFilename': self.bandVRTs['u_VRT'].fileName,
'SourceBand': 1,
'DataType': 6},
{'SourceFilename': self.bandVRTs['v_VRT'].fileName,
'SourceBand': 1,
'DataType': 6}],
'dst': {'wkv': 'wind_to_direction',
'name': 'winddir',
'height': '10 m',
'PixelFunctionType': 'UVToDirectionTo'}})
# create empty VRT dataset with geolocation only
VRT.__init__(self, lat=lat, lon=lon)
# add bands with metadata and corresponding values
# to the empty VRT
self._create_bands(metaDict)
# Add time
self._set_time(fileTime)
def __init__(self, fileName, gdalDataset, gdalMetadata, emrange='VNIR', **kwargs):
''' Create MODIS_L1 VRT '''
# check mapper
try:
INSTRUMENTSHORTNAME = gdalMetadata['INSTRUMENTSHORTNAME']
except:
raise WrongMapperError
if INSTRUMENTSHORTNAME != 'ASTER':
raise WrongMapperError
try:
SHORTNAME = gdalMetadata['SHORTNAME']
except:
raise WrongMapperError
if SHORTNAME != 'ASTL1B':
raise WrongMapperError
# set up metadict for data with various resolution
subDSString = 'HDF4_EOS:EOS_SWATH:"%s":%s:%s'
metaDictVNIR = [
{'src': {'SourceFilename': subDSString % (fileName, 'VNIR_Swath', 'ImageData1' )}, 'dst': {'wavelength': '560'}},
{'src': {'SourceFilename': subDSString % (fileName, 'VNIR_Swath', 'ImageData2' )}, 'dst': {'wavelength': '660'}},
{'src': {'SourceFilename': subDSString % (fileName, 'VNIR_Swath', 'ImageData3N')}, 'dst': {'wavelength': '820'}},
{'src': {'SourceFilename': subDSString % (fileName, 'VNIR_Swath', 'ImageData3B')}, 'dst': {'wavelength': '820'}},
]
metaDictSWIR = [
{'src': {'SourceFilename': subDSString % (fileName, 'SWIR_Swath', 'ImageData4')}, 'dst': {'wavelength': '1650'}},
{'src': {'SourceFilename': subDSString % (fileName, 'SWIR_Swath', 'ImageData5')}, 'dst': {'wavelength': '2165'}},
{'src': {'SourceFilename': subDSString % (fileName, 'SWIR_Swath', 'ImageData6')}, 'dst': {'wavelength': '2205'}},
{'src': {'SourceFilename': subDSString % (fileName, 'SWIR_Swath', 'ImageData7')}, 'dst': {'wavelength': '2260'}},
{'src': {'SourceFilename': subDSString % (fileName, 'SWIR_Swath', 'ImageData8')}, 'dst': {'wavelength': '2330'}},
{'src': {'SourceFilename': subDSString % (fileName, 'SWIR_Swath', 'ImageData9')}, 'dst': {'wavelength': '2395'}},
]
metaDictTIR = [
{'src': {'SourceFilename': subDSString % (fileName, 'TIR_Swath', 'ImageData10')}, 'dst': {'wavelength': '8300'}},
{'src': {'SourceFilename': subDSString % (fileName, 'TIR_Swath', 'ImageData11')}, 'dst': {'wavelength': '8650'}},
{'src': {'SourceFilename': subDSString % (fileName, 'TIR_Swath', 'ImageData12')}, 'dst': {'wavelength': '9100'}},
{'src': {'SourceFilename': subDSString % (fileName, 'TIR_Swath', 'ImageData13')}, 'dst': {'wavelength': '10600'}},
{'src': {'SourceFilename': subDSString % (fileName, 'TIR_Swath', 'ImageData14')}, 'dst': {'wavelength': '11300'}},
]
# select appropriate metaDict based on <emrange> parameter
metaDict = {'VNIR': metaDictVNIR,
'SWIR': metaDictSWIR,
'TIR': metaDictTIR,
}[emrange]
# get 1st EOS subdataset and parse to VRT.__init__()
# for retrieving geo-metadata
try:
gdalSubDataset0 = gdal.Open(metaDict[0]['src']['SourceFilename'])
except (AttributeError, IndexError):
raise WrongMapperError
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalSubDataset0)
# add source band, wkv and suffix
for metaEntry in metaDict:
metaEntry['src']['SourceBand'] = 1
metaEntry['dst']['wkv'] = 'toa_outgoing_spectral_radiance'
metaEntry['dst']['suffix'] = metaEntry['dst']['wavelength']
if 'ImageData3N' in metaEntry['src']['SourceFilename']:
metaEntry['dst']['suffix'] += 'N'
if 'ImageData3B' in metaEntry['src']['SourceFilename']:
metaEntry['dst']['suffix'] += 'B'
# add scale and offset
for metaEntry in metaDict:
bandNo = metaEntry['src']['SourceFilename'].strip().split(':')[-1].replace('ImageData', '')
metaEntry['src']['ScaleRatio'] = float(gdalMetadata['INCL' + bandNo])
metaEntry['src']['ScaleOffset'] = float(gdalMetadata['OFFSET' + bandNo])
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
datetimeString = self.find_metadata(gdalMetadata, "SETTINGTIMEOFPOINTING")
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start',
parse(datetimeString+'+00').isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
parse(datetimeString+'+00').isoformat())
mm = pti.get_gcmd_instrument('ASTER')
ee = pti.get_gcmd_platform('TERRA')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
self.remove_geolocationArray()
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_COUNT=10, **kwargs):
''' Create VRT
Parameters
----------
GCP_COUNT : int
number of GCPs along each dimention
'''
# extension must be .nc
if os.path.splitext(fileName)[1] != '.nc':
raise WrongMapperError
# file must contain navigation_data/longitude
try:
ds = gdal.Open('HDF5:"%s"://navigation_data/longitude' % fileName)
except RuntimeError:
raise WrongMapperError
else:
dsMetadata = ds.GetMetadata()
# title value must be known
if dsMetadata.get('title', '') not in self.titles:
raise WrongMapperError
# get geophysical data variables
subDatasets = gdal.Open(fileName).GetSubDatasets()
metaDict = []
for subDataset in subDatasets:
groupName = subDataset[0].split('/')[-2]
if groupName not in ['geophysical_data', 'navigation_data']:
continue
varName = subDataset[0].split('/')[-1]
subds = gdal.Open(subDataset[0])
b = subds.GetRasterBand(1)
bMetadata = b.GetMetadata()
# set SRC/DST parameters
metaEntry = {'src': {'SourceFilename': subDataset[0],
'sourceBand': 1,
'DataType': b.DataType},
'dst': {'name': varName}}
# replace datatype for l2_flags
if varName == 'l2_flags':
metaEntry['src']['DataType'] = 4
metaEntry['src']['SourceType'] = 'SimpleSource'
# set scale if exist
metaKey = '%s_%s_scale_factor' % (groupName, varName)
if metaKey in bMetadata:
metaEntry['src']['ScaleRatio'] = bMetadata[metaKey]
# set offset if exist
metaKey = '%s_%s_add_offset' % (groupName, varName)
if metaKey in bMetadata:
metaEntry['src']['ScaleOffset'] = bMetadata[metaKey]
# set standard_name if exists
metaKey = '%s_%s_standard_name' % (groupName, varName)
if metaKey in bMetadata:
metaEntry['dst']['wkv'] = bMetadata[metaKey]
# set other metadata
for metaKey in bMetadata:
newMetaKey = metaKey.replace('%s_%s_' % (groupName, varName), '')
if newMetaKey not in ['scale_factor', 'add_offset', 'DIMENSION_LIST', '_FillValue']:
metaEntry['dst'][newMetaKey] = bMetadata[metaKey]
metaDict.append(metaEntry)
# make GCPs
# get lat/lon grids
longitude = gdal.Open('HDF5:"%s"://navigation_data/longitude' % fileName).ReadAsArray()
latitude = gdal.Open('HDF5:"%s"://navigation_data/latitude' % fileName).ReadAsArray()
rasterYSize, rasterXSize = longitude.shape
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT))
gcps = []
k = 0
center_lon = 0
center_lat = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1+0.5, i0+0.5)
gcps.append(gcp)
center_lon += lon
center_lat += lat
k += 1
time_coverage_start = dsMetadata['time_coverage_start']
time_coverage_end = dsMetadata['time_coverage_end']
# create VRT
VRT.__init__(self, srcProjection=NSR().wkt,
srcGCPs=gcps,
srcGCPProjection=NSR().wkt,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
# add bands
self._create_bands(metaDict)
# reproject GCPs
center_lon /= k
center_lat /= k
srs = '+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=%f +lat_0=%f +no_defs' % (center_lon, center_lat)
self.reproject_GCPs(srs)
### BAD, BAd, bad ...
self.dataset.SetProjection(self.dataset.GetGCPProjection())
# use TPS for reprojection
self.tps = True
# add NansenCloud metadata
self.dataset.SetMetadataItem('time_coverage_start',
str(time_coverage_start))
self.dataset.SetMetadataItem('time_coverage_end',
str(time_coverage_end))
self.dataset.SetMetadataItem('source_type', 'Satellite')
self.dataset.SetMetadataItem('mapper', 'obpg_l2_nc')
mm = pti.get_gcmd_instrument(dsMetadata.get('instrument'))
ee = pti.get_gcmd_platform(dsMetadata.get('platform'))
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT from OpenDAP dataset'''
# quit if file is not online
if fileName[:7] != 'http://':
raise WrongMapperError
# open file through OpenDAP using netCDF4 library
f = Dataset(fileName)
# assume CF-compatibility:
# compulsory grid_mapping_name
# find grid_mapping_name
# and get all parameters
# generate proj4 and WKT strings
srcProjection = ''
for varName in f.variables:
var = f.variables[varName]
attrs = var.ncattrs()
if 'grid_mapping_name' in attrs:
proj4str = self.get_proj4_from_ncvar(var)
srcProjection = NSR(proj4str).wkt
break
print 'WKT:', srcProjection
xDim = None
yDim = None
validVars = []
validDims = []
# if grid_mapping_name is found: find all bands that have grid_mapping
# else: find lon/lat variables that have 1D
if srcProjection != '':
# find bands with grid_mapping
for varName in f.variables:
var = f.variables[varName]
attrs = var.ncattrs()
if 'grid_mapping' in attrs:
validVars.append(str(varName))
validDims += [str(dim) for dim in var.dimensions]
# assume NORMAP compatibility:
# dimensions should be
# x, or xc, or lon, or longitude
# and
# y, or yc, or lat, or latitude
# and/or
# time
# and/or
# depth
# and/or
# etc
# assign x, y dimension names
for dim in validDims:
if 'x' in dim or 'lon' in dim or 'east' in dim:
xDim = dim
if 'y' in dim or 'lat' in dim or 'north' in dim:
yDim = dim
else:
# if input file is not CF-compliant but has 1D lon/lat dimensions
# find 1D lon, lat (longitude, latitude) dims
for varName in f.variables:
var = f.variables[varName]
if 'lon' in varName and var.ndim == 1:
xDim = varName
if 'lat' in varName and var.ndim == 1:
yDim = varName
# find all datasets with lon/lat dimensions
for varName in f.variables:
var = f.variables[varName]
if xDim in var.dimensions and yDim in var.dimensions:
validVars.append(str(varName))
validDims += [str(dim) for dim in var.dimensions]
if len(validVars) > 1:
srcProjection = NSR().wkt
else:
# cancel usage of this mapper a input file seem unappropriate
raise
# get X/Y size
var0 = f.variables[validVars[0]]
xDimI = var0.dimensions.index(xDim)
srcRasterXSize = var0.shape[xDimI]
yDimI = var0.dimensions.index(yDim)
srcRasterYSize = var0.shape[yDimI]
print 'x/ySize', srcRasterXSize, srcRasterYSize
# get GDAL GeoTransform
xdata = f.variables[xDim][:]
ydata = f.variables[yDim][:]
x0 = xdata[0]
dx = xdata[1] - xdata[0]
y0 = ydata[0]
dy = ydata[1] - ydata[0]
srcGeoTransform = (x0, dx, 0, y0, 0, dy)
print srcGeoTransform
# make list of metadata dictionary entries
# for each var make [k][i][j][x][y] depending on order of dims
metaDict = []
for varName in validVars:
var = f.variables[varName]
dims = var.dimensions
# find nor X neither Y dimensions (e.g. time, depth, etc)
nonxyDims = []
nonxyDimValues = []
for dim in dims:
if dim != xDim and dim != yDim:
nonxyDims.append(dim)
dimValues = f.variables[dim][:]
dimValuesStr = [str(val) for val in dimValues]
nonxyDimValues.append(dimValues)
# calculate total dimensionality in addition to X/Y
# make vector of nonXY dimensions
nonXYShape = []
for nonxyDim in nonxyDims:
dimVar = f.variables[nonxyDim]
nonXYShape.append(dimVar.shape[0])
if len(nonXYShape) == 0:
nonXYShape = [1]
totNonXYDims = np.cumprod(nonXYShape)[-1]
urls = []
# generate bands for each additional dimension
for nonxyi in range(totNonXYDims):
dstVarName = str(varName)
url = fileName + '?%s.%s' % (varName, varName)
#assert varName != 'v'
# vector of nonX/Y indeces
iVec = np.unravel_index(nonxyi, nonXYShape)
# dim metadata keeps name of dimension and index
# in this dimension (value)
dimMetadata = {}
# add either [x], or [y], or respective index in each dimension
for dim in dims:
if dim == xDim:
url = url + '[x]'
elif dim == yDim:
url += '[y]'
else:
# get index of that dimension
dimN = nonxyDims.index(dim)
dimV = iVec[dimN]
url += '[%d]' % dimV
dimMetadata[dim] = nonxyDimValues[dimN][dimV]
dstVarName += '%03d' % dimV
# get band metadata
attrs = var.ncattrs()
metaEntry = {
'src': {
'SourceFilename': url,
'sourceBand': 1
},
'dst': {
'name': dstVarName
}
}
# put band metadata
for attr in attrs:
metaEntry['dst'][str(attr)] = str(var.getncattr(attr))
# add wkv
if 'standard_name' in attrs:
metaEntry['dst']['wkv'] = metaEntry['dst']['standard_name']
# add dim metadata (location within each dimension)
for dimKey in dimMetadata:
metaEntry['dst'][str(dimKey)] = dimMetadata[dimKey]
metaDict.append(metaEntry)
# read global metadata
srcMetadata = {}
for attr in f.ncattrs():
srcMetadata[str(attr)] = str(attr)
# create VRT with bands
VRT.__init__(self,
srcGeoTransform=srcGeoTransform,
srcProjection=srcProjection,
srcRasterXSize=srcRasterXSize,
srcRasterYSize=srcRasterYSize,
srcMetadata=srcMetadata)
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_STEP=20, MAX_LAT=90, MIN_LAT=50, resolution='low',
**kwargs):
''' Create VRT
Parameters
----------
GCP_COUNT : int
number of GCPs along each dimention
'''
ifile = os.path.split(fileName)[1]
if not ifile.startswith('GW1AM2_') or not ifile.endswith('.h5'):
raise WrongMapperError
try:
ProductName = gdalMetadata['ProductName']
PlatformShortName = gdalMetadata['PlatformShortName']
SensorShortName = gdalMetadata['SensorShortName']
except:
raise WrongMapperError
if (not ProductName == 'AMSR2-L1R' or
not PlatformShortName == 'GCOM-W1' or
not SensorShortName == 'AMSR2'):
raise WrongMapperError
if resolution == 'low':
subDatasetWidth = 243
else:
subDatasetWidth = 486
# get GCPs from lon/lat grids
latGrid = gdal.Open('HDF5:"%s"://Latitude_of_Observation_Point_for_89A' % fileName).ReadAsArray()
lonGrid = gdal.Open('HDF5:"%s"://Longitude_of_Observation_Point_for_89A' % fileName).ReadAsArray()
if subDatasetWidth == 243:
latGrid = latGrid[:, ::2]
lonGrid = lonGrid[:, ::2]
dx = .5
dy = .5
gcps = []
k = 0
maxY = 0
minY = latGrid.shape[0]
for i0 in range(0, latGrid.shape[0], GCP_STEP):
for i1 in range(0, latGrid.shape[1], GCP_STEP):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(lonGrid[i0, i1])
lat = float(latGrid[i0, i1])
if (lon >= -180 and
lon <= 180 and
lat >= MIN_LAT and
lat <= MAX_LAT):
gcp = gdal.GCP(lon, lat, 0, i1 + dx, i0 + dy)
gcps.append(gcp)
k += 1
maxY = max(maxY, i0)
minY = min(minY, i0)
yOff = minY
ySize = maxY - minY
# remove Y-offset from gcps
for gcp in gcps:
gcp.GCPLine -= yOff
metaDict = []
subDatasets = gdalDataset.GetSubDatasets()
metadata = gdalDataset.GetMetadata()
for subDataset in subDatasets:
# select subdatasets fro that resolution (width)
if (subDatasetWidth == int(subDataset[1].split(']')[0].split('x')[-1]) and
'Latitude' not in subDataset[0] and 'Longitude' not in subDataset[0]):
name = subDataset[0].split('/')[-1]
# find scale
scale = 1
for meta in metadata:
if name + '_SCALE' in meta:
scale = float(metadata[meta])
# create meta entry
metaEntry = {'src': {'SourceFilename': subDataset[0],
'sourceBand': 1,
'ScaleRatio': scale,
'ScaleOffset': 0,
'yOff': yOff,
'ySize': ySize,},
'dst': {'name': name}
}
metaDict.append(metaEntry)
# create VRT from one of the subdatasets
gdalSubDataset = gdal.Open(metaEntry['src']['SourceFilename'])
VRT.__init__(self, srcRasterXSize=subDatasetWidth, srcRasterYSize=ySize)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
self.dataset.SetMetadataItem('time_coverage_start',
parse_time(gdalMetadata['ObservationStartDateTime']).isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
parse_time(gdalMetadata['ObservationEndDateTime']).isoformat())
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
self.reproject_GCPs('+proj=stere +datum=WGS84 +ellps=WGS84 +lat_0=90 +lon_0=0 +no_defs')
self.tps = True
mm = pti.get_gcmd_instrument('AMSR2')
ee = pti.get_gcmd_platform('GCOM-W1')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create NCEP VRT '''
if not gdalDataset:
raise WrongMapperError
geotransform = gdalDataset.GetGeoTransform()
if (geotransform != (-0.25, 0.5, 0.0, 90.25, 0.0, -0.5)
or gdalDataset.RasterCount != 2): # Not water proof
raise WrongMapperError
metaDict = [{
'src': {
'SourceFilename': fileName,
'SourceBand': 1
},
'dst': {
'wkv': 'eastward_wind',
'height': '10 m'
}
}, {
'src': {
'SourceFilename': fileName,
'SourceBand': 2
},
'dst': {
'wkv': 'northward_wind',
'height': '10 m'
}
}, {
'src': [{
'SourceFilename': fileName,
'SourceBand': 1,
'DataType': gdalDataset.GetRasterBand(1).DataType
}, {
'SourceFilename': fileName,
'SourceBand': 2,
'DataType': gdalDataset.GetRasterBand(2).DataType
}],
'dst': {
'wkv': 'wind_speed',
'PixelFunctionType': 'UVToMagnitude',
'name': 'windspeed',
'height': '2 m'
}
}, {
'src': [{
'SourceFilename': fileName,
'SourceBand': 1,
'DataType': gdalDataset.GetRasterBand(1).DataType
}, {
'SourceFilename': fileName,
'SourceBand': 2,
'DataType': gdalDataset.GetRasterBand(2).DataType
}],
'dst': {
'wkv': 'wind_from_direction',
'PixelFunctionType': 'UVToDirectionFrom',
'name': 'winddirection',
'height': '2 m'
}
}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time from the GRIB file to dataset
validTime = gdalDataset.GetRasterBand(
1).GetMetadata()['GRIB_VALID_TIME']
self.dataset.SetMetadataItem(
'time_coverage_start', (datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0])).isoformat()))
self.dataset.SetMetadataItem(
'time_coverage_end', (datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0])).isoformat()))
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('computer')
ee = pti.get_gcmd_platform('ncep-gfs')
# TODO: Validate that the found instrument and platform are indeed what we
# want....
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
product_type='RVL',
GCP_COUNT=10,
**kwargs):
'''
Parameters
----------
product_type: string
Sentinel-1 level-2 ocean product type/component, i.e. ocean swell
spectra (OSW), ocean wind field (OWI), or radial surface velocity
(RVL) (RVL is the default)
GCP_COUNT : int
number of GCPs along each dimention
'''
fPathName, fExt = os.path.splitext(fileName)
# List of Sentinel-1 level-2 components
unwanted_product_components = ['osw', 'owi', 'rvl']
# Remove requested 'product_type' from list of unwanted
unwanted_product_components.pop(
unwanted_product_components.index(product_type.lower()))
# Check if it is Sentinel-1 (or ASAR) level-2 (in S1 data format)
if not gdalMetadata or not 'NC_GLOBAL' in gdalMetadata.keys():
raise WrongMapperError
else:
title = gdalMetadata['NC_GLOBAL#TITLE']
# Raise error if it is not Sentinel-1 format
if not 'Sentinel-1' or 'ASA' in title:
raise WrongMapperError
metadata = {}
for key, val in gdalMetadata.iteritems():
new_key = key.split('#')[-1]
metadata[new_key] = val
subDatasets = gdalDataset.GetSubDatasets()
fileNames = [f[0] for f in subDatasets]
rm_bands = []
# Find all data that is not relevant for the selected product type
# and get bands of longitude, latitude and zero doppler time
for i, f in enumerate(fileNames):
if f.split(':')[-1][:3] in unwanted_product_components:
rm_bands.append(i)
if 'Lon' in f.split(':')[-1]:
lon_ds = gdal.Open(f)
rm_bands.append(i)
if 'Lat' in f.split(':')[-1]:
lat_ds = gdal.Open(f)
rm_bands.append(i)
if 'ZeroDopplerTime' in f.split(':')[-1]:
zdt_ds = gdal.Open(f)
rm_bands.append(i)
# Remove bands in rm_bands from the list of bands to add to the Nansat
# object
fileNames = [f for i, f in enumerate(fileNames) if not i in rm_bands]
# (
# 'Lon' in f.split(':')[-1] or
# 'Lat' in f.split(':')[-1] or
# 'ZeroDopplerTime' in f.split(':')[-1] )]
# create empty VRT dataset
VRT.__init__(self, gdal.Open(subDatasets[0][0]), srcMetadata=metadata)
# The zero Doppler time grid is 3-dimensional - the last dimension is a
# char array with the time as year, month, day, etc.
# Will not bother with it yet...
#for iBand in range(zdt_ds.RasterCount):
# subBand = zdt_ds.GetRasterBand(iBand+1)
XSize = lon_ds.RasterXSize
YSize = lon_ds.RasterYSize
# get projection from the lon and lat datasets
longitude = lon_ds.ReadAsArray()
latitude = lat_ds.ReadAsArray()
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT))
self.logger.debug('gcpCount: >%s<, %d %d %f %d %d', title,
latitude.shape[0], latitude.shape[1], GCP_COUNT,
step0, step1)
# estimate pixel/line step of the geolocation arrays
pixelStep = 1
lineStep = 1
self.logger.debug('pixel/lineStep %f %f' % (pixelStep, lineStep))
# generate list of GCPs
dx = .5
dy = .5
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep + dx,
i0 * lineStep + dy)
self.logger.debug('%d %d %d %f %f', k, gcp.GCPPixel,
gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
# define band specific parameters
metaDict = []
geoFileDict = {}
xDatasetSource = ''
yDatasetSource = ''
for i, fileName in enumerate(fileNames):
band = gdal.Open(fileName)
# check that the band size is the same size as the latitude and
# longitude grids
if (band.RasterXSize != XSize or band.RasterYSize != YSize):
raise IndexError(('Size of sub-dataset is different from size '
'of longitude and latitude grids'))
bandMetadata = band.GetMetadata()
# generate src metadata
src = {'SourceFilename': fileName, 'SourceBand': 1}
# Generate dst metadata
short_name = fileName.split(':')[-1]
dst = {
'name': short_name,
'short_name': short_name,
'long_name': bandMetadata[short_name + '#long_name'],
'units': bandMetadata[short_name + '#units'],
#'wkv': ,
}
# append band with src and dst dictionaries
metaDict.append({'src': src, 'dst': dst})
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
metaDict = []
for i in range(self.dataset.RasterCount):
if 'Nrcs' in self.dataset.GetRasterBand(i +
1).GetMetadata()['name']:
metaDict.append({
'src': {
'SourceFilename': (self.dataset.GetRasterBand(
i + 1).GetMetadata()['SourceFilename']),
'SourceBand':
1
},
'dst': {
'short_name':
'sigma0',
'wkv':
'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType':
'dB2pow',
'polarization':
(self.dataset.GetMetadata()['POLARISATION']),
'suffix':
self.dataset.GetMetadata()['POLARISATION'],
'dataType':
6,
}
})
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self.dataset.SetMetadataItem(
'time_coverage_start',
parse(self.dataset.GetMetadata()
['SOURCE_ACQUISITION_UTC_TIME']).isoformat())
def __init__(self, inputFileName, gdalDataset, gdalMetadata, **kwargs):
''' Create Radarsat2 VRT '''
fPathName, fExt = os.path.splitext(inputFileName)
if zipfile.is_zipfile(inputFileName):
# Open zip file using VSI
fPath, fName = os.path.split(fPathName)
fileName = '/vsizip/%s/%s' % (inputFileName, fName)
if not 'RS' in fName[0:2]:
raise WrongMapperError('Provided data is not Radarsat-2')
gdalDataset = gdal.Open(fileName)
gdalMetadata = gdalDataset.GetMetadata()
else:
fileName = inputFileName
#if it is not RADARSAT-2, return
if (not gdalMetadata or
not 'SATELLITE_IDENTIFIER' in gdalMetadata.keys()):
raise WrongMapperError
elif gdalMetadata['SATELLITE_IDENTIFIER'] != 'RADARSAT-2':
raise WrongMapperError
import ipdb; ipdb.set_trace()
if zipfile.is_zipfile(inputFileName):
# Open product.xml to get additional metadata
zz = zipfile.ZipFile(inputFileName)
productXmlName = os.path.join(os.path.basename(inputFileName).split('.')[0],'product.xml')
productXml = zz.open(productXmlName).read()
else:
# product.xml to get additionali metadata
productXmlName = os.path.join(fileName,'product.xml')
if not os.path.isfile(productXmlName):
raise WrongMapperError
productXml = open(productXmlName).read()
# Get additional metadata from product.xml
rs2_0 = Node.create(productXml)
rs2_1 = rs2_0.node('sourceAttributes')
rs2_2 = rs2_1.node('radarParameters')
if rs2_2['antennaPointing'].lower() == 'right':
antennaPointing = 90
else:
antennaPointing = -90
rs2_3 = rs2_1.node('orbitAndAttitude').node('orbitInformation')
passDirection = rs2_3['passDirection']
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
#define dictionary of metadata and band specific parameters
pol = []
metaDict = []
# Get the subdataset with calibrated sigma0 only
for dataset in gdalDataset.GetSubDatasets():
if dataset[1] == 'Sigma Nought calibrated':
s0dataset = gdal.Open(dataset[0])
s0datasetName = dataset[0][:]
band = s0dataset.GetRasterBand(1)
s0datasetPol = band.GetMetadata()['POLARIMETRIC_INTERP']
for i in range(1, s0dataset.RasterCount+1):
iBand = s0dataset.GetRasterBand(i)
polString = iBand.GetMetadata()['POLARIMETRIC_INTERP']
suffix = polString
# The nansat data will be complex
# if the SAR data is of type 10
dtype = iBand.DataType
if dtype == 10:
# add intensity band
metaDict.append(
{'src': {'SourceFilename':
('RADARSAT_2_CALIB:SIGMA0:'
+ fileName + '/product.xml'),
'SourceBand': i,
'DataType': dtype},
'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType': 'intensity',
'SourceTransferType': gdal.GetDataTypeName(dtype),
'suffix': suffix,
'polarization': polString,
'dataType': 6}})
# modify suffix for adding the compled band below
suffix = polString+'_complex'
pol.append(polString)
metaDict.append(
{'src': {'SourceFilename': ('RADARSAT_2_CALIB:SIGMA0:'
+ fileName
+ '/product.xml'),
'SourceBand': i,
'DataType': dtype},
'dst': {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave',
'suffix': suffix,
'polarization': polString}})
if dataset[1] == 'Beta Nought calibrated':
b0dataset = gdal.Open(dataset[0])
b0datasetName = dataset[0][:]
for j in range(1, b0dataset.RasterCount+1):
jBand = b0dataset.GetRasterBand(j)
polString = jBand.GetMetadata()['POLARIMETRIC_INTERP']
if polString == s0datasetPol:
b0datasetBand = j
###############################
# Add SAR look direction
###############################
d = Domain(ds=gdalDataset)
lon, lat = d.get_geolocation_grids(100)
'''
(GDAL?) Radarsat-2 data is stored with maximum latitude at first
element of each column and minimum longitude at first element of each
row (e.g. np.shape(lat)=(59,55) -> latitude maxima are at lat[0,:],
and longitude minima are at lon[:,0])
In addition, there is an interpolation error for direct estimate along
azimuth. We therefore estimate the heading along range and add 90
degrees to get the "satellite" heading.
'''
if str(passDirection).upper() == 'DESCENDING':
sat_heading = initial_bearing(lon[:, :-1], lat[:, :-1],
lon[:, 1:], lat[:, 1:]) + 90
elif str(passDirection).upper() == 'ASCENDING':
sat_heading = initial_bearing(lon[:, 1:], lat[:, 1:],
lon[:, :-1], lat[:, :-1]) + 90
else:
print 'Can not decode pass direction: ' + str(passDirection)
# Calculate SAR look direction
look_direction = sat_heading + antennaPointing
# Interpolate to regain lost row
look_direction = np.mod(look_direction, 360)
look_direction = scipy.ndimage.interpolation.zoom(
look_direction, (1, 11./10.))
# Decompose, to avoid interpolation errors around 0 <-> 360
look_direction_u = np.sin(np.deg2rad(look_direction))
look_direction_v = np.cos(np.deg2rad(look_direction))
look_u_VRT = VRT(array=look_direction_u, lat=lat, lon=lon)
look_v_VRT = VRT(array=look_direction_v, lat=lat, lon=lon)
# Note: If incidence angle and look direction are stored in
# same VRT, access time is about twice as large
lookVRT = VRT(lat=lat, lon=lon)
lookVRT._create_band(
[{'SourceFilename': look_u_VRT.fileName, 'SourceBand': 1},
{'SourceFilename': look_v_VRT.fileName, 'SourceBand': 1}],
{'PixelFunctionType': 'UVToDirectionTo'})
# Blow up to full size
lookVRT = lookVRT.get_resized_vrt(gdalDataset.RasterXSize,
gdalDataset.RasterYSize)
# Store VRTs so that they are accessible later
self.bandVRTs['look_u_VRT'] = look_u_VRT
self.bandVRTs['look_v_VRT'] = look_v_VRT
self.bandVRTs['lookVRT'] = lookVRT
# Add band to full sized VRT
lookFileName = self.bandVRTs['lookVRT'].fileName
metaDict.append({'src': {'SourceFilename': lookFileName,
'SourceBand': 1},
'dst': {'wkv': 'sensor_azimuth_angle',
'name': 'look_direction'}})
###############################
# Create bands
###############################
self._create_bands(metaDict)
###################################################
# Add derived band (incidence angle) calculated
# using pixel function "BetaSigmaToIncidence":
###################################################
src = [{'SourceFilename': b0datasetName,
'SourceBand': b0datasetBand,
'DataType': dtype},
{'SourceFilename': s0datasetName,
'SourceBand': 1,
'DataType': dtype}]
dst = {'wkv': 'angle_of_incidence',
'PixelFunctionType': 'BetaSigmaToIncidence',
'SourceTransferType': gdal.GetDataTypeName(dtype),
'_FillValue': -10000, # NB: this is also hard-coded in
# pixelfunctions.c
'dataType': 6,
'name': 'incidence_angle'}
self._create_band(src, dst)
self.dataset.FlushCache()
###################################################################
# Add sigma0_VV - pixel function of sigma0_HH and beta0_HH
# incidence angle is calculated within pixel function
# It is assummed that HH is the first band in sigma0 and
# beta0 sub datasets
###################################################################
if 'VV' not in pol and 'HH' in pol:
s0datasetNameHH = pol.index('HH')+1
src = [{'SourceFilename': s0datasetName,
'SourceBand': s0datasetNameHH,
'DataType': 6},
{'SourceFilename': b0datasetName,
'SourceBand': b0datasetBand,
'DataType': 6}]
dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType': 'Sigma0HHBetaToSigma0VV',
'polarization': 'VV',
'suffix': 'VV'}
self._create_band(src, dst)
self.dataset.FlushCache()
############################################
# Add SAR metadata
############################################
if antennaPointing == 90:
self.dataset.SetMetadataItem('ANTENNA_POINTING', 'RIGHT')
if antennaPointing == -90:
self.dataset.SetMetadataItem('ANTENNA_POINTING', 'LEFT')
self.dataset.SetMetadataItem('ORBIT_DIRECTION',
str(passDirection).upper())
# set valid time
self.dataset.SetMetadataItem('time_coverage_start',
(parse(gdalMetadata['FIRST_LINE_TIME']).
isoformat()))
self.dataset.SetMetadataItem('time_coverage_end',
(parse(gdalMetadata['LAST_LINE_TIME']).
isoformat()))
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('sar')
ee = pti.get_gcmd_platform('radarsat-2')
# TODO: Validate that the found instrument and platform are indeed what we
# want....
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, latlonGrid=None,
mask='', **kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is GLOBCOLOUR L3B
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
#print 'idir:', iDir, iFile, iFileName[0:5], iFileExt[0:8]
if iFileName[0:4] != 'L3b_' or iFileExt != '.nc':
raise WrongMapperError
# define shape of GLOBCOLOUR grid
GLOBCOLOR_ROWS = 180 * 24
GLOBCOLOR_COLS = 360 * 24
# define lon/lat grids for projected var
if latlonGrid is None:
latlonGrid = np.mgrid[90:-90:4320j,
-180:180:8640j].astype('float32')
#latlonGrid = np.mgrid[80:50:900j, -10:30:1200j].astype('float16')
#latlonGrid = np.mgrid[47:39:300j, 25:45:500j].astype('float32')
# create empty VRT dataset with geolocation only
VRT.__init__(self, lon=latlonGrid[1], lat=latlonGrid[0])
# get list of similar (same date) files in the directory
simFilesMask = os.path.join(iDir, iFileName[0:30] + '*' + mask + '.nc')
simFiles = glob.glob(simFilesMask)
simFiles.sort()
metaDict = []
self.bandVRTs = {'mask': [], 'lonlat': []}
mask = None
for simFile in simFiles:
print 'sim: ', simFile
f = netcdf_file(simFile)
# get iBinned, index for converting from binned into GLOBCOLOR-grid
colBinned = f.variables['col'][:]
rowBinned = f.variables['row'][:]
iBinned = (colBinned.astype('uint32') +
(rowBinned.astype('uint32') - 1) * GLOBCOLOR_COLS)
colBinned = None
rowBinned = None
# get iRawPro, index for converting
# from GLOBCOLOR-grid to latlonGrid
yRawPro = np.rint(1 + (GLOBCOLOR_ROWS - 1) *
(latlonGrid[0] + 90) / 180)
lon_step_Mat = 1 / np.cos(np.pi * latlonGrid[0] / 180.) / 24.
xRawPro = np.rint(1 + (latlonGrid[1] + 180) / lon_step_Mat)
iRawPro = xRawPro + (yRawPro - 1) * GLOBCOLOR_COLS
iRawPro[iRawPro < 0] = 0
iRawPro = np.rint(iRawPro).astype('uint32')
yRawPro = None
xRawPro = None
for varName in f.variables:
# find variable with _mean, eg CHL1_mean
if '_mean' in varName:
var = f.variables[varName]
break
# skip variable if no WKV is give in Globcolour
if varName not in self.varname2wkv:
continue
# get WKV
varWKV = self.varname2wkv[varName]
# read binned data
varBinned = var[:]
# convert to GLOBCOLOR grid
varRawPro = np.zeros([GLOBCOLOR_ROWS, GLOBCOLOR_COLS], 'float32')
varRawPro.flat[iBinned] = varBinned
# convert to latlonGrid
varPro = varRawPro.flat[iRawPro.flat[:]].reshape(iRawPro.shape)
#plt.imshow(varPro);plt.colorbar();plt.show()
# add mask band
if mask is None:
mask = np.zeros(varPro.shape, 'uint8')
mask[:] = 1
mask[varPro > 0] = 64
# add VRT with array with data from projected variable
self.bandVRTs['mask'].append(VRT(array=mask))
# add metadata to the dictionary
metaDict.append({
'src': {'SourceFilename': (self.bandVRTs['mask'][-1].
fileName),
'SourceBand': 1},
'dst': {'name': 'mask'}})
# add VRT with array with data from projected variable
self.bandVRTs['lonlat'].append(VRT(array=varPro))
# add metadata to the dictionary
metaEntry = {
'src': {'SourceFilename': self.bandVRTs['lonlat'][-1].fileName,
'SourceBand': 1},
'dst': {'wkv': varWKV, 'original_name': varName}}
# add wavelength for nLw
longName = 'Fully normalised water leaving radiance'
if longName in f.variables[varName].long_name:
simWavelength = varName.split('L')[1].split('_mean')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add all metadata from NC-file
for attr in var._attributes:
metaEntry['dst'][attr] = var._attributes[attr]
metaDict.append(metaEntry)
# add Rrsw band
metaEntry2 = self.make_rrsw_meta_entry(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# add time
startDate = datetime.datetime(int(iFileName[4:8]),
int(iFileName[8:10]),
int(iFileName[10:12]))
self._set_time(startDate)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create NCEP VRT '''
if not gdalDataset:
raise WrongMapperError
geotransform = gdalDataset.GetGeoTransform()
if (geotransform == (-0.25, 0.5, 0.0, 90.25, 0.0, -0.5) or
geotransform == (-0.5, 1.0, 0.0, 90.5, 0.0, -1.0)):
if gdalDataset.RasterCount == 4:
srcBandId = {'temperature': 2,
'u-component': 3,
'v-component': 4}
elif gdalDataset.RasterCount == 9:
srcBandId = {'temperature': 6,
'u-component': 8,
'v-component': 9}
else:
raise WrongMapperError
else:
raise WrongMapperError # Not water proof
metaDict = [{'src': {'SourceFilename': fileName,
'SourceBand': srcBandId['u-component']},
'dst': {'wkv': 'eastward_wind',
'height': '10 m'}},
{'src': {'SourceFilename': fileName,
'SourceBand': srcBandId['v-component']},
'dst': {'wkv': 'northward_wind',
'height': '10 m'}},
{'src': [{'SourceFilename': fileName,
'SourceBand': srcBandId['u-component'],
'DataType': (gdalDataset.GetRasterBand(srcBandId['u-component']).DataType)
},
{'SourceFilename': fileName,
'SourceBand': srcBandId['v-component'],
'DataType': gdalDataset.GetRasterBand(srcBandId['v-component']).DataType
}],
'dst': {'wkv': 'wind_speed',
'PixelFunctionType': 'UVToMagnitude',
'name': 'windspeed',
'height': '2 m'
}},
{'src': [{'SourceFilename': fileName,
'SourceBand': srcBandId['u-component'],
'DataType': gdalDataset.GetRasterBand(srcBandId['u-component']).DataType
},
{'SourceFilename': fileName,
'SourceBand': srcBandId['v-component'],
'DataType': gdalDataset.GetRasterBand(srcBandId['v-component']).DataType
}],
'dst': {'wkv': 'wind_from_direction',
'PixelFunctionType': 'UVToDirectionFrom',
'name': 'winddirection',
'height': '2 m'
}},
{'src': {'SourceFilename': fileName,
'SourceBand': srcBandId['temperature']},
'dst': {'wkv': 'air_temperature',
'name': 'air_t',
'height': '2 m'}
}]
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time from the GRIB file to dataset
band = gdalDataset.GetRasterBand(srcBandId['u-component'])
validTime = band.GetMetadata()['GRIB_VALID_TIME']
self.dataset.SetMetadataItem('time_coverage_start',
(datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0])).isoformat()))
self.dataset.SetMetadataItem('time_coverage_end',
((datetime.datetime.utcfromtimestamp(
int(validTime.strip().split(' ')[0]))
+ datetime.timedelta(hours=3)).isoformat()))
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('computer')
ee = pti.get_gcmd_platform('ncep-gfs')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
latlonGrid=None,
mask='',
**kwargs):
''' Create VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is ASCAT
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
try:
assert iFileName[0:6] == 'ascat_' and iFileExt == '.nc'
except:
raise WrongMapperError
# Create geolocation
subDataset = gdal.Open('NETCDF:"' + fileName + '":lat')
self.GeolocVRT = VRT(srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize)
GeolocMetaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lon'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': -360
},
'dst': {}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lat'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': 0
},
'dst': {}
}]
self.GeolocVRT._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.GeolocVRT,
yVRT=self.GeolocVRT,
# x = lon, y = lat
xBand=1,
yBand=2,
lineOffset=0,
pixelOffset=0,
lineStep=1,
pixelStep=1)
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
gdalDataset=subDataset,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Scale and NODATA should ideally be taken directly from raw file
metaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_speed'),
'ScaleRatio': 0.01,
'NODATA': -32767
},
'dst': {
'name': 'windspeed',
'wkv': 'wind_speed'
}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_dir'),
'ScaleRatio': 0.1,
'NODATA': -32767
},
'dst': {
'name': 'winddirection',
'wkv': 'wind_from_direction'
}
}]
self._create_bands(metaDict)
# This should not be necessary
# - should be provided by GeolocationArray!
self.dataset.SetProjection(GeolocObject.d['SRS'])
# Add time
startTime = datetime.datetime(int(iFileName[6:10]),
int(iFileName[10:12]),
int(iFileName[12:14]),
int(iFileName[15:17]),
int(iFileName[17:19]),
int(iFileName[19:21]))
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start',
startTime.isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
startTime.isoformat())
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('ascat')
ee = pti.get_gcmd_platform('metop-a')
# TODO: Validate that the found instrument and platform are indeed what
# we want....
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
latlonGrid=None,
mask='',
**kwargs):
''' Create VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is ASCAT
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
try:
assert iFileName[0:6] == 'ascat_' and iFileExt == '.nc'
except:
raise WrongMapperError
# Create geolocation
subDataset = gdal.Open('NETCDF:"' + fileName + '":lat')
self.GeolocVRT = VRT(srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize)
GeolocMetaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lon'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': -360
},
'dst': {}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lat'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': 0
},
'dst': {}
}]
self.GeolocVRT._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.GeolocVRT,
yVRT=self.GeolocVRT,
# x = lon, y = lat
xBand=1,
yBand=2,
lineOffset=0,
pixelOffset=0,
lineStep=1,
pixelStep=1)
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
gdalDataset=subDataset,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Scale and NODATA should ideally be taken directly from raw file
metaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_speed'),
'ScaleRatio': 0.01,
'NODATA': -32767
},
'dst': {
'name': 'wind_speed',
'wkv': 'wind_speed'
}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_dir'),
'ScaleRatio': 0.1,
'NODATA': -32767
},
'dst': {
'name': 'wind_direction',
'wkv': 'wind_direction'
}
}]
self._create_bands(metaDict)
# This should not be necessary
# - should be provided by GeolocationArray!
self.dataset.SetProjection(GeolocObject.d['SRS'])
# Add time
startTime = datetime.datetime(int(iFileName[6:10]),
int(iFileName[10:12]),
int(iFileName[12:14]),
int(iFileName[15:17]),
int(iFileName[17:19]),
int(iFileName[19:21]))
self._set_time(startTime)
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
GCP_COUNT=10,
**kwargs):
''' Create VRT
Parameters
----------
GCP_COUNT : int
number of GCPs along each dimention
'''
titles = [
'HMODISA Level-2 Data', 'MODISA Level-2 Data',
'MERIS Level-2 Data', 'GOCI Level-2 Data', 'VIIRSN Level-2 Data'
]
# should raise error in case of not obpg_l2 file
try:
title = gdalMetadata["Title"]
except:
raise WrongMapperError
if title not in titles:
raise WrongMapperError
# get subdataset and parse to VRT.__init__()
# for retrieving geo-metadata
# but NOT from longitude or latitude because it can be smaller!
subDatasets = gdalDataset.GetSubDatasets()
for subDataset in subDatasets:
if ('longitude' not in subDataset[1]
and 'latitude' not in subDataset[1]):
gdalSubDataset = gdal.Open(subDataset[0])
break
if title is 'GOCI Level-2 Data':
# set GOCI projection parameters
rasterXSize = 5567
rasterYSize = 5685
proj4 = '+proj=ortho +lat_0=36 +lon_0=130 units=m +ellps=WGS84 +datum=WGS84 +no_defs'
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4)
projection = srs.ExportToWkt()
geoTransform = (-1391500.0, 500.0, 0.0, 1349500.0, 0.0, -500.0)
# create empty VRT dataset with georeference only
VRT.__init__(self,
srcGeoTransform=geoTransform,
srcProjection=projection,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
else:
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalSubDataset)
# parts of dictionary for all Reflectances
#dictRrs = {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '412'} }
# dictionary for all possible bands
allBandsDict = {
'Rrs': {
'src': {},
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air'
}
},
'Kd': {
'src': {},
'dst': {
'wkv':
'volume_attenuation_coefficient_of_downwelling_radiative_flux_in_sea_water'
}
},
'chlor_a': {
'src': {},
'dst': {
'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'case': 'I'
}
},
'cdom_index': {
'src': {},
'dst': {
'wkv':
'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'case': 'II'
}
},
'sst': {
'src': {},
'dst': {
'wkv': 'sea_surface_temperature'
}
},
'sst4': {
'src': {},
'dst': {
'wkv': 'sea_surface_temperature'
}
},
'l2_flags': {
'src': {
'SourceType': 'SimpleSource',
'DataType': 4
},
'dst': {
'wkv': 'quality_flags',
'dataType': 4
}
},
'qual_sst': {
'src': {
'SourceType': 'SimpleSource',
'DataType': 4
},
'dst': {
'wkv': 'quality_flags',
'name': 'qual_sst',
'dataType': 4
}
},
'qual_sst4': {
'src': {
'SourceType': 'SimpleSource',
'DataType': 4
},
'dst': {
'wkv': 'quality_flags',
'name': 'qual_sst',
'dataType': 4
}
},
'latitude': {
'src': {},
'dst': {
'wkv': 'latitude'
}
},
'longitude': {
'src': {},
'dst': {
'wkv': 'longitude'
}
}
}
# loop through available bands and generate metaDict (non fixed)
metaDict = []
bandNo = 0
for subDataset in subDatasets:
# get sub dataset name
subDatasetName = subDataset[1].split(' ')[1]
self.logger.debug('Subdataset: %s' % subDataset[1])
self.logger.debug('Subdataset name: "%s"' % subDatasetName)
# get wavelength if applicable, get dataset name without wavelength
try:
wavelength = int(subDatasetName.split('_')[-1])
except:
wavelength = None
subBandName = subDatasetName
else:
subBandName = subDatasetName.split('_')[0]
self.logger.debug('subBandName, wavelength: %s %s' %
(subBandName, str(wavelength)))
if subBandName in allBandsDict:
# get name, slope, intercept
self.logger.debug('name: %s' % subBandName)
tmpSubDataset = gdal.Open(subDataset[0])
tmpSubMetadata = tmpSubDataset.GetMetadata()
slope = tmpSubMetadata.get('slope', '1')
intercept = tmpSubMetadata.get('intercept', '0')
self.logger.debug('slope, intercept: %s %s ' %
(slope, intercept))
# create meta entry
metaEntry = {
'src': {
'SourceFilename': subDataset[0],
'sourceBand': 1,
'ScaleRatio': slope,
'ScaleOffset': intercept
},
'dst': {}
}
# add more to src
for srcKey in allBandsDict[subBandName]['src']:
metaEntry['src'][srcKey] = (
allBandsDict[subBandName]['src'][srcKey])
# add dst from allBandsDict
for dstKey in allBandsDict[subBandName]['dst']:
metaEntry['dst'][dstKey] = (
allBandsDict[subBandName]['dst'][dstKey])
# add wavelength, band name to dst
if wavelength is not None:
metaEntry['dst']['suffix'] = str(wavelength)
metaEntry['dst']['wavelength'] = str(wavelength)
# append band metadata to metaDict
self.logger.debug('metaEntry: %d => %s' %
(bandNo, str(metaEntry)))
metaDict.append(metaEntry)
bandNo += 1
if subBandName == 'Rrs':
metaEntryRrsw = {
'src': [{
'SourceFilename': subDataset[0],
'SourceBand': 1,
'ScaleRatio': slope,
'ScaleOffset': intercept,
'DataType': 6
}],
'dst': {
'wkv':
'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix':
str(wavelength),
'wavelength':
str(wavelength),
'PixelFunctionType':
'NormReflectanceToRemSensReflectance',
}
}
# append band metadata to metaDict
self.logger.debug('metaEntry: %d => %s' %
(bandNo, str(metaEntryRrsw)))
metaDict.append(metaEntryRrsw)
bandNo += 1
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set TIME
startYear = int(gdalMetadata['Start Year'])
startDay = int(gdalMetadata['Start Day'])
startMillisec = int(gdalMetadata['Start Millisec'])
startDate = datetime(startYear, 1, 1) + timedelta(
startDay - 1, 0, 0, startMillisec)
self._set_time(startDate)
# skip adding georeference for GOCI
if title is 'GOCI Level-2 Data':
return
self._remove_geotransform()
# add geolocation
geoMeta = self.geolocationArray.d
if len(geoMeta) > 0:
self.dataset.SetMetadata(geoMeta, 'GEOLOCATION')
# add GCPs
geolocationMetadata = gdalSubDataset.GetMetadata('GEOLOCATION')
xDatasetSource = geolocationMetadata['X_DATASET']
xDataset = gdal.Open(xDatasetSource)
yDatasetSource = geolocationMetadata['Y_DATASET']
yDataset = gdal.Open(yDatasetSource)
longitude = xDataset.ReadAsArray()
latitude = yDataset.ReadAsArray()
# estimate pixel/line step of the geolocation arrays
pixelStep = int(
ceil(
float(gdalSubDataset.RasterXSize) /
float(xDataset.RasterXSize)))
lineStep = int(
ceil(
float(gdalSubDataset.RasterYSize) /
float(xDataset.RasterYSize)))
self.logger.debug('pixel/lineStep %f %f' % (pixelStep, lineStep))
# ==== ADD GCPs and Pojection ====
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT))
if str(title) == 'VIIRSN Level-2 Data':
step0 = 64
self.logger.debug('gcpCount: >%s<, %d %d %f %d %d', title,
latitude.shape[0], latitude.shape[1], GCP_COUNT,
step0, step1)
# generate list of GCPs
dx = .5
dy = .5
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep + dx,
i0 * lineStep + dy)
self.logger.debug('%d %d %d %f %f', k, gcp.GCPPixel,
gcp.GCPLine, gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create VRT '''
ThreddsBase = 'http://thredds.met.no/thredds/dodsC/myocean/siw-tac/siw-metno-svalbard/'
# First check if mapper is called with keyword syntax:
# filename = metno_hires_seaice:YYYYmmdd
keywordBase = 'metno_hires_seaice'
foundDataset = False
if fileName[0:len(keywordBase)] == keywordBase:
keywordTime = fileName[len(keywordBase)+1:]
requestedTime = datetime.strptime(keywordTime, '%Y%m%d')
# Search for nearest available file, within the closest 3 days
for deltaDay in [0, -1, 1, -2, 2, -3, 3]:
validTime = (requestedTime + timedelta(days=deltaDay) +
timedelta(hours=15))
fileName = (ThreddsBase +
validTime.strftime(
'%Y/%m/ice_conc_svalbard_%Y%m%d1500.nc'))
try:
urllib2.urlopen(fileName + '.dds')
foundDataset = True
# Data is found for this day
break
except:
# No data for this day
pass
if not foundDataset:
raise WrongMapperError
# Then check if a valid OPeNDAP URL is given
# (or has been constructed from keyword)
if fileName[0:len(ThreddsBase)] != ThreddsBase:
AttributeError("Not Met.no Svalbard-ice Thredds URL")
else:
timestr = fileName[-15:-3]
validTime = datetime.strptime(timestr, '%Y%m%d%H%M')
fileName = fileName + '?ice_concentration[0][y][x]'
srcProjection = osr.SpatialReference()
srcProjection.ImportFromProj4('+proj=stere lon_0=0.0 +lat_0=90 +datum=WGS84 +ellps=WGS84 +units=km +no_defs')
srcProjection = srcProjection.ExportToWkt()
# From thredds web, with manual shift
srcGeotransform = (-1243.008 - 1, 1, 0, -3190.026 - 7, 0, 1)
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcGeoTransform=srcGeotransform,
srcProjection=srcProjection,
srcRasterXSize=3812,
srcRasterYSize=2980)
metaDict = [{'src': {'SourceFilename': fileName,
'sourceBand': 1},
'dst': {'name': 'sea_ice_area_fraction',
'wkv': 'sea_ice_area_fraction'}}]
# Add band
self._create_bands(metaDict)
# Set time
self.logger.info('Valid time: %s', str(validTime))
self._set_time(validTime)
def __init__(self, fileName, gdalDataset, gdalMetadata,
cache='', lons=None, lats=None, **kwargs):
''' Create NCEP VRT
Parameters:
fileName : str
sstcci_online:analysed_sst:2010-01-01
sstcci_online:analysis_error:2010-01-01
sstcci_online:sea_ice_fraction:2010-01-01
sstcci_online:sea_ice_fraction_error:2010-01-01
sstcci_online:mask:2010-01-01
cache : str or bool
if str - name of the cahcing directory
If False or None - no caching
lon : list
minimum and maimum values of longitude
lat : list
minimum and maimum values of latitude
'''
keywordBase = 'sstcci_online'
if not fileName.startswith(keywordBase):
raise WrongMapperError
# create caching directory
if cache == '':
cache = os.path.curdir
if cache and not os.path.exists(cache):
os.mkdir(cache)
# Get prod name
prodName = fileName.split(':')[1]
# Get date
iDate = parse(fileName.split(':')[2])
# create dataset URL
dsURL = iDate.strftime(self.SST_CCI_URL_FORMAT)
# get lon, lat, time dimensions from the OC CCI Dataset
self.lon, self.lat = self.get_lon_lat(cache, dsURL)
# get rows and cols that contain predefined spatial domain
self.rows, self.cols, lons, lats, geoTransform = self.get_rows_cols(lons, lats)
# create VRT with correct lon/lat (geotransform)
VRT.__init__(self, srcProjection=NSR().wkt,
srcRasterXSize=len(self.cols),
srcRasterYSize=len(self.rows),
srcGeoTransform=geoTransform)
# Get SourceFilename either from memory array or from cached file
sourceFilename = self.get_sourcefilename(cache, dsURL, iDate, prodName, lons, lats)
metaDict = [{'src': {
'SourceFilename': sourceFilename,
'SourceBand': 1,
},
'dst': {
'name': prodName,
}
}]
self._create_bands(metaDict)
# set time
self.dataset.SetMetadataItem('time_coverage_start', iDate.isoformat())
def __init__(self,
inputFileName,
gdalDataset,
gdalMetadata,
logLevel=30,
rmMetadatas=[
'NETCDF_VARNAME', '_Unsigned', 'ScaleRatio',
'ScaleOffset', 'dods_variable'
],
**kwargs):
# Remove 'NC_GLOBAL#' and 'GDAL_' and 'NANSAT_'
# from keys in gdalDataset
tmpGdalMetadata = {}
geoMetadata = {}
origin_is_nansat = False
if not gdalMetadata:
raise WrongMapperError
for key in gdalMetadata.keys():
newKey = key.replace('NC_GLOBAL#', '').replace('GDAL_', '')
if 'NANSAT_' in newKey:
geoMetadata[newKey.replace('NANSAT_', '')] = gdalMetadata[key]
origin_is_nansat = True
else:
tmpGdalMetadata[newKey] = gdalMetadata[key]
gdalMetadata = tmpGdalMetadata
fileExt = os.path.splitext(inputFileName)[1]
# Get file names from dataset or subdataset
subDatasets = gdalDataset.GetSubDatasets()
if len(subDatasets) == 0:
fileNames = [inputFileName]
else:
fileNames = [f[0] for f in subDatasets]
# add bands with metadata and corresponding values to the empty VRT
metaDict = []
xDatasetSource = ''
yDatasetSource = ''
firstXSize = 0
firstYSize = 0
for _, fileName in enumerate(fileNames):
subDataset = gdal.Open(fileName)
# choose the first dataset whith grid
if (firstXSize == 0 and firstYSize == 0
and subDataset.RasterXSize > 1
and subDataset.RasterYSize > 1):
firstXSize = subDataset.RasterXSize
firstYSize = subDataset.RasterYSize
firstSubDataset = subDataset
# get projection from the first subDataset
projection = firstSubDataset.GetProjection()
# take bands whose sizes are same as the first band.
if (subDataset.RasterXSize == firstXSize
and subDataset.RasterYSize == firstYSize):
if projection == '':
projection = subDataset.GetProjection()
if ('GEOLOCATION_X_DATASET' in fileName
or 'longitude' in fileName):
xDatasetSource = fileName
elif ('GEOLOCATION_Y_DATASET' in fileName
or 'latitude' in fileName):
yDatasetSource = fileName
else:
for iBand in range(subDataset.RasterCount):
subBand = subDataset.GetRasterBand(iBand + 1)
bandMetadata = subBand.GetMetadata_Dict()
if 'PixelFunctionType' in bandMetadata:
bandMetadata.pop('PixelFunctionType')
sourceBands = iBand + 1
# sourceBands = i*subDataset.RasterCount + iBand + 1
# generate src metadata
src = {
'SourceFilename': fileName,
'SourceBand': sourceBands
}
# set scale ratio and scale offset
scaleRatio = bandMetadata.get(
'ScaleRatio',
bandMetadata.get(
'scale', bandMetadata.get('scale_factor', '')))
if len(scaleRatio) > 0:
src['ScaleRatio'] = scaleRatio
scaleOffset = bandMetadata.get(
'ScaleOffset',
bandMetadata.get(
'offset', bandMetadata.get('add_offset', '')))
if len(scaleOffset) > 0:
src['ScaleOffset'] = scaleOffset
# sate DataType
src['DataType'] = subBand.DataType
# generate dst metadata
# get all metadata from input band
dst = bandMetadata
# set wkv and bandname
dst['wkv'] = bandMetadata.get('standard_name', '')
# first, try the name metadata
if 'name' in bandMetadata:
bandName = bandMetadata['name']
else:
# if it doesn't exist get name from NETCDF_VARNAME
bandName = bandMetadata.get('NETCDF_VARNAME', '')
if len(bandName) == 0:
bandName = bandMetadata.get(
'dods_variable', '')
# remove digits added by gdal in
# exporting to netcdf...
if (len(bandName) > 0 and origin_is_nansat
and fileExt == '.nc'):
if bandName[-1:].isdigit():
bandName = bandName[:-1]
if bandName[-1:].isdigit():
bandName = bandName[:-1]
# if still no bandname, create one
if len(bandName) == 0:
bandName = 'band_%03d' % iBand
dst['name'] = bandName
# remove non-necessary metadata from dst
for rmMetadata in rmMetadatas:
if rmMetadata in dst:
dst.pop(rmMetadata)
# append band with src and dst dictionaries
metaDict.append({'src': src, 'dst': dst})
# create empty VRT dataset with geolocation only
VRT.__init__(self, firstSubDataset, srcMetadata=gdalMetadata)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Create complex data bands from 'xxx_real' and 'xxx_imag' bands
# using pixelfunctions
rmBands = []
for iBandNo in range(self.dataset.RasterCount):
iBand = self.dataset.GetRasterBand(iBandNo + 1)
iBandName = iBand.GetMetadataItem('name')
# find real data band
if iBandName.find("_real") != -1:
realBandNo = iBandNo
realBand = self.dataset.GetRasterBand(realBandNo + 1)
realDtype = realBand.GetMetadataItem('DataType')
bandName = iBandName.replace(iBandName.split('_')[-1],
'')[0:-1]
for jBandNo in range(self.dataset.RasterCount):
jBand = self.dataset.GetRasterBand(jBandNo + 1)
jBandName = jBand.GetMetadataItem('name')
# find an imaginary data band corresponding to the real
# data band and create complex data band from the bands
if jBandName.find(bandName + '_imag') != -1:
imagBandNo = jBandNo
imagBand = self.dataset.GetRasterBand(imagBandNo + 1)
imagDtype = imagBand.GetMetadataItem('DataType')
dst = imagBand.GetMetadata()
dst['name'] = bandName
dst['PixelFunctionType'] = 'ComplexData'
dst['dataType'] = 10
src = [{
'SourceFilename': fileNames[realBandNo],
'SourceBand': 1,
'DataType': realDtype
}, {
'SourceFilename': fileNames[imagBandNo],
'SourceBand': 1,
'DataType': imagDtype
}]
self._create_band(src, dst)
self.dataset.FlushCache()
rmBands.append(realBandNo + 1)
rmBands.append(imagBandNo + 1)
# Delete real and imaginary bands
if len(rmBands) != 0:
self.delete_bands(rmBands)
if len(projection) == 0:
# projection was not set automatically
# get projection from GCPProjection
projection = geoMetadata.get('GCPProjection', '')
if len(projection) == 0:
# no projection was found in dataset or metadata:
# generate WGS84 by default
projection = NSR().wkt
# fix problem with MET.NO files where a, b given in m and XC/YC in km
if ('UNIT["kilometre"' in projection
and ',SPHEROID["Spheroid",6378273,7.331926543631893e-12]'
in projection):
projection = projection.replace(
',SPHEROID["Spheroid",6378273,7.331926543631893e-12]', '')
# set projection
self.dataset.SetProjection(self.repare_projection(projection))
# check if GCPs were added from input dataset
gcps = firstSubDataset.GetGCPs()
gcpProjection = firstSubDataset.GetGCPProjection()
# if no GCPs in input dataset: try to add GCPs from metadata
if not gcps:
gcps = self.add_gcps_from_metadata(geoMetadata)
# if yet no GCPs: try to add GCPs from variables
if not gcps:
gcps = self.add_gcps_from_variables(inputFileName)
if gcps:
if len(gcpProjection) == 0:
# get GCP projection and repare
gcpProjection = self.repare_projection(
geoMetadata.get('GCPProjection', ''))
# add GCPs to dataset
self.dataset.SetGCPs(gcps, gcpProjection)
self.dataset.SetProjection('')
self._remove_geotransform()
# Find proper bands and insert GEOLOCATION ARRAY into dataset
if len(xDatasetSource) > 0 and len(yDatasetSource) > 0:
self.add_geolocationArray(
GeolocationArray(xDatasetSource, yDatasetSource))
elif not gcps:
# if no GCPs found and not GEOLOCATION ARRAY set:
# Set Nansat Geotransform if it is not set automatically
geoTransform = self.dataset.GetGeoTransform()
if len(geoTransform) == 0:
geoTransformStr = geoMetadata.get('GeoTransform',
'(0|1|0|0|0|0|1)')
geoTransform = eval(geoTransformStr.replace('|', ','))
self.dataset.SetGeoTransform(geoTransform)
subMetadata = firstSubDataset.GetMetadata()
### GET START TIME from METADATA
time_coverage_start = None
if 'start_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['start_time'])
elif 'start_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['start_date'])
elif 'time_coverage_start' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_start'])
### GET END TIME from METADATA
time_coverage_end = None
if 'stop_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['stop_time'])
elif 'stop_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['stop_date'])
elif 'time_coverage_stop' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_stop'])
elif 'end_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['end_time'])
elif 'end_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['end_date'])
elif 'time_coverage_end' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['time_coverage_end'])
### GET start time from time variable
if (time_coverage_start is None and cfunitsInstalled
and 'time#standard_name' in subMetadata
and subMetadata['time#standard_name'] == 'time'
and 'time#units' in subMetadata
and 'time#calendar' in subMetadata):
# get data from netcdf data
ncFile = netcdf_file(inputFileName, 'r')
timeLength = ncFile.variables['time'].shape[0]
timeValueStart = ncFile.variables['time'][0]
timeValueEnd = ncFile.variables['time'][-1]
ncFile.close()
try:
timeDeltaStart = Units.conform(
timeValueStart,
Units(subMetadata['time#units'],
calendar=subMetadata['time#calendar']),
Units('days since 1950-01-01'))
except ValueError:
self.logger.error('calendar units are wrong: %s' %
subMetadata['time#calendar'])
else:
time_coverage_start = (
datetime.datetime(1950, 1, 1) +
datetime.timedelta(float(timeDeltaStart)))
if timeLength > 1:
timeDeltaEnd = Units.conform(
timeValueStart,
Units(subMetadata['time#units'],
calendar=subMetadata['time#calendar']),
Units('days since 1950-01-01'))
else:
timeDeltaEnd = timeDeltaStart + 1
time_coverage_end = (datetime.datetime(1950, 1, 1) +
datetime.timedelta(float(timeDeltaEnd)))
## finally set values of time_coverage start and end if available
if time_coverage_start is not None:
self.dataset.SetMetadataItem('time_coverage_start',
time_coverage_start.isoformat())
if time_coverage_end is not None:
self.dataset.SetMetadataItem('time_coverage_end',
time_coverage_end.isoformat())
if 'sensor' not in gdalMetadata:
self.dataset.SetMetadataItem('sensor', 'unknown')
if 'satellite' not in gdalMetadata:
self.dataset.SetMetadataItem('satellite', 'unknown')
if 'source_type' not in gdalMetadata:
self.dataset.SetMetadataItem('source_type', 'unknown')
if 'platform' not in gdalMetadata:
self.dataset.SetMetadataItem('platform', 'unknown')
if 'instrument' not in gdalMetadata:
self.dataset.SetMetadataItem('instrument', 'unknown')
self.logger.info('Use generic mapper - OK!')
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create CSKS VRT '''
if fileName.split('/')[-1][0:4] != "CSKS":
raise WrongMapperError
# Get coordinates
metadata = gdalMetadata['Estimated_Bottom_Left_Geodetic_Coordinates']
bottom_left_lon = float(metadata.split(' ')[1])
bottom_left_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Estimated_Bottom_Right_Geodetic_Coordinates']
bottom_right_lon = float(metadata.split(' ')[1])
bottom_right_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Estimated_Top_Left_Geodetic_Coordinates']
top_left_lon = float(metadata.split(' ')[1])
top_left_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Estimated_Top_Right_Geodetic_Coordinates']
top_right_lon = float(metadata.split(' ')[1])
top_right_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Scene_Centre_Geodetic_Coordinates']
center_lon = float(metadata.split(' ')[1])
center_lat = float(metadata.split(' ')[0])
# Get sub-datasets
subDatasets = gdalDataset.GetSubDatasets()
# Get file names from dataset or subdataset
if subDatasets.__len__() == 1:
fileNames = [fileName]
else:
fileNames = [f[0] for f in subDatasets]
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'QLK':
fileNames.pop(i)
#print fileNames
subDataset = gdal.Open(fileNames[0])
# generate list of GCPs
gcps = []
# create GCP with X,Y,Z(?),pixel,line from lat/lon matrices
gcp = gdal.GCP(float(bottom_left_lon), float(bottom_left_lat), 0, 0, 0)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 0, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(bottom_right_lon), float(bottom_right_lat), 0,
subDataset.RasterXSize, 0)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 1, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_left_lon), float(top_left_lat), 0, 0,
subDataset.RasterYSize)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 2, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_right_lon), float(top_right_lat), 0,
subDataset.RasterXSize, subDataset.RasterYSize)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 3, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(center_lon), float(center_lat), 0,
int(np.round(subDataset.RasterXSize / 2.)),
int(round(subDataset.RasterYSize / 2.)))
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 4, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
# append GCPs and lat/lon projection to the vsiDataset
latlongSRS = osr.SpatialReference()
latlongSRS.ImportFromProj4(
"+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
latlongSRSWKT = latlongSRS.ExportToWkt()
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
srcGCPs=gcps,
srcGCPProjection=latlongSRSWKT)
#print self.fileName
# Read all bands later
#band='S01'
#res='SBI'
# Use only full size "original" datasets
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'SBI':
# Add real and imaginary raw counts as bands
src = {
'SourceFilename': fileNames[i],
'SourceBand': 1,
'DataType': gdal.GDT_Int16
}
dst = {
'dataType':
gdal.GDT_Float32,
'name':
'RawCounts_%s_real' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation']
}
self._create_band(src, dst)
src = {
'SourceFilename': fileNames[i],
'SourceBand': 2,
'DataType': gdal.GDT_Int16
}
dst = {
'dataType':
gdal.GDT_Float32,
'name':
'RawCounts_%s_imaginary' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation']
}
self._create_band(src, dst)
self.dataset.FlushCache()
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'SBI':
# Calculate sigma0 scaling factor
Rref = float(gdalMetadata['Reference_Slant_Range'])
Rexp = float(gdalMetadata['Reference_Slant_Range_Exponent'])
alphaRef = float(gdalMetadata['Reference_Incidence_Angle'])
F = float(gdalMetadata['Rescaling_Factor'])
K = float(gdalMetadata[fileNames[i][-7:-4] +
'_Calibration_Constant'])
Ftot = Rref**(2. * Rexp)
Ftot *= np.sin(alphaRef * np.pi / 180.0)
Ftot /= F**2.
Ftot /= K
#print Ftot
src = [{
'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2 * i + 1,
'ScaleRatio': np.sqrt(Ftot)
}, {
'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2 * i + 2,
'ScaleRatio': np.sqrt(Ftot)
}]
dst = {
'wkv':
'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType':
'RawcountsToSigma0_CosmoSkymed_SBI',
'polarisation':
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation'],
'name':
'sigma0_%s' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation'],
'SatelliteID':
gdalMetadata['Satellite_ID'],
'dataType':
gdal.GDT_Float32
}
#'pass': gdalMetadata['']
# - I can't find this in the metadata...
self._create_band(src, dst)
self.dataset.FlushCache()
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
resolution='low',
**kwargs):
''' Create LANDSAT VRT from multiple tif files or single tar.gz file'''
mtlFileName = ''
bandFileNames = []
bandSizes = []
bandDatasets = []
fname = os.path.split(fileName)[1]
if (fileName.endswith('.tar') or fileName.endswith('.tar.gz')
or fileName.endswith('.tgz')):
# try to open .tar or .tar.gz or .tgz file with tar
try:
tarFile = tarfile.open(fileName)
except:
raise WrongMapperError
# collect names of bands and corresponding sizes
# into bandsInfo dict and bandSizes list
tarNames = sorted(tarFile.getnames())
for tarName in tarNames:
# check if TIF files inside TAR qualify
if (tarName[0] in ['L', 'M']
and os.path.splitext(tarName)[1] in ['.TIF', '.tif']):
# open TIF file from TAR using VSI
sourceFilename = '/vsitar/%s/%s' % (fileName, tarName)
gdalDatasetTmp = gdal.Open(sourceFilename)
# keep name, GDALDataset and size
bandFileNames.append(sourceFilename)
bandSizes.append(gdalDatasetTmp.RasterXSize)
bandDatasets.append(gdalDatasetTmp)
elif (tarName.endswith('MTL.txt')
or tarName.endswith('MTL.TXT')):
# get mtl file
mtlFileName = tarName
elif ((fname.startswith('L') or fname.startswith('M'))
and (fname.endswith('.tif') or fname.endswith('.TIF')
or fname.endswith('._MTL.txt'))):
# try to find TIF/tif files with the same name as input file
path, coreName = os.path.split(fileName)
coreName = os.path.splitext(coreName)[0].split('_')[0]
coreNameMask = coreName + '*[tT][iI][fF]'
tifNames = sorted(glob.glob(os.path.join(path, coreNameMask)))
for tifName in tifNames:
sourceFilename = tifName
gdalDatasetTmp = gdal.Open(sourceFilename)
# keep name, GDALDataset and size
bandFileNames.append(sourceFilename)
bandSizes.append(gdalDatasetTmp.RasterXSize)
bandDatasets.append(gdalDatasetTmp)
# get mtl file
mtlFiles = glob.glob(coreName + '*[mM][tT][lL].[tT][xX][tT]')
if len(mtlFiles) > 0:
mtlFileName = mtlFiles[0]
else:
raise WrongMapperError
# if not TIF files found - not appropriate mapper
if not bandFileNames:
raise WrongMapperError
# get appropriate band size based on number of unique size and
# required resoltuion
if resolution == 'low':
bandXSise = min(bandSizes)
elif resolution in ['high', 'hi']:
bandXSise = max(bandSizes)
else:
raise OptionError('Wrong resolution %s for file %s' %
(resolution, fileName))
# find bands with appropriate size and put to metaDict
metaDict = []
for bandFileName, bandSize, bandDataset in zip(bandFileNames,
bandSizes,
bandDatasets):
if bandSize == bandXSise:
# let last part of file name be suffix
bandSuffix = os.path.splitext(bandFileName)[0].split('_')[-1]
metaDict.append({
'src': {
'SourceFilename': bandFileName,
'SourceBand': 1,
'ScaleRatio': 0.1
},
'dst': {
'wkv': 'toa_outgoing_spectral_radiance',
'suffix': bandSuffix
}
})
gdalDataset4Use = bandDataset
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset4Use)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
if len(mtlFileName) > 0:
mtlFileName = os.path.join(
os.path.split(bandFileNames[0])[0], mtlFileName)
mtlFileLines = [
line.strip() for line in self.read_xml(mtlFileName).split('\n')
]
dateString = [
line.split('=')[1].strip() for line in mtlFileLines
if ('DATE_ACQUIRED' in line or 'ACQUISITION_DATE' in line)
][0]
timeStr = [
line.split('=')[1].strip() for line in mtlFileLines
if ('SCENE_CENTER_TIME' in line
or 'SCENE_CENTER_SCAN_TIME' in line)
][0]
time_start = parse_time(dateString + 'T' + timeStr).isoformat()
time_end = (parse_time(dateString + 'T' + timeStr) +
datetime.timedelta(microseconds=60000000)).isoformat()
self.dataset.SetMetadataItem('time_coverage_start', time_start)
self.dataset.SetMetadataItem('time_coverage_end', time_end)
# set platform
platform = 'LANDSAT'
if fname[2].isdigit():
platform += '-' + fname[2]
ee = pti.get_gcmd_platform(platform)
self.dataset.SetMetadataItem('platform', json.dumps(ee))
# set instrument
instrument = {
'LANDSAT': 'MSS',
'LANDSAT-1': 'MSS',
'LANDSAT-2': 'MSS',
'LANDSAT-3': 'MSS',
'LANDSAT-4': 'TM',
'LANDSAT-5': 'TM',
'LANDSAT-7': 'ETM+',
'LANDSAT-8': 'OLI'
}[platform]
ee = pti.get_gcmd_instrument(instrument)
self.dataset.SetMetadataItem('instrument', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create SMOS VRT '''
# check extension
fName = os.path.split(fileName)[1]
fExt = os.path.splitext(fileName)[1]
if fExt == '.MAT' or fExt == '.mat' and 'OSUDP2' in fName:
# load file
matFile = loadmat(fileName)
else:
raise WrongMapperError
# get geolocation
geolocArray = matFile['geolocation'][0]
srcProj4 = (
'+proj=stere +lon_0=%f +lat_0=%f +datum=WGS84 +ellps=WGS84 +units=km +no_defs'
% (geolocArray[0], geolocArray[1]))
srcProjection = osr.SpatialReference()
srcProjection.ImportFromProj4(srcProj4)
srcProjection = srcProjection.ExportToWkt()
srcGeotransform = (geolocArray[2], geolocArray[4], 0, geolocArray[3],
0, geolocArray[5])
lon = matFile['longitude']
#lat = matFile['latitude']
srcRasterYSize, srcRasterXSize = lon.shape
# create VRT from lat/lon
# VRT.__init__(self, lon=lon, lat=lat)
VRT.__init__(self,
srcGeoTransform=srcGeotransform,
srcProjection=srcProjection,
srcRasterXSize=srcRasterXSize,
srcRasterYSize=srcRasterYSize)
# add the following variables
varNames = [
'SSS1', 'SSS2', 'SSS3', 'SST', 'Sigma_SSS1', 'Sigma_SSS2',
'Sigma_SSS3', 'Control_Flags_1', 'Control_Flags_2',
'Control_Flags_3', 'Control_Flags_4', 'Science_Flags_1',
'Science_Flags_2', 'Science_Flags_3', 'Science_Flags_4'
]
metaDict = []
for varName in varNames:
var = matFile[varName]
self.bandVRTs[varName] = VRT(array=var)
metaDict.append({
'src': {
'SourceFilename': self.bandVRTs[varName].fileName,
'sourceBand': 1
},
'dst': {
'name': varName
}
})
# create mask
cloudBits = [2, 3, 4, 5, 6]
maxSigma = 3.0
mask = np.zeros(lon.shape, 'uint16')
mask[:] = 128
mask[np.isnan(matFile['SSS1'])] = 0
mask[matFile['Sigma_SSS1'] > maxSigma] = 1
mask[matFile['Sigma_SSS2'] > maxSigma] = 1
mask[matFile['Sigma_SSS3'] > maxSigma] = 1
for cloudBit in cloudBits:
for cfi in range(1, 5):
bitMap = np.bitwise_and(matFile['Control_Flags_%d' % cfi],
np.power(2, cloudBit))
mask[bitMap > 0] = 1
self.bandVRTs['mask'] = VRT(array=mask)
metaDict.append({
'src': {
'SourceFilename': self.bandVRTs['mask'].fileName,
'sourceBand': 1
},
'dst': {
'name': 'mask'
}
})
self.logger.debug('metaDict: %s' % metaDict)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_COUNT=10, **kwargs):
''' Create VRT
Parameters
----------
GCP_COUNT : int
number of GCPs along each dimention
'''
# should raise error in case of not obpg_l2 file
try:
title = gdalMetadata["Title"]
except:
raise WrongMapperError
if title not in self.titles:
raise WrongMapperError
# get subdataset and parse to VRT.__init__()
# for retrieving geo-metadata
# but NOT from longitude or latitude because it can be smaller!
subDatasets = gdalDataset.GetSubDatasets()
for subDataset in subDatasets:
if ('longitude' not in subDataset[1] and
'latitude' not in subDataset[1]):
gdalSubDataset = gdal.Open(subDataset[0])
break
if title is 'GOCI Level-2 Data':
# set GOCI projection parameters
rasterXSize = 5567
rasterYSize = 5685
proj4 = '+proj=ortho +lat_0=36 +lon_0=130 units=m +ellps=WGS84 +datum=WGS84 +no_defs'
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4)
projection = srs.ExportToWkt()
geoTransform = (-1391500.0, 500.0, 0.0, 1349500.0, 0.0, -500.0)
# create empty VRT dataset with georeference only
VRT.__init__(self, srcGeoTransform=geoTransform,
srcProjection=projection,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
else:
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalSubDataset)
# parts of dictionary for all Reflectances
#dictRrs = {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air', 'wavelength': '412'} }
# dictionary for all possible bands
allBandsDict = {'Rrs': {'src': {},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air'}},
'Kd': {'src': {},
'dst': {'wkv': 'volume_attenuation_coefficient_of_downwelling_radiative_flux_in_sea_water'}},
'chlor_a': {'src': {},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'case': 'I'}},
'cdom_index': {'src': {},
'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'case': 'II'}},
'sst': {'src': {},
'dst': {'wkv': 'sea_surface_temperature'}},
'sst4': {'src': {},
'dst': {'wkv': 'sea_surface_temperature'}},
'l2_flags': {'src': {'SourceType': 'SimpleSource',
'DataType': 4},
'dst': {'wkv': 'quality_flags',
'dataType': 4}},
'qual_sst': {'src': {'SourceType': 'SimpleSource',
'DataType': 4},
'dst': {'wkv': 'quality_flags',
'name': 'qual_sst',
'dataType': 4}},
'qual_sst4': {'src': {'SourceType': 'SimpleSource',
'DataType': 4},
'dst': {'wkv': 'quality_flags',
'name': 'qual_sst',
'dataType': 4}},
'latitude': {'src': {},
'dst': {'wkv': 'latitude'}},
'longitude': {'src': {},
'dst': {'wkv': 'longitude'}},
'par': {'src': {},
'dst': {'wkv': 'downwelling_photosynthetic_photon_radiance_in_sea_water'}},
'ipar': {'src': {},
'dst': {'wkv': 'instantaneous_downwelling_photosynthetic_photon_radiance_in_sea_water'}},
}
# loop through available bands and generate metaDict (non fixed)
metaDict = []
bandNo = 0
for subDataset in subDatasets:
# get sub dataset name
subDatasetName = subDataset[1].split(' ')[1]
self.logger.debug('Subdataset: %s' % subDataset[1])
self.logger.debug('Subdataset name: "%s"' % subDatasetName)
# get wavelength if applicable, get dataset name without wavelength
try:
wavelength = int(subDatasetName.split('_')[-1])
except:
wavelength = None
subBandName = subDatasetName
else:
subBandName = subDatasetName.split('_')[0]
self.logger.debug('subBandName, wavelength: %s %s' %
(subBandName, str(wavelength)))
if subBandName in allBandsDict:
# get name, slope, intercept
self.logger.debug('name: %s' % subBandName)
tmpSubDataset = gdal.Open(subDataset[0])
tmpSubMetadata = tmpSubDataset.GetMetadata()
slope = tmpSubMetadata.get('slope', '1')
intercept = tmpSubMetadata.get('intercept', '0')
self.logger.debug('slope, intercept: %s %s ' %
(slope, intercept))
# create meta entry
metaEntry = {'src': {'SourceFilename': subDataset[0],
'sourceBand': 1,
'ScaleRatio': slope,
'ScaleOffset': intercept},
'dst': {}
}
# add more to src
for srcKey in allBandsDict[subBandName]['src']:
metaEntry['src'][srcKey] = (allBandsDict[subBandName]
['src'][srcKey])
# add dst from allBandsDict
for dstKey in allBandsDict[subBandName]['dst']:
metaEntry['dst'][dstKey] = (allBandsDict[subBandName]
['dst'][dstKey])
# add wavelength, band name to dst
if wavelength is not None:
metaEntry['dst']['suffix'] = str(wavelength)
metaEntry['dst']['wavelength'] = str(wavelength)
# append band metadata to metaDict
self.logger.debug('metaEntry: %d => %s' % (bandNo,
str(metaEntry)))
metaDict.append(metaEntry)
bandNo += 1
if subBandName == 'Rrs':
rrsSubDataset = subDataset[0]
metaEntryRrsw = {
'src': [{
'SourceFilename': subDataset[0],
'SourceBand': 1,
'ScaleRatio': slope,
'ScaleOffset': intercept,
'DataType': 6}],
'dst': {
'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix': str(wavelength),
'wavelength': str(wavelength),
'PixelFunctionType': 'NormReflectanceToRemSensReflectance',
}}
# append band metadata to metaDict
self.logger.debug('metaEntry: %d => %s' %
(bandNo, str(metaEntryRrsw)))
metaDict.append(metaEntryRrsw)
bandNo += 1
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set TIME
startYear = int(gdalMetadata['Start Year'])
startDay = int(gdalMetadata['Start Day'])
startMillisec = int(gdalMetadata['Start Millisec'])
startDate = datetime(startYear, 1, 1) + timedelta(startDay-1, 0, 0,
startMillisec)
# skip adding georeference for GOCI
if title is 'GOCI Level-2 Data':
return
self._remove_geotransform()
# add geolocation
geoMeta = self.geolocationArray.d
if len(geoMeta) > 0:
self.dataset.SetMetadata(geoMeta, 'GEOLOCATION')
# add GCPs
geolocationMetadata = gdalSubDataset.GetMetadata('GEOLOCATION')
xDatasetSource = geolocationMetadata['X_DATASET']
xDataset = gdal.Open(xDatasetSource)
yDatasetSource = geolocationMetadata['Y_DATASET']
yDataset = gdal.Open(yDatasetSource)
longitude = xDataset.ReadAsArray()
latitude = yDataset.ReadAsArray()
# estimate pixel/line step of the geolocation arrays
pixelStep = int(ceil(float(gdalSubDataset.RasterXSize) /
float(xDataset.RasterXSize)))
lineStep = int(ceil(float(gdalSubDataset.RasterYSize) /
float(xDataset.RasterYSize)))
self.logger.debug('pixel/lineStep %f %f' % (pixelStep, lineStep))
# ==== ADD GCPs and Pojection ====
# estimate step of GCPs
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT))
if str(title) == 'VIIRSN Level-2 Data':
step0 = 64
self.logger.debug('gcpCount: >%s<, %d %d %f %d %d',
title,
latitude.shape[0], latitude.shape[1],
GCP_COUNT, step0, step1)
# generate list of GCPs
dx = .5
dy = .5
gcps = []
k = 0
center_lon = 0
center_lat = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep + dx,
i0 * lineStep + dy)
self.logger.debug('%d %d %d %f %f',
k, gcp.GCPPixel, gcp.GCPLine,
gcp.GCPX, gcp.GCPY)
gcps.append(gcp)
center_lon += gcp.GCPX
center_lat += gcp.GCPY
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
self.remove_geolocationArray()
# reproject GCPs
center_lon /= k
center_lat /= k
srs = '+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=%f +lat_0=%f +no_defs' % (center_lon, center_lat)
self.reproject_GCPs(srs)
# use TPS for reprojection
self.tps = True
# add NansenCloud metadata
self.dataset.SetMetadataItem('time_coverage_start',
(parse(
gdalMetadata['time_coverage_start']).
isoformat()))
self.dataset.SetMetadataItem('time_coverage_end',
(parse(
gdalMetadata['time_coverage_stop']).
isoformat()))
instrument = gdalMetadata['Sensor Name'][1:-1]
platform = {'A': 'AQUA', 'T': 'TERRA'}[gdalMetadata['Sensor Name'][-1]]
mm = pti.get_gcmd_instrument(instrument)
ee = pti.get_gcmd_platform(platform)
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Create CSKS VRT '''
if fileName.split('/')[-1][0:4] != "CSKS":
raise WrongMapperError
# Get coordinates
metadata = gdalMetadata['Estimated_Bottom_Left_Geodetic_Coordinates']
bottom_left_lon = float(metadata.split(' ')[1])
bottom_left_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Estimated_Bottom_Right_Geodetic_Coordinates']
bottom_right_lon = float(metadata.split(' ')[1])
bottom_right_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Estimated_Top_Left_Geodetic_Coordinates']
top_left_lon = float(metadata.split(' ')[1])
top_left_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Estimated_Top_Right_Geodetic_Coordinates']
top_right_lon = float(metadata.split(' ')[1])
top_right_lat = float(metadata.split(' ')[0])
metadata = gdalMetadata['Scene_Centre_Geodetic_Coordinates']
center_lon = float(metadata.split(' ')[1])
center_lat = float(metadata.split(' ')[0])
# Get sub-datasets
subDatasets = gdalDataset.GetSubDatasets()
# Get file names from dataset or subdataset
if subDatasets.__len__() == 1:
fileNames = [fileName]
else:
fileNames = [f[0] for f in subDatasets]
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'QLK':
fileNames.pop(i)
#print fileNames
subDataset = gdal.Open(fileNames[0])
# generate list of GCPs
gcps = []
# create GCP with X,Y,Z(?),pixel,line from lat/lon matrices
gcp = gdal.GCP(float(bottom_left_lon),
float(bottom_left_lat), 0, 0, 0)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 0, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(bottom_right_lon),
float(bottom_right_lat),
0,
subDataset.RasterXSize,
0)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 1, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_left_lon),
float(top_left_lat),
0,
0,
subDataset.RasterYSize)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 2, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(top_right_lon),
float(top_right_lat),
0,
subDataset.RasterXSize,
subDataset.RasterYSize)
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 3, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
gcp = gdal.GCP(float(center_lon),
float(center_lat),
0,
int(np.round(subDataset.RasterXSize/2.)),
int(round(subDataset.RasterYSize/2.)))
gcps.append(gcp)
#self.logger.debug('%d %d %d %f %f', 4, gcp.GCPPixel, gcp.GCPLine,
# gcp.GCPX, gcp.GCPY)
# append GCPs and lat/lon projection to the vsiDataset
latlongSRS = osr.SpatialReference()
latlongSRS.ImportFromProj4("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
latlongSRSWKT = latlongSRS.ExportToWkt()
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
srcGCPs=gcps,
srcGCPProjection=latlongSRSWKT)
#print self.fileName
# Read all bands later
#band='S01'
#res='SBI'
# Use only full size "original" datasets
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'SBI':
# Add real and imaginary raw counts as bands
src = {'SourceFilename': fileNames[i],
'SourceBand': 1,
'DataType': gdal.GDT_Int16}
dst = {'dataType': gdal.GDT_Float32,
'name': 'RawCounts_%s_real' %
gdalMetadata[fileNames[i][-7:-4]+'_Polarisation']}
self._create_band(src, dst)
src = {'SourceFilename': fileNames[i],
'SourceBand': 2,
'DataType': gdal.GDT_Int16}
dst = {'dataType': gdal.GDT_Float32,
'name': 'RawCounts_%s_imaginary' %
gdalMetadata[fileNames[i][-7:-4] + '_Polarisation']}
self._create_band(src, dst)
self.dataset.FlushCache()
for i, elem in enumerate(fileNames):
if fileNames[i][-3:] == 'SBI':
# Calculate sigma0 scaling factor
Rref = float(gdalMetadata['Reference_Slant_Range'])
Rexp = float(gdalMetadata['Reference_Slant_Range_Exponent'])
alphaRef = float(gdalMetadata['Reference_Incidence_Angle'])
F = float(gdalMetadata['Rescaling_Factor'])
K = float(gdalMetadata[fileNames[i][-7:-4] +
'_Calibration_Constant'])
Ftot = Rref**(2.*Rexp)
Ftot *= np.sin(alphaRef*np.pi / 180.0)
Ftot /= F**2.
Ftot /= K
#print Ftot
src = [{'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2*i+1,
'ScaleRatio': np.sqrt(Ftot)},
{'SourceFilename': self.fileName,
'DataType': gdal.GDT_Float32,
'SourceBand': 2*i+2,
'ScaleRatio': np.sqrt(Ftot)}]
dst = {'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave',
'PixelFunctionType': 'RawcountsToSigma0_CosmoSkymed_SBI',
'polarisation': gdalMetadata[fileNames[i][-7:-4] +
'_Polarisation'],
'name': 'sigma0_%s' % gdalMetadata[fileNames[i][-7:-4] +
'_Polarisation'],
'SatelliteID': gdalMetadata['Satellite_ID'],
'dataType': gdal.GDT_Float32}
#'pass': gdalMetadata['']
# - I can't find this in the metadata...
self._create_band(src, dst)
self.dataset.FlushCache()
self.dataset.SetMetadataItem('time_coverage_start',
parse_time(gdalMetadata['Scene_Sensing_Start_UTC']).isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
parse_time(gdalMetadata['Scene_Sensing_Stop_UTC']).isoformat())
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' OBPG L3 VRT '''
# test the product
try:
assert gdalMetadata['PlatformShortName'] == 'GCOM-W1'
assert gdalMetadata['SensorShortName'] == 'AMSR2'
assert gdalMetadata['ProductName'] == 'AMSR2-L3'
except:
raise WrongMapperError
# get list of similar (same date, A/D orbit) files in the directory
iDir, iFile = os.path.split(fileName)
iFileMask = iFile[:30] + '%02d' + iFile[32:]
simFiles = []
for freq in self.freqs:
simFile = os.path.join(iDir, iFileMask % freq)
#print simFile
if os.path.exists(simFile):
simFiles.append(simFile)
metaDict = []
for freq in self.freqs:
simFile = os.path.join(iDir, iFileMask % freq)
if simFile not in simFiles:
continue
#print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
if simSupDataset is None:
# skip this similar file
#print 'No dataset: %s not a supported SMI file' % simFile
continue
# get subdatasets from the similar file
simSubDatasets = simSupDataset.GetSubDatasets()
for simSubDataset in simSubDatasets:
#print 'simSubDataset', simSubDataset
if 'Brightness_Temperature' in simSubDataset[0]:
# get SourceFilename from subdataset
metaEntry = {
'src': {'SourceFilename': simSubDataset[0],
'SourceBand': 1,
'ScaleRatio': 0.0099999998,
'ScaleOffset': 0},
'dst': {'wkv': 'brightness_temperature',
'frequency': '%02d' % freq,
'polarisation': simSubDataset[0][-2:-1],
'suffix': ('%02d%s' %
(freq, simSubDataset[0][-2:-1]))}}
metaDict.append(metaEntry)
# initiate VRT for the NSIDC 10 km grid
VRT.__init__(self,
srcGeoTransform=(-3850000, 10000, 0.0,
5850000, 0.0, -10000),
srcProjection=NSR(3411).wkt,
srcRasterXSize=760,
srcRasterYSize=1120)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start', parse(gdalMetadata['ObservationStartDateTime']).isoformat())
self.dataset.SetMetadataItem('time_coverage_end', parse(gdalMetadata['ObservationStartDateTime']).isoformat())
mm = pti.get_gcmd_instrument('AMSR2')
ee = pti.get_gcmd_platform('GCOM-W1')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, varName=None, **kwargs):
''' Create VRT from OpenDAP dataset'''
raise WrongMapperError
# quit if file is not online
if fileName[:7] not in ['http://', 'https:/']:
raise WrongMapperError
if bandName is None:
WrongMapperError('Please specify band name')
# open file through OpenDAP using netCDF4 library
f = Dataset(fileName)
# assume CF-compatibility:
# compulsory grid_mapping_name
# find grid_mapping_name
# and get all parameters
# generate proj4 and WKT strings
srcProjection = ''
for varName in f.variables:
var = f.variables[varName]
attrs = var.ncattrs()
if 'grid_mapping_name' in attrs:
proj4str = self.get_proj4_from_ncvar(var)
srcProjection = NSR(proj4str).wkt
break
print 'WKT:', srcProjection
xDim = None
yDim = None
validVars = []
validDims = []
# if grid_mapping_name is found: find all bands that have grid_mapping
# else: find lon/lat variables that have 1D
import ipdb; ipdb.set_trace()
if srcProjection != '':
# find bands with grid_mapping
var = f.variables[varName]
attrs = var.ncattrs()
if 'grid_mapping' in attrs:
validVars.append(str(varName))
validDims += [str(dim) for dim in var.dimensions]
# assume NORMAP compatibility:
# dimensions should be
# x, or xc, or lon, or longitude
# and
# y, or yc, or lat, or latitude
# and/or
# time
# and/or
# depth
# and/or
# etc
# assign x, y dimension names
for dim in validDims:
if 'x' in dim or 'lon' in dim or 'east' in dim:
xDim = dim
if 'y' in dim or 'lat' in dim or 'north' in dim:
yDim = dim
else:
# if input file is not CF-compliant but has 1D lon/lat dimensions
# find 1D lon, lat (longitude, latitude) dims
var = f.variables[varName]
if 'lon' in varName and var.ndim == 1:
xDim = varName
if 'lat' in varName and var.ndim == 1:
yDim = varName
# find all datasets with lon/lat dimensions
for varName in f.variables:
var = f.variables[varName]
if xDim in var.dimensions and yDim in var.dimensions:
validVars.append(str(varName))
validDims += [str(dim) for dim in var.dimensions]
if len(validVars) > 1:
srcProjection = NSR().wkt
else:
# cancel usage of this mapper a input file seem unappropriate
raise
# get X/Y size
var0 = f.variables[validVars[0]]
xDimI = var0.dimensions.index(xDim)
srcRasterXSize = var0.shape[xDimI]
yDimI = var0.dimensions.index(yDim)
srcRasterYSize = var0.shape[yDimI]
print 'x/ySize', srcRasterXSize, srcRasterYSize
# get GDAL GeoTransform
xdata = f.variables[xDim][:]
ydata = f.variables[yDim][:]
x0 = xdata[0]
dx = xdata[1] - xdata[0]
y0 = ydata[0]
dy = ydata[1] - ydata[0]
srcGeoTransform = (x0, dx, 0, y0, 0, dy)
print srcGeoTransform
# make list of metadata dictionary entries
# for each var make [k][i][j][x][y] depending on order of dims
metaDict = []
for varName in validVars:
var = f.variables[varName]
dims = var.dimensions
# find nor X neither Y dimensions (e.g. time, depth, etc)
nonxyDims = []
nonxyDimValues = []
for dim in dims:
if dim != xDim and dim != yDim:
nonxyDims.append(dim)
dimValues = f.variables[dim][:]
dimValuesStr = [str(val) for val in dimValues]
nonxyDimValues.append(dimValues)
# calculate total dimensionality in addition to X/Y
# make vector of nonXY dimensions
nonXYShape = []
for nonxyDim in nonxyDims:
dimVar = f.variables[nonxyDim]
nonXYShape.append(dimVar.shape[0])
if len(nonXYShape) == 0:
nonXYShape = [1]
totNonXYDims = np.cumprod(nonXYShape)[-1]
urls = []
# generate bands for each additional dimension
for nonxyi in range(totNonXYDims):
dstVarName = str(varName)
url = fileName + '?%s.%s' % (varName, varName)
#assert varName != 'v'
# vector of nonX/Y indeces
iVec = np.unravel_index(nonxyi, nonXYShape)
# dim metadata keeps name of dimension and index
# in this dimension (value)
dimMetadata = {}
# add either [x], or [y], or respective index in each dimension
for dim in dims:
if dim == xDim:
url = url + '[x]'
elif dim == yDim:
url += '[y]'
else:
# get index of that dimension
dimN = nonxyDims.index(dim)
dimV = iVec[dimN]
url += '[%d]' % dimV
dimMetadata[dim] = nonxyDimValues[dimN][dimV]
dstVarName += '%03d' % dimV
# get band metadata
attrs = var.ncattrs()
metaEntry = {'src': {'SourceFilename': url, 'sourceBand': 1},
'dst': {'name': dstVarName}
}
# put band metadata
for attr in attrs:
attrKey = attr.encode('ascii', 'ignore')
attrVal = var.getncattr(attr)
if type(attrVal) in [str, unicode]:
attrVal = attrVal.encode('ascii', 'ignore')
else:
attrVal = str(attrVal)
metaEntry['dst'][attrKey] = attrVal
# add wkv
if 'standard_name' in attrs:
metaEntry['dst']['wkv'] = metaEntry['dst']['standard_name']
# add dim metadata (location within each dimension)
for dimKey in dimMetadata:
metaEntry['dst'][str(dimKey)] = dimMetadata[dimKey]
metaDict.append(metaEntry)
# read global metadata
srcMetadata = {}
for attr in f.ncattrs():
srcMetadata[str(attr)] = str(attr)
# create VRT with bands
VRT.__init__(self, srcGeoTransform=srcGeoTransform,
srcProjection=srcProjection,
srcRasterXSize=srcRasterXSize,
srcRasterYSize=srcRasterYSize,
srcMetadata=srcMetadata)
self._create_bands(metaDict)
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' Ocean Productivity website VRT '''
try:
assert 'IDL' in gdalMetadata['Projection Category']
assert '-9999' in gdalMetadata['Hole Value']
except:
raise WrongMapperError
print 'Ocean Productivity website data'
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
simFilesMask = os.path.join(iDir, '*' + iFileName[4:11] + iFileExt)
#print 'simFilesMask', simFilesMask
simFiles = glob.glob(simFilesMask)
#print 'simFiles', simFiles
metaDict = []
for simFile in simFiles:
#print 'simFile',simFile
# open subdataset with GDAL
tmpSourceFilename = simFile
tmpGdalDataset = gdal.Open(tmpSourceFilename)
# get metadata, get 'Parameter'
tmpGdalMetadata = tmpGdalDataset.GetMetadata()
iDir, ifileName = os.path.split(tmpSourceFilename)
#print 'ifileName',ifileName
simParameter = ifileName[0:3]
# set params of the similar file
simSourceFilename = tmpSourceFilename
simGdalDataset = tmpGdalDataset
simGdalMetadata = tmpGdalMetadata
# get WKV from the similar file
for param in self.param2wkv:
#print 'param', param
if param in simParameter:
simWKV = self.param2wkv[param]
break
#print 'simWKV', simWKV
# generate entry to metaDict
metaEntry = {
'src': {
'SourceFilename': simSourceFilename,
'SourceBand': 1,
'ScaleRatio': float(simGdalMetadata['Slope']),
'ScaleOffset': float(simGdalMetadata['Intercept'])
},
'dst': {
'wkv': simWKV,
'name': self.bandNames[simWKV],
'Parameter': simParameter
}
}
#print 'metaEntry', metaEntry
# append entry to metaDict
metaDict.append(metaEntry)
#get array with data and make 'mask'
a = simGdalDataset.ReadAsArray()
mask = np.zeros(a.shape, 'uint8') + 128
mask[a < -9990] = 1
self.bandVRTs = {'maskVRT': VRT(array=mask)}
metaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['maskVRT'].fileName),
'SourceBand': 1
},
'dst': {
'name': 'mask'
}
})
# create empty VRT dataset with geolocation only
# print 'simGdalMetadata', simGdalMetadata
latitudeStep = 0.08333334
longitudeStep = 0.08333334
numberOfColumns = 4320
numberOfLines = 2160
#longitudeStep = float(simGdalMetadata['Longitude Step'])
VRT.__init__(
self,
srcGeoTransform=(-180.0, longitudeStep, 0.0, 90.0, 0.0,
-longitudeStep),
srcProjection=
'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]',
srcRasterXSize=numberOfColumns,
srcRasterYSize=numberOfLines)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add valid time
startYear = int(iFile[4:8])
startDay = int(iFile[8:11])
self._set_time(
datetime.datetime(startYear, 1, 1) + datetime.timedelta(startDay))
def __init__(self, fileName, gdalDataset, gdalMetadata,
geolocation=False, zoomSize=500, step=1, **kwargs):
''' Create MER2 VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
geolocation : bool (default is False)
if True, add gdal geolocation
zoomSize: int (used in envisat.py)
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int (used in envisat.py)
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
'''
self.setup_ads_parameters(fileName, gdalMetadata)
if self.product[0:9] != "MER_FRS_2" and self.product[0:9] != "MER_RR__2":
raise WrongMapperError(fileName)
metaDict = [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '412'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 2},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '443'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 3},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '490'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 4},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '510'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 5},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '560'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 6},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '620'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 7},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '665'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 8},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '680'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 9},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '708'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 10},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '753'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 11},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '761'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 12},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '778'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 13},
'dst': {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air',
'wavelength': '864'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 15},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1_log', 'case': 'I'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 16},
'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 17},
'dst': {'wkv': 'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 18},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2_log', 'case': 'II'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 22},
'dst': {'wkv': 'quality_flags', 'suffix': 'l2'}}
]
# add 'name' to 'parameters'
for bandDict in metaDict:
if 'wavelength' in bandDict['dst']:
bandDict['dst']['suffix'] = bandDict['dst']['wavelength']
#get GADS from header
scales = self.read_scaling_gads(range(7, 20) + [20, 21, 22, 20])
offsets = self.read_scaling_gads(range(33, 46) + [46, 47, 48, 46])
# set scale/offset to the band metadata (only reflectance)
for i, bandDict in enumerate(metaDict[:-1]):
bandDict['src']['ScaleRatio'] = str(scales[i])
bandDict['src']['ScaleOffset'] = str(offsets[i])
# add log10-scaled variables
metaDict += [{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '1', 'case': 'I',
'expression': 'np.power(10., self["chlor_a_1_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'mass_concentration_of_chlorophyll_a_in_sea_water',
'suffix': '2', 'case': 'II',
'expression': 'np.power(10., self["chlor_a_2_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'volume_absorption_coefficient_of_radiative_flux_in_sea_water_due_to_dissolved_organic_matter',
'suffix': '2', 'case': 'II',
'expression': 'np.power(10., self["cdom_a_2_log"])'}},
{'src': {'SourceFilename': fileName, 'SourceBand': 1},
'dst': {'wkv': 'mass_concentration_of_suspended_matter_in_sea_water',
'suffix': '2', 'case': 'II',
'expression': 'np.power(10., self["tsm_2_log"])'}}
]
# get list with resized VRTs from ADS
self.bandVRTs = {'adsVRTs': self.get_ads_vrts(gdalDataset,
['sun zenith angles',
'sun azimuth angles',
'zonal winds',
'meridional winds'],
zoomSize=zoomSize,
step=step)}
# add bands from the ADS VRTs
for adsVRT in self.bandVRTs['adsVRTs']:
metaDict.append({'src': {'SourceFilename': adsVRT.fileName,
'SourceBand': 1},
'dst': {'name': (adsVRT.dataset.GetRasterBand(1).
GetMetadataItem('name')),
'units': (adsVRT.dataset.GetRasterBand(1).
GetMetadataItem('units'))}
})
# create empty VRT dataset with geolocation only
VRT.__init__(self, gdalDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# set time
self._set_envisat_time(gdalMetadata)
# add geolocation arrays
if geolocation:
self.add_geolocation_from_ads(gdalDataset,
zoomSize=zoomSize, step=step)
# set time
self._set_envisat_time(gdalMetadata)
self.dataset.SetMetadataItem('sensor', 'MERIS')
self.dataset.SetMetadataItem('satellite', 'ENVISAT')
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
GCP_COUNT=10,
**kwargs):
''' Create VRT
Parameters
----------
GCP_COUNT : int
number of GCPs along each dimention
'''
# extension must be .nc
if os.path.splitext(fileName)[1] != '.nc':
raise WrongMapperError
# file must contain navigation_data/longitude
try:
ds = gdal.Open('HDF5:"%s"://navigation_data/longitude' % fileName)
except RuntimeError:
raise WrongMapperError
else:
dsMetadata = ds.GetMetadata()
# title value must be known
if dsMetadata.get('title', '') not in self.titles:
raise WrongMapperError
# get geophysical data variables
subDatasets = gdal.Open(fileName).GetSubDatasets()
metaDict = []
for subDataset in subDatasets:
groupName = subDataset[0].split('/')[-2]
if groupName not in ['geophysical_data', 'navigation_data']:
continue
varName = subDataset[0].split('/')[-1]
subds = gdal.Open(subDataset[0])
b = subds.GetRasterBand(1)
bMetadata = b.GetMetadata()
# set SRC/DST parameters
metaEntry = {
'src': {
'SourceFilename': subDataset[0],
'sourceBand': 1,
'DataType': b.DataType
},
'dst': {
'name': varName
}
}
# replace datatype for l2_flags
if varName == 'l2_flags':
metaEntry['src']['DataType'] = 4
metaEntry['src']['SourceType'] = 'SimpleSource'
# set scale if exist
metaKey = '%s_%s_scale_factor' % (groupName, varName)
if metaKey in bMetadata:
metaEntry['src']['ScaleRatio'] = bMetadata[metaKey]
# set offset if exist
metaKey = '%s_%s_add_offset' % (groupName, varName)
if metaKey in bMetadata:
metaEntry['src']['ScaleOffset'] = bMetadata[metaKey]
# set standard_name if exists
metaKey = '%s_%s_standard_name' % (groupName, varName)
if metaKey in bMetadata:
metaEntry['dst']['wkv'] = bMetadata[metaKey]
# set other metadata
for metaKey in bMetadata:
newMetaKey = metaKey.replace('%s_%s_' % (groupName, varName),
'')
if newMetaKey not in [
'scale_factor', 'add_offset', 'DIMENSION_LIST',
'_FillValue'
]:
metaEntry['dst'][newMetaKey] = bMetadata[metaKey]
metaDict.append(metaEntry)
# make GCPs
# get lat/lon grids
longitude = gdal.Open('HDF5:"%s"://navigation_data/longitude' %
fileName).ReadAsArray()
latitude = gdal.Open('HDF5:"%s"://navigation_data/latitude' %
fileName).ReadAsArray()
rasterYSize, rasterXSize = longitude.shape
step0 = max(1, int(float(latitude.shape[0]) / GCP_COUNT))
step1 = max(1, int(float(latitude.shape[1]) / GCP_COUNT))
gcps = []
k = 0
center_lon = 0
center_lat = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 + 0.5, i0 + 0.5)
gcps.append(gcp)
center_lon += lon
center_lat += lat
k += 1
time_coverage_start = dsMetadata['time_coverage_start']
time_coverage_end = dsMetadata['time_coverage_end']
# create VRT
VRT.__init__(self,
srcProjection=NSR().wkt,
srcGCPs=gcps,
srcGCPProjection=NSR().wkt,
srcRasterXSize=rasterXSize,
srcRasterYSize=rasterYSize)
# add bands
self._create_bands(metaDict)
# reproject GCPs
center_lon /= k
center_lat /= k
srs = '+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=%f +lat_0=%f +no_defs' % (
center_lon, center_lat)
self.reproject_GCPs(srs)
### BAD, BAd, bad ...
self.dataset.SetProjection(self.dataset.GetGCPProjection())
# use TPS for reprojection
self.tps = True
# add NansenCloud metadata
self.dataset.SetMetadataItem('time_coverage_start',
str(time_coverage_start))
self.dataset.SetMetadataItem('time_coverage_end',
str(time_coverage_end))
self.dataset.SetMetadataItem('source_type', 'Satellite')
self.dataset.SetMetadataItem('mapper', 'obpg_l2_nc')
platform = {
'Orbview-2': 'SEASTAR'
}.get(dsMetadata.get('platform'), dsMetadata.get('platform'))
mm = pti.get_gcmd_instrument(dsMetadata.get('instrument'))
ee = pti.get_gcmd_platform(platform)
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
########################################
# Read metadata from binary file
########################################
try:
fp = open(fileName, 'rb')
except IOError:
raise WrongMapperError
fp.seek(24)
try:
satID = int(struct.unpack('<l', fp.read(4))[0])
except:
raise WrongMapperError
##################
# Read time
##################
fp.seek(44)
year = int(struct.unpack('<l', fp.read(4))[0])
dayofyear = int(struct.unpack('<l', fp.read(4))[0])
millisecondsOfDay = int(struct.unpack('<l', fp.read(4))[0])
try:
time = (datetime.datetime(year, 1, 1) +
datetime.timedelta(dayofyear - 1,
milliseconds=millisecondsOfDay))
except:
raise WrongMapperError
fp.seek(72)
numScanLines = int(struct.unpack('<l', fp.read(4))[0])
missingScanLines = int(struct.unpack('<l', fp.read(4))[0])
numCalibratedScanLines = int(struct.unpack('<l', fp.read(4))[0])
if missingScanLines != 0:
print('WARNING: Missing scanlines: ' + str(missingScanLines))
fp.seek(88)
dataFormatNum = int(struct.unpack('<l', fp.read(4))[0])
dataFormat = dataFormats[dataFormatNum]
# Determine if we have channel 3A (daytime) or channel 3B (nighttime)
def int2bitstring(s):
return str(s) if s <= 1 else int2bitstring(s >> 1) + str(s & 1)
fp.seek(headerLength + 20)
scanlinebitFirstline = int(struct.unpack('<L', fp.read(4))[0])
fp.seek(headerLength + recordLength*(numCalibratedScanLines-2) + 20)
scanlinebitLastline = int(struct.unpack('<L', fp.read(4))[0])
if int2bitstring(scanlinebitFirstline)[-1] == '0':
startsWith3A = True
else:
startsWith3A = False
if int2bitstring(scanlinebitLastline)[-1] == '0':
endsWith3A = True
else:
endsWith3A = False
if startsWith3A != endsWith3A:
print '############################################'
print 'WARNING: channel 3 switches '
print 'between daytime and nighttime (3A <-> 3B)'
print '###########################################'
###########################
# Make Geolocation Arrays
###########################
srcRasterYSize = numCalibratedScanLines
# Making VRT with raw (unscaled) lon and lat
# (smaller bands than full dataset)
self.bandVRTs = {'RawGeolocVRT': VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)}
RawGeolocMetaDict = []
for lonlatNo in range(1, 3):
RawGeolocMetaDict.append(
{'src': {'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'DataType': gdal.GDT_Int32,
'ImageOffset': (headerLength + 676 +
(lonlatNo - 1) * 4),
'PixelOffset': 8,
'LineOffset': recordLength,
'ByteOrder': 'LSB'},
'dst': {}})
self.bandVRTs['RawGeolocVRT']._create_bands(RawGeolocMetaDict)
# Make derived GeolocVRT with scaled lon and lat
self.bandVRTs['GeolocVRT'] = VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
GeolocMetaDict = []
for lonlatNo in range(1, 3):
GeolocMetaDict.append(
{'src': {'SourceFilename': (self.bandVRTs['RawGeolocVRT'].
fileName),
'SourceBand': lonlatNo,
'ScaleRatio': 0.0001,
'ScaleOffset': 0,
'DataType': gdal.GDT_Int32},
'dst': {}})
self.bandVRTs['GeolocVRT']._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(xVRT=self.bandVRTs['GeolocVRT'],
yVRT=self.bandVRTs['GeolocVRT'],
xBand=2, yBand=1, # x = lon, y = lat
lineOffset=0, pixelOffset=25,
lineStep=1, pixelStep=40)
#######################
# Initialize dataset
#######################
# create empty VRT dataset with geolocation only
# (from Geolocation Array)
VRT.__init__(self,
srcRasterXSize=2048,
srcRasterYSize=numCalibratedScanLines,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Since warping quality is horrible using geolocation arrays
# which are much smaller than raster bands (due to a bug in GDAL:
# http://trac.osgeo.org/gdal/ticket/4907), the geolocation arrays
# are here converted to GCPs. Only a subset of GCPs is added,
# significantly increasing speed when using -tps warping
reductionFactor = 2
self.convert_GeolocationArray2GPCs(1 * reductionFactor,
40 * reductionFactor)
##################
# Create bands
##################
self.bandVRTs['RawBandsVRT'] = VRT(
srcRasterXSize=2048,
srcRasterYSize=numCalibratedScanLines)
RawMetaDict = []
metaDict = []
centralWavelengths = [0.63, 0.86, np.NaN, 10.8, 12.0]
if startsWith3A:
centralWavelengths[2] = 1.6
firstIRband = 4
else:
centralWavelengths[2] = 3.7
firstIRband = 3
for bandNo in range(1, 6):
RawMetaDict.append(
{'src': {'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'dataType': gdal.GDT_UInt16,
'ImageOffset': imageOffset + (bandNo - 1) * 2,
'PixelOffset': 10,
'LineOffset': recordLength,
'ByteOrder': 'LSB'},
'dst': {'dataType': gdal.GDT_UInt16}})
if bandNo < firstIRband:
wkv = 'albedo'
minmax = '0 60'
else:
wkv = 'brightness_temperature'
minmax = '290 210'
metaDict.append(
{'src': {'SourceFilename': (self.bandVRTs['RawBandsVRT'].
fileName),
'SourceBand': bandNo,
'ScaleRatio': 0.01,
'ScaleOffset': 0,
'DataType': gdal.GDT_UInt16},
'dst': {'originalFilename': fileName,
'dataType': gdal.GDT_Float32,
'wkv': wkv,
'colormap': 'gray',
'wavelength': centralWavelengths[bandNo-1],
'minmax': minmax}})
# Add temperature difference between ch3 and ch 4 as pixelfunction
if not startsWith3A: # Only if ch3 is IR (nighttime)
metaDict.append(
{'src': [{'SourceFilename': (self.bandVRTs['RawBandsVRT'].
fileName),
'ScaleRatio': 0.01,
'ScaleOffset': 0,
'SourceBand': 4},
{'SourceFilename': (self.bandVRTs['RawBandsVRT'].
fileName),
'ScaleRatio': 0.01,
'ScaleOffset': 0,
'SourceBand': 3}],
'dst': {'PixelFunctionType': 'diff',
'originalFilename': fileName,
'dataType': gdal.GDT_Float32,
'name': 'ch4-ch3',
'short_name': 'ch4-ch3',
'long_name': 'AVHRR ch4 - ch3 temperature difference',
'colormap': 'gray',
'units': 'kelvin',
'minmax': '-3 3'}})
self.self.bandVRTs['RawBandsVRT']._create_bands(RawMetaDict)
self._create_bands(metaDict)
globalMetadata = {}
globalMetadata['satID'] = str(satID)
globalMetadata['daytime'] = str(int(startsWith3A))
self.dataset.SetMetadata(globalMetadata)
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start', time.isoformat())
self.dataset.SetMetadataItem('time_coverage_end', time.isoformat())
return
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
########################################
# Read metadata from binary file
########################################
try:
fp = open(fileName, 'rb')
except IOError:
raise WrongMapperError
fp.seek(72)
try:
satNum = int(struct.unpack('<H', fp.read(2))[0])
except:
raise WrongMapperError
if satNum >= 11:
isMetop = True
else:
isMetop = False
if satNum in satIDs.keys():
satID = satIDs[satNum]
else:
raise WrongMapperError
fp.seek(76)
dataFormatNum = int(struct.unpack('<H', fp.read(2))[0])
if dataFormatNum in dataFormats.keys():
dataFormat = dataFormats[dataFormatNum]
else:
raise WrongMapperError
fp.seek(dataSetQualityIndicatorOffset + 14)
numScanLines = int(struct.unpack('<H', fp.read(2))[0])
numCalibratedScanLines = int(struct.unpack('<H', fp.read(2))[0])
missingScanLines = int(struct.unpack('<H', fp.read(2))[0])
if missingScanLines != 0:
print('WARNING: Missing scanlines: ' + str(missingScanLines))
##################
# Read time
##################
fp.seek(84)
year = int(struct.unpack('<H', fp.read(2))[0])
dayofyear = int(struct.unpack('<H', fp.read(2))[0])
millisecondsOfDay = int(struct.unpack('<l', fp.read(4))[0])
time = datetime.datetime(year, 1, 1) + \
datetime.timedelta(dayofyear-1, milliseconds=millisecondsOfDay)
##################################
# Read calibration information
##################################
#IRcalibration = {}
#fp.seek(202)
#avh_h_irttcoef=[]
#for i in range(24):
# avh_h_irttcoef.append(int(struct.unpack('<H', fp.read(2))[0]))
##print avh_h_irttcoef
#avh_h_albcnv=[]
#for i in range(6):
# avh_h_albcnv.append(int(struct.unpack('<l', fp.read(4))[0]))
##print avh_h_albcnv
#fp.seek(280)
#avh_h_radtempcnv = np.zeros((3,3))
#for IRchannelNo in range(3):
# for coeffNo in range(3):
# avh_h_radtempcnv[IRchannelNo, coeffNo] = \
# int(struct.unpack('<l', fp.read(4))[0])
#print avh_h_radtempcnv
#IRcalibration['centralWavenumber'] = (avh_h_radtempcnv[:,0] /
# [1E2, 1E3, 1E3])
#IRcalibration['c1'] = avh_h_radtempcnv[:,1] / 1E5
#IRcalibration['c2'] = avh_h_radtempcnv[:,2] / 1E6
########################################################
# Read visible calibration coefficients per scanline
# - for channels 1, 2, 3A
########################################################
#for scanline in range(1):
# avh_calvis=np.zeros((3,3,5))
# fp.seek(headerLength + recordLength*scanline + 48)
# for VISchannel in range(3):
# for sets in range(3):
# for coeff in range(5):
# avh_calvis[sets, VISchannel, coeff] = \
# int(struct.unpack('<l', fp.read(4))[0])
# print avh_calvis
# print '----'
########################################################
# Read IR calibration coefficients per scanline
# - for channels 3B, 4, 5
########################################################
#for scanline in range(1):
# avh_calir=np.zeros((2,3,3))
# fp.seek(headerLength + recordLength*scanline + 228)
# for IRchannelNo in range(3):
# for setNo in range(2):
# for coeffNo in range(3):
# avh_calir[setNo, IRchannelNo, coeffNo] = \
# int(struct.unpack('<l', fp.read(4))[0])
#avh_filler2 = np.zeros(3)
#for fillerNo in range(3):
# avh_filler2[fillerNo] = int(struct.unpack('<l', fp.read(4))[0])
#setNo = 0 # Use operational set (the only available)
#a = np.zeros((3,3))
#for IRchannelNo in range(3):
# for coeffNo in range(3):
# # NB: apparently stored "backwards", therefore 2-coeffNo
# a[IRchannelNo, 2-coeffNo] = (avh_calir[setNo, IRchannelNo,
# coeffNo]
# / np.power(10,
# avh_filler2[coeffNo]))
#
###########################
# Apply calibration
###########################
#C = 410
#for IRchannelNo in range(3):
# Ne = a[IRchannelNo,0] + a[IRchannelNo,1]*C + a[IRchannelNo,2]*C*C
# vC = IRcalibration['centralWavenumber'][IRchannelNo]
# c1 = -IRcalibration['c1'][IRchannelNo] # Note minus
# c2 = IRcalibration['c2'][IRchannelNo]
#print '-----'
#print a[IRchannelNo,:]
#print vC, c1, c2
#TeStar = c2*vC/np.log(1 + (c1*vC*vC*vC)/Ne)
#Te = c1 + c2*TeStar
#print Ne, TeStar, Te
#print '-----'
#sys.exit('stop')
###########################
# Make Geolocation Arrays
###########################
srcRasterYSize = numCalibratedScanLines
# Making VRT with raw (unscaled) lon and lat
# (smaller bands than full dataset)
self.bandVRTs = {'RawGeolocVRT': VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)}
RawGeolocMetaDict = []
for lonlatNo in range(1, 3):
RawGeolocMetaDict.append(
{'src': {'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'DataType': gdal.GDT_Int32,
'ImageOffset': (headerLength + 640 +
(lonlatNo - 1) * 4),
'PixelOffset': 8,
'LineOffset': recordLength,
'ByteOrder': 'LSB'},
'dst': {}})
self.bandVRTs['RawGeolocVRT']._create_bands(RawGeolocMetaDict)
# Make derived GeolocVRT with scaled lon and lat
self.bandVRTs['GeolocVRT'] = VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
GeolocMetaDict = []
for lonlatNo in range(1, 3):
GeolocMetaDict.append(
{'src': {'SourceFilename': (self.bandVRTs['RawGeolocVRT'].
fileName),
'SourceBand': lonlatNo,
'ScaleRatio': 0.0001,
'ScaleOffset': 0,
'DataType': gdal.GDT_Int32},
'dst': {}})
self.bandVRTs['GeolocVRT']._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(xVRT=self.bandVRTs['GeolocVRT'],
yVRT=self.bandVRTs['GeolocVRT'],
xBand=2, yBand=1, # x = lon, y = lat
lineOffset=0, pixelOffset=25,
lineStep=1, pixelStep=40)
#######################
# Initialize dataset
#######################
# create empty VRT dataset with geolocation only
# (from Geolocation Array)
VRT.__init__(self,
srcRasterXSize=2048,
srcRasterYSize=numCalibratedScanLines,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Since warping quality is horrible using geolocation arrays
# which are much smaller than raster bands (due to a bug in GDAL:
# http://trac.osgeo.org/gdal/ticket/4907), the geolocation arrays
# are here converted to GCPs. Only a subset of GCPs is added,
# significantly increasing speed when using -tps warping
reductionFactor = 2
self.convert_GeolocationArray2GPCs(1*reductionFactor,
40*reductionFactor)
##################
# Create bands
##################
metaDict = []
ch = ({}, {}, {}, {}, {}, {})
ch[1]['wavelength'] = 0.63
ch[2]['wavelength'] = 0.86
ch[3]['wavelength'] = '1.6 or 3.7 mum'
ch[4]['wavelength'] = 10.8
ch[5]['wavelength'] = 12.0
ch[1]['minmax'] = '0 700'
ch[2]['minmax'] = '0 700'
ch[3]['minmax'] = '0 800'
ch[4]['minmax'] = '400 1000'
ch[5]['minmax'] = '400 1000'
for bandNo in range(1, 6):
metaDict.append({'src': {'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'dataType': gdal.GDT_UInt16,
'ImageOffset': imageOffset + (bandNo-1)*2,
'PixelOffset': 10,
'LineOffset': recordLength,
'ByteOrder': 'LSB'},
'dst': {'dataType': gdal.GDT_UInt16,
'wkv': 'raw_counts',
'colormap': 'gray',
'wavelength': ch[bandNo]['wavelength'],
'minmax': ch[bandNo]['minmax'],
'unit': "1"}})
self._create_bands(metaDict)
# Adding valid time to dataset
self._set_time(time)
return
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
''' OBPG L3 VRT '''
try:
assert 'Level-3 Standard Mapped Image' in gdalMetadata['Title']
except:
raise WrongMapperError
# get list of similar (same date) files in the directory
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
simFilesMask = os.path.join(iDir, iFileName)
simFiles = glob.glob(simFilesMask + iFileExt[0:6] + '*')
#print 'simFilesMask, simFiles', simFilesMask, simFiles
metaDict = []
for simFile in simFiles:
#print 'simFile', simFile
# open file, get metadata and get parameter name
simSupDataset = gdal.Open(simFile)
if simSupDataset is None:
# skip this similar file
#print 'No dataset: %s not a supported SMI file' % simFile
continue
# get subdatasets from the similar file
simSubDatasets = simSupDataset.GetSubDatasets()
if len(simSubDatasets) > 0:
for simSubDataset in simSubDatasets:
#print 'simSubDataset', simSubDataset
if 'l3m_data' in simSubDataset[1]:
# get SourceFilename from subdataset
tmpSourceFilename = simSubDataset[0]
break
else:
# get SourceFilename from dataset
tmpSourceFilename = simFile
# open subdataset with GDAL
#print 'tmpSourceFilename', tmpSourceFilename
tmpGdalDataset = gdal.Open(tmpSourceFilename)
try:
# get metadata, get 'Parameter'
tmpGdalMetadata = tmpGdalDataset.GetMetadata()
simParameter = tmpGdalMetadata['Parameter']
except:
print 'No parameter: %s not a supported SMI file' % simFile
continue
else:
# set params of the similar file
simSourceFilename = tmpSourceFilename
simGdalDataset = tmpGdalDataset
simGdalMetadata = tmpGdalMetadata
# get WKV from the similar file
#print 'simParameter', simParameter
for param in self.param2wkv:
#print 'param', param
if param in simParameter:
simWKV = self.param2wkv[param]
break
# generate entry to metaDict
metaEntry = {'src': {'SourceFilename': simSourceFilename,
'SourceBand': 1,
'ScaleRatio': float(simGdalMetadata['Slope']),
'ScaleOffset': float(simGdalMetadata['Intercept'])},
'dst': {'wkv': simWKV}}
# add wavelength and BandName
if ' at ' in simParameter and ' nm' in simParameter:
simWavelength = simParameter.split(' at ')[1].split(' nm')[0]
metaEntry['dst']['suffix'] = simWavelength
metaEntry['dst']['wavelength'] = simWavelength
# add band with Rrsw
metaEntry2 = None
if simWKV == 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_air':
metaEntry2 = {'src': [metaEntry['src']]}
metaEntry2['dst'] = {'wkv': 'surface_ratio_of_upwelling_radiance_emerging_from_sea_water_to_downwelling_radiative_flux_in_water',
'suffix': simWavelength,
'wavelength': simWavelength,
'PixelFunctionType': 'NormReflectanceToRemSensReflectance',
}
# append entry to metaDict
metaDict.append(metaEntry)
if metaEntry2 is not None:
metaDict.append(metaEntry2)
#get array with data and make 'mask'
a = simGdalDataset.ReadAsArray()
mask = np.zeros(a.shape, 'uint8') + 64
mask[a < -32000] = 1
self.bandVRTs = {'mask': VRT(array=mask)}
metaDict.append(
{'src': {'SourceFilename': self.bandVRTs['mask'].fileName,
'SourceBand': 1},
'dst': {'name': 'mask'}})
# create empty VRT dataset with geolocation only
# print 'simGdalMetadata', simGdalMetadata
latitudeStep = float(simGdalMetadata.
get('Latitude Step',
simGdalMetadata.get('Latitude_Step', 1)))
longitudeStep = float(simGdalMetadata.
get('Longitude Step',
simGdalMetadata.get('Longitude_Step', 1)))
numberOfColumns = int(simGdalMetadata.
get('Number of Columns',
simGdalMetadata.get('Number_of_Columns', 1)))
numberOfLines = int(simGdalMetadata.
get('Number of Lines',
simGdalMetadata.get('Number_of_Lines', 1)))
#longitudeStep = float(simGdalMetadata['Longitude Step'])
VRT.__init__(self,
srcGeoTransform=(-180.0, longitudeStep, 0.0,
90.0, 0.0, -longitudeStep),
srcProjection=NSR().wkt,
srcRasterXSize=numberOfColumns,
srcRasterYSize=numberOfLines)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Add valid time
startYear = int(simGdalMetadata.get('Start Year',
simGdalMetadata.
get('Start_Year', 1)))
startDay = int(simGdalMetadata.get('Start Day',
simGdalMetadata.
get('Start)Day', 1)))
self.dataset.SetMetadataItem('time_coverage_start',
(datetime.datetime(startYear, 1, 1) +
datetime.timedelta(startDay)).isoformat())
def __init__(self, fileName, gdalDataset, gdalMetadata,
GCP_COUNT=10,
bandNames=['VNIR_Band1', 'VNIR_Band2', 'VNIR_Band3N'],
bandWaves=[560, 660, 820], **kwargs):
''' Create VRT
Parameters
-----------
GCP_COUNT : int
number of GCPs along each dimention
bandNames : list of string (band name)
bandWaves : list of integer (waves corresponding to band name)
Band name and waves
--------------------
'VNIR_Band3B' : 820, 'SWIR_Band4' : 1650, 'SWIR_Band5' : 2165,
'SWIR_Band6' : 2205, 'SWIR_Band7' : 2260, 'SWIR_Band8' : 2330,
'SWIR_Band9' : 2395, 'TIR_Band10' : 8300, 'TIR_Band11' : 8650,
'TIR_Band12' : 9100, 'TIR_Band13' : 10600, 'TIR_Band14' : 11300
'''
# check if it is ASTER L1A
try:
assert 'AST_L1A_' in fileName
shortName = gdalMetadata['INSTRUMENTSHORTNAME']
assert shortName == 'ASTER'
except:
raise WrongMapperError
subDatasets = gdalDataset.GetSubDatasets()
# find datasets for each band and generate metaDict
metaDict = []
bandDatasetMask = 'HDF4_EOS:EOS_SWATH:"%s":%s:ImageData'
for bandName, bandWave in zip(bandNames, bandWaves):
metaEntry = {'src': {'SourceFilename': (bandDatasetMask
% (fileName, bandName)),
'SourceBand': 1,
'DataType': 6,
},
'dst': {'wkv': 'toa_outgoing_spectral_radiance',
'wavelength': str(bandWave),
'suffix': str(bandWave),
}}
metaDict.append(metaEntry)
# create empty VRT dataset with geolocation only
gdalSubDataset = gdal.Open(metaDict[0]['src']['SourceFilename'])
VRT.__init__(self, gdalSubDataset)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# find largest lon/lat subdatasets
latShape0 = 0
for subDataset in subDatasets:
if 'Latitude' in subDataset[1]:
ls = int(subDataset[1].strip().split('[')[1].split('x')[0])
if ls >= latShape0:
latShape0 = ls
latSubDS = subDataset[0]
if 'Longitude' in subDataset[1]:
ls = int(subDataset[1].strip().split('[')[1].split('x')[0])
if ls >= latShape0:
latShape0 = ls
lonSubDS = subDataset[0]
self.logger.debug(latSubDS)
self.logger.debug(lonSubDS)
# get lat/lon matrices
xDataset = gdal.Open(lonSubDS)
yDataset = gdal.Open(latSubDS)
longitude = xDataset.ReadAsArray()
latitude = yDataset.ReadAsArray()
step0 = longitude.shape[0] / GCP_COUNT
step1 = longitude.shape[1] / GCP_COUNT
# estimate pixel/line step
pixelStep = int(ceil(float(gdalSubDataset.RasterXSize) /
float(xDataset.RasterXSize)))
lineStep = int(ceil(float(gdalSubDataset.RasterYSize) /
float(xDataset.RasterYSize)))
self.logger.debug('steps: %d %d %d %d' % (step0, step1,
pixelStep, lineStep))
# generate list of GCPs
gcps = []
k = 0
for i0 in range(0, latitude.shape[0], step0):
for i1 in range(0, latitude.shape[1], step1):
# create GCP with X,Y,pixel,line from lat/lon matrices
lon = float(longitude[i0, i1])
lat = float(latitude[i0, i1])
if (lon >= -180 and lon <= 180 and lat >= -90 and lat <= 90):
gcp = gdal.GCP(lon, lat, 0, i1 * pixelStep, i0 * lineStep)
self.logger.debug('%d %d %d %f %f' % (k, gcp.GCPPixel,
gcp.GCPLine,
gcp.GCPX, gcp.GCPY))
gcps.append(gcp)
k += 1
# append GCPs and lat/lon projection to the vsiDataset
self.dataset.SetGCPs(gcps, NSR().wkt)
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start',
parse(gdalMetadata['FIRSTPACKETTIME']).isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
parse(gdalMetadata['LASTPACKETTIME']).isoformat())
mm = pti.get_gcmd_instrument('ASTER')
ee = pti.get_gcmd_platform('TERRA')
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
