def _create_empty(self, gdal_dataset, gdal_metadata):
subfiles = self.sub_filenames(gdal_dataset)
sub0 = gdal.Open(subfiles[0])
super(Mapper, self).__init__(gdalDataset=sub0,
srcMetadata=gdal_metadata)
if not (self.dataset.GetGeoTransform() or self.geolocationArray.xVRT):
# Set geolocation array from bands
self.add_geolocationArray(
GeolocationArray([
s for s in subfiles
if 'longitude' in s or 'GEOLOCATION_X_DATASET' in s
][0], [
s for s in subfiles
if 'latitude' in s or 'GEOLOCATION_Y_DATASET' in s
][0]))
if not self.get_projection():
# Get band projections
projections = [
gdal.Open(sub).GetProjection() for sub in subfiles
if gdal.Open(sub).GetProjection()
]
if not projections:
raise WrongMapperError
# Check that projection is the same for all bands
assert all(proj == projections[0] for proj in projections)
# Set projection
self.dataset.SetProjection(projections[0])
def add_geolocation_from_ads(self, gdalDataset, zoomSize=500, step=1):
''' Add geolocation domain metadata to the dataset
Get VRTs with zoomed arrays of lon and lat
Create geolocation object and add to the metadata
Parameters
----------
gdalDataset: GDAL Dataset
input dataset
prodType: str
'ASA_' or 'MER_'
zoomSize: int, optional, 500
size, to which the ADS array will be zoomed using scipy
array of this size will be stored in memory
step: int
step of pixel and line in GeolocationArrays. lat/lon grids are
generated at that step
Modifies:
---------
Adds Geolocation Array metadata
'''
# get VRTs with lon and lat
xyVRTs = self.get_ads_vrts(gdalDataset, self.lonlatNames, zoomSize,
step)
# Add geolocation domain metadata to the dataset
self.add_geolocationArray(
GeolocationArray(xVRT=xyVRTs[0],
yVRT=xyVRTs[1],
xBand=1,
yBand=1,
srs=gdalDataset.GetGCPProjection(),
lineOffset=0,
lineStep=step,
pixelOffset=0,
pixelStep=step))
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
latlonGrid=None,
mask='',
**kwargs):
''' Create VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is ASCAT
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
try:
assert iFileName[0:6] == 'ascat_' and iFileExt == '.nc'
except:
raise WrongMapperError
# Create geolocation
subDataset = gdal.Open('NETCDF:"' + fileName + '":lat')
self.GeolocVRT = VRT(srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize)
GeolocMetaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lon'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': -360
},
'dst': {}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lat'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': 0
},
'dst': {}
}]
self.GeolocVRT._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.GeolocVRT,
yVRT=self.GeolocVRT,
# x = lon, y = lat
xBand=1,
yBand=2,
lineOffset=0,
pixelOffset=0,
lineStep=1,
pixelStep=1)
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
gdalDataset=subDataset,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Scale and NODATA should ideally be taken directly from raw file
metaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_speed'),
'ScaleRatio': 0.01,
'NODATA': -32767
},
'dst': {
'name': 'wind_speed',
'wkv': 'wind_speed'
}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_dir'),
'ScaleRatio': 0.1,
'NODATA': -32767
},
'dst': {
'name': 'wind_direction',
'wkv': 'wind_direction'
}
}]
self._create_bands(metaDict)
# This should not be necessary
# - should be provided by GeolocationArray!
self.dataset.SetProjection(GeolocObject.d['SRS'])
# Add time
startTime = datetime.datetime(int(iFileName[6:10]),
int(iFileName[10:12]),
int(iFileName[12:14]),
int(iFileName[15:17]),
int(iFileName[17:19]),
int(iFileName[19:21]))
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start',
startTime.isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
startTime.isoformat())
# set SADCAT specific metadata
self.dataset.SetMetadataItem('sensor', 'ASCAT')
self.dataset.SetMetadataItem('satellite', 'Metop-A')
warnings.warn("Setting satellite to Metop-A - update mapper if it is" \
" e.g. Metop-B")
self.dataset.SetMetadataItem('mapper', 'ascat_nasa')
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
########################################
# Read metadata from binary file
########################################
try:
fp = open(fileName, 'rb')
except IOError:
raise WrongMapperError
fp.seek(24)
try:
satID = int(struct.unpack('<l', fp.read(4))[0])
except:
raise WrongMapperError
##################
# Read time
##################
fp.seek(44)
year = int(struct.unpack('<l', fp.read(4))[0])
dayofyear = int(struct.unpack('<l', fp.read(4))[0])
millisecondsOfDay = int(struct.unpack('<l', fp.read(4))[0])
try:
time = (datetime.datetime(year, 1, 1) + datetime.timedelta(
dayofyear - 1, milliseconds=millisecondsOfDay))
except:
raise WrongMapperError
fp.seek(72)
numScanLines = int(struct.unpack('<l', fp.read(4))[0])
missingScanLines = int(struct.unpack('<l', fp.read(4))[0])
numCalibratedScanLines = int(struct.unpack('<l', fp.read(4))[0])
if missingScanLines != 0:
print('WARNING: Missing scanlines: ' + str(missingScanLines))
fp.seek(88)
dataFormatNum = int(struct.unpack('<l', fp.read(4))[0])
dataFormat = dataFormats[dataFormatNum]
# Determine if we have channel 3A (daytime) or channel 3B (nighttime)
def int2bitstring(s):
return str(s) if s <= 1 else int2bitstring(s >> 1) + str(s & 1)
fp.seek(headerLength + 20)
scanlinebitFirstline = int(struct.unpack('<L', fp.read(4))[0])
fp.seek(headerLength + recordLength * (numCalibratedScanLines - 2) +
20)
scanlinebitLastline = int(struct.unpack('<L', fp.read(4))[0])
if int2bitstring(scanlinebitFirstline)[-1] == '0':
startsWith3A = True
else:
startsWith3A = False
if int2bitstring(scanlinebitLastline)[-1] == '0':
endsWith3A = True
else:
endsWith3A = False
if startsWith3A != endsWith3A:
print '############################################'
print 'WARNING: channel 3 switches '
print 'between daytime and nighttime (3A <-> 3B)'
print '###########################################'
###########################
# Make Geolocation Arrays
###########################
srcRasterYSize = numCalibratedScanLines
# Making VRT with raw (unscaled) lon and lat
# (smaller bands than full dataset)
self.bandVRTs = {
'RawGeolocVRT': VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
}
RawGeolocMetaDict = []
for lonlatNo in range(1, 3):
RawGeolocMetaDict.append({
'src': {
'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'DataType': gdal.GDT_Int32,
'ImageOffset': (headerLength + 676 + (lonlatNo - 1) * 4),
'PixelOffset': 8,
'LineOffset': recordLength,
'ByteOrder': 'LSB'
},
'dst': {}
})
self.bandVRTs['RawGeolocVRT']._create_bands(RawGeolocMetaDict)
# Make derived GeolocVRT with scaled lon and lat
self.bandVRTs['GeolocVRT'] = VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
GeolocMetaDict = []
for lonlatNo in range(1, 3):
GeolocMetaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['RawGeolocVRT'].fileName),
'SourceBand': lonlatNo,
'ScaleRatio': 0.0001,
'ScaleOffset': 0,
'DataType': gdal.GDT_Int32
},
'dst': {}
})
self.bandVRTs['GeolocVRT']._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.bandVRTs['GeolocVRT'],
yVRT=self.bandVRTs['GeolocVRT'],
xBand=2,
yBand=1, # x = lon, y = lat
lineOffset=0,
pixelOffset=25,
lineStep=1,
pixelStep=40)
#######################
# Initialize dataset
#######################
# create empty VRT dataset with geolocation only
# (from Geolocation Array)
VRT.__init__(self,
srcRasterXSize=2048,
srcRasterYSize=numCalibratedScanLines,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Since warping quality is horrible using geolocation arrays
# which are much smaller than raster bands (due to a bug in GDAL:
# http://trac.osgeo.org/gdal/ticket/4907), the geolocation arrays
# are here converted to GCPs. Only a subset of GCPs is added,
# significantly increasing speed when using -tps warping
reductionFactor = 2
self.convert_GeolocationArray2GPCs(1 * reductionFactor,
40 * reductionFactor)
##################
# Create bands
##################
self.bandVRTs['RawBandsVRT'] = VRT(
srcRasterXSize=2048, srcRasterYSize=numCalibratedScanLines)
RawMetaDict = []
metaDict = []
centralWavelengths = [0.63, 0.86, np.NaN, 10.8, 12.0]
if startsWith3A:
centralWavelengths[2] = 1.6
firstIRband = 4
else:
centralWavelengths[2] = 3.7
firstIRband = 3
for bandNo in range(1, 6):
RawMetaDict.append({
'src': {
'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'dataType': gdal.GDT_UInt16,
'ImageOffset': imageOffset + (bandNo - 1) * 2,
'PixelOffset': 10,
'LineOffset': recordLength,
'ByteOrder': 'LSB'
},
'dst': {
'dataType': gdal.GDT_UInt16
}
})
if bandNo < firstIRband:
wkv = 'albedo'
minmax = '0 60'
else:
wkv = 'brightness_temperature'
minmax = '290 210'
metaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['RawBandsVRT'].fileName),
'SourceBand': bandNo,
'ScaleRatio': 0.01,
'ScaleOffset': 0,
'DataType': gdal.GDT_UInt16
},
'dst': {
'originalFilename': fileName,
'dataType': gdal.GDT_Float32,
'wkv': wkv,
'colormap': 'gray',
'wavelength': centralWavelengths[bandNo - 1],
'minmax': minmax
}
})
# Add temperature difference between ch3 and ch 4 as pixelfunction
if not startsWith3A: # Only if ch3 is IR (nighttime)
metaDict.append({
'src': [{
'SourceFilename': (self.bandVRTs['RawBandsVRT'].fileName),
'ScaleRatio':
0.01,
'ScaleOffset':
0,
'SourceBand':
4
}, {
'SourceFilename': (self.bandVRTs['RawBandsVRT'].fileName),
'ScaleRatio':
0.01,
'ScaleOffset':
0,
'SourceBand':
3
}],
'dst': {
'PixelFunctionType': 'diff',
'originalFilename': fileName,
'dataType': gdal.GDT_Float32,
'name': 'ch4-ch3',
'short_name': 'ch4-ch3',
'long_name': 'AVHRR ch4 - ch3 temperature difference',
'colormap': 'gray',
'units': 'kelvin',
'minmax': '-3 3'
}
})
self.self.bandVRTs['RawBandsVRT']._create_bands(RawMetaDict)
self._create_bands(metaDict)
globalMetadata = {}
globalMetadata['satID'] = str(satID)
globalMetadata['daytime'] = str(int(startsWith3A))
self.dataset.SetMetadata(globalMetadata)
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start', time.isoformat())
self.dataset.SetMetadataItem('time_coverage_end', time.isoformat())
return
def __init__(self,
fileName,
gdalDataset,
gdalMetadata,
latlonGrid=None,
mask='',
**kwargs):
''' Create VRT
Parameters
-----------
fileName : string
gdalDataset : gdal dataset
gdalMetadata : gdal metadata
latlonGrid : numpy 2 layered 2D array with lat/lons of desired grid
'''
# test if input files is ASCAT
iDir, iFile = os.path.split(fileName)
iFileName, iFileExt = os.path.splitext(iFile)
try:
assert iFileName[0:6] == 'ascat_' and iFileExt == '.nc'
except:
raise WrongMapperError
# Create geolocation
subDataset = gdal.Open('NETCDF:"' + fileName + '":lat')
self.GeolocVRT = VRT(srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize)
GeolocMetaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lon'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': -360
},
'dst': {}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":lat'),
'SourceBand': 1,
'ScaleRatio': 0.00001,
'ScaleOffset': 0
},
'dst': {}
}]
self.GeolocVRT._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.GeolocVRT,
yVRT=self.GeolocVRT,
# x = lon, y = lat
xBand=1,
yBand=2,
lineOffset=0,
pixelOffset=0,
lineStep=1,
pixelStep=1)
# create empty VRT dataset with geolocation only
VRT.__init__(self,
srcRasterXSize=subDataset.RasterXSize,
srcRasterYSize=subDataset.RasterYSize,
gdalDataset=subDataset,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Scale and NODATA should ideally be taken directly from raw file
metaDict = [{
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_speed'),
'ScaleRatio': 0.01,
'NODATA': -32767
},
'dst': {
'name': 'windspeed',
'wkv': 'wind_speed'
}
}, {
'src': {
'SourceFilename': ('NETCDF:"' + fileName + '":wind_dir'),
'ScaleRatio': 0.1,
'NODATA': -32767
},
'dst': {
'name': 'winddirection',
'wkv': 'wind_from_direction'
}
}]
self._create_bands(metaDict)
# This should not be necessary
# - should be provided by GeolocationArray!
self.dataset.SetProjection(GeolocObject.d['SRS'])
# Add time
startTime = datetime.datetime(int(iFileName[6:10]),
int(iFileName[10:12]),
int(iFileName[12:14]),
int(iFileName[15:17]),
int(iFileName[17:19]),
int(iFileName[19:21]))
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start',
startTime.isoformat())
self.dataset.SetMetadataItem('time_coverage_end',
startTime.isoformat())
# Get dictionary describing the instrument and platform according to
# the GCMD keywords
mm = pti.get_gcmd_instrument('ascat')
ee = pti.get_gcmd_platform('metop-a')
# TODO: Validate that the found instrument and platform are indeed what
# we want....
self.dataset.SetMetadataItem('instrument', json.dumps(mm))
self.dataset.SetMetadataItem('platform', json.dumps(ee))
def __init__(self, fileName, gdalDataset, gdalMetadata, **kwargs):
########################################
# Read metadata from binary file
########################################
try:
fp = open(fileName, 'rb')
except IOError:
raise WrongMapperError
fp.seek(72)
try:
satNum = int(struct.unpack('<H', fp.read(2))[0])
except:
raise WrongMapperError
if satNum >= 11:
isMetop = True
else:
isMetop = False
if satNum in satIDs.keys():
satID = satIDs[satNum]
else:
raise WrongMapperError
fp.seek(76)
dataFormatNum = int(struct.unpack('<H', fp.read(2))[0])
if dataFormatNum in dataFormats.keys():
dataFormat = dataFormats[dataFormatNum]
else:
raise WrongMapperError
fp.seek(dataSetQualityIndicatorOffset + 14)
numScanLines = int(struct.unpack('<H', fp.read(2))[0])
numCalibratedScanLines = int(struct.unpack('<H', fp.read(2))[0])
missingScanLines = int(struct.unpack('<H', fp.read(2))[0])
if missingScanLines != 0:
print('WARNING: Missing scanlines: ' + str(missingScanLines))
##################
# Read time
##################
fp.seek(84)
year = int(struct.unpack('<H', fp.read(2))[0])
dayofyear = int(struct.unpack('<H', fp.read(2))[0])
millisecondsOfDay = int(struct.unpack('<l', fp.read(4))[0])
time = (
datetime.datetime(year, 1, 1) +
datetime.timedelta(dayofyear - 1, milliseconds=millisecondsOfDay))
##################################
# Read calibration information
##################################
#IRcalibration = {}
#fp.seek(202)
#avh_h_irttcoef=[]
#for i in range(24):
# avh_h_irttcoef.append(int(struct.unpack('<H', fp.read(2))[0]))
##print avh_h_irttcoef
#avh_h_albcnv=[]
#for i in range(6):
# avh_h_albcnv.append(int(struct.unpack('<l', fp.read(4))[0]))
##print avh_h_albcnv
#fp.seek(280)
#avh_h_radtempcnv = np.zeros((3,3))
#for IRchannelNo in range(3):
# for coeffNo in range(3):
# avh_h_radtempcnv[IRchannelNo, coeffNo] = \
# int(struct.unpack('<l', fp.read(4))[0])
#print avh_h_radtempcnv
#IRcalibration['centralWavenumber'] = (avh_h_radtempcnv[:,0] /
# [1E2, 1E3, 1E3])
#IRcalibration['c1'] = avh_h_radtempcnv[:,1] / 1E5
#IRcalibration['c2'] = avh_h_radtempcnv[:,2] / 1E6
########################################################
# Read visible calibration coefficients per scanline
# - for channels 1, 2, 3A
########################################################
#for scanline in range(1):
# avh_calvis=np.zeros((3,3,5))
# fp.seek(headerLength + recordLength*scanline + 48)
# for VISchannel in range(3):
# for sets in range(3):
# for coeff in range(5):
# avh_calvis[sets, VISchannel, coeff] = \
# int(struct.unpack('<l', fp.read(4))[0])
# print avh_calvis
# print '----'
########################################################
# Read IR calibration coefficients per scanline
# - for channels 3B, 4, 5
########################################################
#for scanline in range(1):
# avh_calir=np.zeros((2,3,3))
# fp.seek(headerLength + recordLength*scanline + 228)
# for IRchannelNo in range(3):
# for setNo in range(2):
# for coeffNo in range(3):
# avh_calir[setNo, IRchannelNo, coeffNo] = \
# int(struct.unpack('<l', fp.read(4))[0])
#avh_filler2 = np.zeros(3)
#for fillerNo in range(3):
# avh_filler2[fillerNo] = int(struct.unpack('<l', fp.read(4))[0])
#setNo = 0 # Use operational set (the only available)
#a = np.zeros((3,3))
#for IRchannelNo in range(3):
# for coeffNo in range(3):
# # NB: apparently stored "backwards", therefore 2-coeffNo
# a[IRchannelNo, 2-coeffNo] = (avh_calir[setNo, IRchannelNo,
# coeffNo]
# / np.power(10,
# avh_filler2[coeffNo]))
#
###########################
# Apply calibration
###########################
#C = 410
#for IRchannelNo in range(3):
# Ne = a[IRchannelNo,0] + a[IRchannelNo,1]*C + a[IRchannelNo,2]*C*C
# vC = IRcalibration['centralWavenumber'][IRchannelNo]
# c1 = -IRcalibration['c1'][IRchannelNo] # Note minus
# c2 = IRcalibration['c2'][IRchannelNo]
#print '-----'
#print a[IRchannelNo,:]
#print vC, c1, c2
#TeStar = c2*vC/np.log(1 + (c1*vC*vC*vC)/Ne)
#Te = c1 + c2*TeStar
#print Ne, TeStar, Te
#print '-----'
#sys.exit('stop')
###########################
# Make Geolocation Arrays
###########################
srcRasterYSize = numCalibratedScanLines
# Making VRT with raw (unscaled) lon and lat
# (smaller bands than full dataset)
self.bandVRTs = {
'RawGeolocVRT': VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
}
RawGeolocMetaDict = []
for lonlatNo in range(1, 3):
RawGeolocMetaDict.append({
'src': {
'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'DataType': gdal.GDT_Int32,
'ImageOffset': (headerLength + 640 + (lonlatNo - 1) * 4),
'PixelOffset': 8,
'LineOffset': recordLength,
'ByteOrder': 'LSB'
},
'dst': {}
})
self.bandVRTs['RawGeolocVRT']._create_bands(RawGeolocMetaDict)
# Make derived GeolocVRT with scaled lon and lat
self.bandVRTs['GeolocVRT'] = VRT(srcRasterXSize=51,
srcRasterYSize=srcRasterYSize)
GeolocMetaDict = []
for lonlatNo in range(1, 3):
GeolocMetaDict.append({
'src': {
'SourceFilename': (self.bandVRTs['RawGeolocVRT'].fileName),
'SourceBand': lonlatNo,
'ScaleRatio': 0.0001,
'ScaleOffset': 0,
'DataType': gdal.GDT_Int32
},
'dst': {}
})
self.bandVRTs['GeolocVRT']._create_bands(GeolocMetaDict)
GeolocObject = GeolocationArray(
xVRT=self.bandVRTs['GeolocVRT'],
yVRT=self.bandVRTs['GeolocVRT'],
xBand=2,
yBand=1, # x = lon, y = lat
lineOffset=0,
pixelOffset=25,
lineStep=1,
pixelStep=40)
#######################
# Initialize dataset
#######################
# create empty VRT dataset with geolocation only
# (from Geolocation Array)
VRT.__init__(self,
srcRasterXSize=2048,
srcRasterYSize=numCalibratedScanLines,
geolocationArray=GeolocObject,
srcProjection=GeolocObject.d['SRS'])
# Since warping quality is horrible using geolocation arrays
# which are much smaller than raster bands (due to a bug in GDAL:
# http://trac.osgeo.org/gdal/ticket/4907), the geolocation arrays
# are here converted to GCPs. Only a subset of GCPs is added,
# significantly increasing speed when using -tps warping
reductionFactor = 2
self.convert_GeolocationArray2GPCs(1 * reductionFactor,
40 * reductionFactor)
##################
# Create bands
##################
metaDict = []
ch = ({}, {}, {}, {}, {}, {})
ch[1]['wavelength'] = 0.63
ch[2]['wavelength'] = 0.86
ch[3]['wavelength'] = '1.6 or 3.7 mum'
ch[4]['wavelength'] = 10.8
ch[5]['wavelength'] = 12.0
ch[1]['minmax'] = '0 700'
ch[2]['minmax'] = '0 700'
ch[3]['minmax'] = '0 800'
ch[4]['minmax'] = '400 1000'
ch[5]['minmax'] = '400 1000'
for bandNo in range(1, 6):
metaDict.append({
'src': {
'SourceFilename': fileName,
'SourceBand': 0,
'SourceType': "RawRasterBand",
'dataType': gdal.GDT_UInt16,
'ImageOffset': imageOffset + (bandNo - 1) * 2,
'PixelOffset': 10,
'LineOffset': recordLength,
'ByteOrder': 'LSB'
},
'dst': {
'dataType': gdal.GDT_UInt16,
'wkv': 'raw_counts',
'colormap': 'gray',
'wavelength': ch[bandNo]['wavelength'],
'minmax': ch[bandNo]['minmax'],
'unit': "1"
}
})
self._create_bands(metaDict)
# Adding valid time to dataset
self.dataset.SetMetadataItem('time_coverage_start', time.isoformat())
self.dataset.SetMetadataItem('time_coverage_end', time.isoformat())
return
def __init__(self,
inputFileName,
gdalDataset,
gdalMetadata,
logLevel=30,
rmMetadatas=[
'NETCDF_VARNAME', '_Unsigned', 'ScaleRatio',
'ScaleOffset', 'dods_variable'
],
**kwargs):
# Remove 'NC_GLOBAL#' and 'GDAL_' and 'NANSAT_'
# from keys in gdalDataset
tmpGdalMetadata = {}
geoMetadata = {}
origin_is_nansat = False
if not gdalMetadata:
raise WrongMapperError
for key in gdalMetadata.keys():
newKey = key.replace('NC_GLOBAL#', '').replace('GDAL_', '')
if 'NANSAT_' in newKey:
geoMetadata[newKey.replace('NANSAT_', '')] = gdalMetadata[key]
origin_is_nansat = True
else:
tmpGdalMetadata[newKey] = gdalMetadata[key]
gdalMetadata = tmpGdalMetadata
fileExt = os.path.splitext(inputFileName)[1]
# Get file names from dataset or subdataset
subDatasets = gdalDataset.GetSubDatasets()
if len(subDatasets) == 0:
fileNames = [inputFileName]
else:
fileNames = [f[0] for f in subDatasets]
# add bands with metadata and corresponding values to the empty VRT
metaDict = []
xDatasetSource = ''
yDatasetSource = ''
firstXSize = 0
firstYSize = 0
for _, fileName in enumerate(fileNames):
subDataset = gdal.Open(fileName)
# choose the first dataset whith grid
if (firstXSize == 0 and firstYSize == 0
and subDataset.RasterXSize > 1
and subDataset.RasterYSize > 1):
firstXSize = subDataset.RasterXSize
firstYSize = subDataset.RasterYSize
firstSubDataset = subDataset
# get projection from the first subDataset
projection = firstSubDataset.GetProjection()
# take bands whose sizes are same as the first band.
if (subDataset.RasterXSize == firstXSize
and subDataset.RasterYSize == firstYSize):
if projection == '':
projection = subDataset.GetProjection()
if ('GEOLOCATION_X_DATASET' in fileName
or 'longitude' in fileName):
xDatasetSource = fileName
elif ('GEOLOCATION_Y_DATASET' in fileName
or 'latitude' in fileName):
yDatasetSource = fileName
else:
for iBand in range(subDataset.RasterCount):
subBand = subDataset.GetRasterBand(iBand + 1)
bandMetadata = subBand.GetMetadata_Dict()
if 'PixelFunctionType' in bandMetadata:
bandMetadata.pop('PixelFunctionType')
sourceBands = iBand + 1
# sourceBands = i*subDataset.RasterCount + iBand + 1
# generate src metadata
src = {
'SourceFilename': fileName,
'SourceBand': sourceBands
}
# set scale ratio and scale offset
scaleRatio = bandMetadata.get(
'ScaleRatio',
bandMetadata.get(
'scale', bandMetadata.get('scale_factor', '')))
if len(scaleRatio) > 0:
src['ScaleRatio'] = scaleRatio
scaleOffset = bandMetadata.get(
'ScaleOffset',
bandMetadata.get(
'offset', bandMetadata.get('add_offset', '')))
if len(scaleOffset) > 0:
src['ScaleOffset'] = scaleOffset
# sate DataType
src['DataType'] = subBand.DataType
# generate dst metadata
# get all metadata from input band
dst = bandMetadata
# set wkv and bandname
dst['wkv'] = bandMetadata.get('standard_name', '')
# first, try the name metadata
if 'name' in bandMetadata:
bandName = bandMetadata['name']
else:
# if it doesn't exist get name from NETCDF_VARNAME
bandName = bandMetadata.get('NETCDF_VARNAME', '')
if len(bandName) == 0:
bandName = bandMetadata.get(
'dods_variable', '')
# remove digits added by gdal in
# exporting to netcdf...
if (len(bandName) > 0 and origin_is_nansat
and fileExt == '.nc'):
if bandName[-1:].isdigit():
bandName = bandName[:-1]
if bandName[-1:].isdigit():
bandName = bandName[:-1]
# if still no bandname, create one
if len(bandName) == 0:
bandName = 'band_%03d' % iBand
dst['name'] = bandName
# remove non-necessary metadata from dst
for rmMetadata in rmMetadatas:
if rmMetadata in dst:
dst.pop(rmMetadata)
# append band with src and dst dictionaries
metaDict.append({'src': src, 'dst': dst})
# create empty VRT dataset with geolocation only
VRT.__init__(self, firstSubDataset, srcMetadata=gdalMetadata)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Create complex data bands from 'xxx_real' and 'xxx_imag' bands
# using pixelfunctions
rmBands = []
for iBandNo in range(self.dataset.RasterCount):
iBand = self.dataset.GetRasterBand(iBandNo + 1)
iBandName = iBand.GetMetadataItem('name')
# find real data band
if iBandName.find("_real") != -1:
realBandNo = iBandNo
realBand = self.dataset.GetRasterBand(realBandNo + 1)
realDtype = realBand.GetMetadataItem('DataType')
bandName = iBandName.replace(iBandName.split('_')[-1],
'')[0:-1]
for jBandNo in range(self.dataset.RasterCount):
jBand = self.dataset.GetRasterBand(jBandNo + 1)
jBandName = jBand.GetMetadataItem('name')
# find an imaginary data band corresponding to the real
# data band and create complex data band from the bands
if jBandName.find(bandName + '_imag') != -1:
imagBandNo = jBandNo
imagBand = self.dataset.GetRasterBand(imagBandNo + 1)
imagDtype = imagBand.GetMetadataItem('DataType')
dst = imagBand.GetMetadata()
dst['name'] = bandName
dst['PixelFunctionType'] = 'ComplexData'
dst['dataType'] = 10
src = [{
'SourceFilename': fileNames[realBandNo],
'SourceBand': 1,
'DataType': realDtype
}, {
'SourceFilename': fileNames[imagBandNo],
'SourceBand': 1,
'DataType': imagDtype
}]
self._create_band(src, dst)
self.dataset.FlushCache()
rmBands.append(realBandNo + 1)
rmBands.append(imagBandNo + 1)
# Delete real and imaginary bands
if len(rmBands) != 0:
self.delete_bands(rmBands)
if len(projection) == 0:
# projection was not set automatically
# get projection from GCPProjection
projection = geoMetadata.get('GCPProjection', '')
if len(projection) == 0:
# no projection was found in dataset or metadata:
# generate WGS84 by default
projection = NSR().wkt
# fix problem with MET.NO files where a, b given in m and XC/YC in km
if ('UNIT["kilometre"' in projection
and ',SPHEROID["Spheroid",6378273,7.331926543631893e-12]'
in projection):
projection = projection.replace(
',SPHEROID["Spheroid",6378273,7.331926543631893e-12]', '')
# set projection
self.dataset.SetProjection(self.repare_projection(projection))
# check if GCPs were added from input dataset
gcps = firstSubDataset.GetGCPs()
gcpProjection = firstSubDataset.GetGCPProjection()
# if no GCPs in input dataset: try to add GCPs from metadata
if not gcps:
gcps = self.add_gcps_from_metadata(geoMetadata)
# if yet no GCPs: try to add GCPs from variables
if not gcps:
gcps = self.add_gcps_from_variables(inputFileName)
if gcps:
if len(gcpProjection) == 0:
# get GCP projection and repare
gcpProjection = self.repare_projection(
geoMetadata.get('GCPProjection', ''))
# add GCPs to dataset
self.dataset.SetGCPs(gcps, gcpProjection)
self.dataset.SetProjection('')
self._remove_geotransform()
# Find proper bands and insert GEOLOCATION ARRAY into dataset
if len(xDatasetSource) > 0 and len(yDatasetSource) > 0:
self.add_geolocationArray(
GeolocationArray(xDatasetSource, yDatasetSource))
elif not gcps:
# if no GCPs found and not GEOLOCATION ARRAY set:
# Set Nansat Geotransform if it is not set automatically
geoTransform = self.dataset.GetGeoTransform()
if len(geoTransform) == 0:
geoTransformStr = geoMetadata.get('GeoTransform',
'(0|1|0|0|0|0|1)')
geoTransform = eval(geoTransformStr.replace('|', ','))
self.dataset.SetGeoTransform(geoTransform)
subMetadata = firstSubDataset.GetMetadata()
### GET START TIME from METADATA
time_coverage_start = None
if 'start_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['start_time'])
elif 'start_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['start_date'])
elif 'time_coverage_start' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_start'])
### GET END TIME from METADATA
time_coverage_end = None
if 'stop_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['stop_time'])
elif 'stop_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['stop_date'])
elif 'time_coverage_stop' in gdalMetadata:
time_coverage_start = parse_time(
gdalMetadata['time_coverage_stop'])
elif 'end_time' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['end_time'])
elif 'end_date' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['end_date'])
elif 'time_coverage_end' in gdalMetadata:
time_coverage_start = parse_time(gdalMetadata['time_coverage_end'])
### GET start time from time variable
if (time_coverage_start is None and cfunitsInstalled
and 'time#standard_name' in subMetadata
and subMetadata['time#standard_name'] == 'time'
and 'time#units' in subMetadata
and 'time#calendar' in subMetadata):
# get data from netcdf data
ncFile = netcdf_file(inputFileName, 'r')
timeLength = ncFile.variables['time'].shape[0]
timeValueStart = ncFile.variables['time'][0]
timeValueEnd = ncFile.variables['time'][-1]
ncFile.close()
try:
timeDeltaStart = Units.conform(
timeValueStart,
Units(subMetadata['time#units'],
calendar=subMetadata['time#calendar']),
Units('days since 1950-01-01'))
except ValueError:
self.logger.error('calendar units are wrong: %s' %
subMetadata['time#calendar'])
else:
time_coverage_start = (
datetime.datetime(1950, 1, 1) +
datetime.timedelta(float(timeDeltaStart)))
if timeLength > 1:
timeDeltaEnd = Units.conform(
timeValueStart,
Units(subMetadata['time#units'],
calendar=subMetadata['time#calendar']),
Units('days since 1950-01-01'))
else:
timeDeltaEnd = timeDeltaStart + 1
time_coverage_end = (datetime.datetime(1950, 1, 1) +
datetime.timedelta(float(timeDeltaEnd)))
## finally set values of time_coverage start and end if available
if time_coverage_start is not None:
self.dataset.SetMetadataItem('time_coverage_start',
time_coverage_start.isoformat())
if time_coverage_end is not None:
self.dataset.SetMetadataItem('time_coverage_end',
time_coverage_end.isoformat())
if 'sensor' not in gdalMetadata:
self.dataset.SetMetadataItem('sensor', 'unknown')
if 'satellite' not in gdalMetadata:
self.dataset.SetMetadataItem('satellite', 'unknown')
if 'source_type' not in gdalMetadata:
self.dataset.SetMetadataItem('source_type', 'unknown')
if 'platform' not in gdalMetadata:
self.dataset.SetMetadataItem('platform', 'unknown')
if 'instrument' not in gdalMetadata:
self.dataset.SetMetadataItem('instrument', 'unknown')
self.logger.info('Use generic mapper - OK!')
def __init__(self, inputFileName, gdalDataset, gdalMetadata, logLevel=30,
rmMetadatas=['NETCDF_VARNAME', '_Unsigned',
'ScaleRatio', 'ScaleOffset', 'dods_variable'],
**kwargs):
# Remove 'NC_GLOBAL#' and 'GDAL_' and 'NANSAT_'
# from keys in gdalDataset
tmpGdalMetadata = {}
geoMetadata = {}
origin_is_nansat = False
if not gdalMetadata:
raise WrongMapperError
for key in gdalMetadata.keys():
newKey = key.replace('NC_GLOBAL#', '').replace('GDAL_', '')
if 'NANSAT_' in newKey:
geoMetadata[newKey.replace('NANSAT_', '')] = gdalMetadata[key]
origin_is_nansat = True
else:
tmpGdalMetadata[newKey] = gdalMetadata[key]
gdalMetadata = tmpGdalMetadata
fileExt = os.path.splitext(inputFileName)[1]
# Get file names from dataset or subdataset
subDatasets = gdalDataset.GetSubDatasets()
if len(subDatasets) == 0:
fileNames = [inputFileName]
else:
fileNames = [f[0] for f in subDatasets]
# add bands with metadata and corresponding values to the empty VRT
metaDict = []
geoFileDict = {}
xDatasetSource = ''
yDatasetSource = ''
firstXSize = 0
firstYSize = 0
for i, fileName in enumerate(fileNames):
subDataset = gdal.Open(fileName)
# choose the first dataset whith grid
if (firstXSize == 0 and firstYSize == 0 and
subDataset.RasterXSize > 1 and subDataset.RasterYSize > 1):
firstXSize = subDataset.RasterXSize
firstYSize = subDataset.RasterYSize
firstSubDataset = subDataset
# get projection from the first subDataset
projection = firstSubDataset.GetProjection()
# take bands whose sizes are same as the first band.
if (subDataset.RasterXSize == firstXSize and
subDataset.RasterYSize == firstYSize):
if projection == '':
projection = subDataset.GetProjection()
if ('GEOLOCATION_X_DATASET' in fileName or
'longitude' in fileName):
xDatasetSource = fileName
elif ('GEOLOCATION_Y_DATASET' in fileName or
'latitude' in fileName):
yDatasetSource = fileName
else:
for iBand in range(subDataset.RasterCount):
subBand = subDataset.GetRasterBand(iBand+1)
bandMetadata = subBand.GetMetadata_Dict()
if 'PixelFunctionType' in bandMetadata:
bandMetadata.pop('PixelFunctionType')
sourceBands = iBand + 1
#sourceBands = i*subDataset.RasterCount + iBand + 1
# generate src metadata
src = {'SourceFilename': fileName,
'SourceBand': sourceBands}
# set scale ratio and scale offset
scaleRatio = bandMetadata.get(
'ScaleRatio',
bandMetadata.get(
'scale',
bandMetadata.get('scale_factor', '')))
if len(scaleRatio) > 0:
src['ScaleRatio'] = scaleRatio
scaleOffset = bandMetadata.get(
'ScaleOffset',
bandMetadata.get(
'offset',
bandMetadata.get(
'add_offset', '')))
if len(scaleOffset) > 0:
src['ScaleOffset'] = scaleOffset
# sate DataType
src['DataType'] = subBand.DataType
# generate dst metadata
# get all metadata from input band
dst = bandMetadata
# set wkv and bandname
dst['wkv'] = bandMetadata.get('standard_name', '')
# first, try the name metadata
bandName = bandMetadata.get('name', '')
# if it doesn't exist get name from NETCDF_VARNAME
if len(bandName) == 0:
bandName = bandMetadata.get('NETCDF_VARNAME', '')
if len(bandName) == 0:
bandName = bandMetadata.get('dods_variable',
'')
if len(bandName) > 0:
if origin_is_nansat and fileExt == '.nc':
# remove digits added by gdal in
# exporting to netcdf...
if bandName[-1:].isdigit():
bandName = bandName[:-1]
if bandName[-1:].isdigit():
bandName = bandName[:-1]
dst['name'] = bandName
# remove non-necessary metadata from dst
for rmMetadata in rmMetadatas:
if rmMetadata in dst:
dst.pop(rmMetadata)
# append band with src and dst dictionaries
metaDict.append({'src': src, 'dst': dst})
# create empty VRT dataset with geolocation only
VRT.__init__(self, firstSubDataset, srcMetadata=gdalMetadata)
# add bands with metadata and corresponding values to the empty VRT
self._create_bands(metaDict)
# Create complex data bands from 'xxx_real' and 'xxx_imag' bands
# using pixelfunctions
rmBands = []
for iBandNo in range(self.dataset.RasterCount):
iBand = self.dataset.GetRasterBand(iBandNo + 1)
iBandName = iBand.GetMetadataItem('name')
# find real data band
if iBandName.find("_real") != -1:
realBandNo = iBandNo
realBand = self.dataset.GetRasterBand(realBandNo + 1)
realDtype = realBand.GetMetadataItem('DataType')
bandName = iBandName.replace(iBandName.split('_')[-1],
'')[0:-1]
for jBandNo in range(self.dataset.RasterCount):
jBand = self.dataset.GetRasterBand(jBandNo + 1)
jBandName = jBand.GetMetadataItem('name')
# find an imaginary data band corresponding to the real
# data band and create complex data band from the bands
if jBandName.find(bandName+'_imag') != -1:
imagBandNo = jBandNo
imagBand = self.dataset.GetRasterBand(imagBandNo + 1)
imagDtype = imagBand.GetMetadataItem('DataType')
dst = imagBand.GetMetadata()
dst['name'] = bandName
dst['PixelFunctionType'] = 'ComplexData'
dst['dataType'] = 10
src = [{'SourceFilename': fileNames[realBandNo],
'SourceBand': 1,
'DataType': realDtype},
{'SourceFilename': fileNames[imagBandNo],
'SourceBand': 1,
'DataType': imagDtype}]
self._create_band(src, dst)
self.dataset.FlushCache()
rmBands.append(realBandNo + 1)
rmBands.append(imagBandNo + 1)
# Delete real and imaginary bands
if len(rmBands) != 0:
self.delete_bands(rmBands)
if len(projection) == 0:
# projection was not set automatically
# get projection from GCPProjection
projection = geoMetadata.get('GCPProjection', '')
if len(projection) == 0:
# no projection was found in dataset or metadata:
# generate WGS84 by default
projection = NSR().wkt
# set projection
self.dataset.SetProjection(self.repare_projection(projection))
# check if GCPs were added from input dataset
gcps = firstSubDataset.GetGCPs()
# if no GCPs in input dataset: try to add GCPs from metadata
if not gcps:
gcps = self.add_gcps_from_metadata(geoMetadata)
# if yet no GCPs: try to add GCPs from variables
if not gcps:
gcps = self.add_gcps_from_variables(inputFileName)
if gcps:
# get GCP projection and repare
projection = self.repare_projection(geoMetadata.
get('GCPProjection', ''))
# add GCPs to dataset
self.dataset.SetGCPs(gcps, projection)
self._remove_geotransform()
# Find proper bands and insert GEOLOCATION ARRAY into dataset
if len(xDatasetSource) > 0 and len(yDatasetSource) > 0:
self.add_geolocationArray(GeolocationArray(xDatasetSource,
yDatasetSource))
elif not gcps:
# if no GCPs found and not GEOLOCATION ARRAY set:
# Set Nansat Geotransform if it is not set automatically
geoTransform = self.dataset.GetGeoTransform()
if len(geoTransform) == 0:
geoTransformStr = geoMetadata.get('GeoTransform',
'(0|1|0|0|0|0|1)')
geoTransform = eval(geoTransformStr.replace('|', ','))
self.dataset.SetGeoTransform(geoTransform)
if 'start_date' in gdalMetadata:
try:
startDate = parse(gdalMetadata['start_date'])
except ValueError:
self.logger.error('Time format is wrong in input file!')
else:
self._set_time(startDate)
self.logger.warning('Use generic mapper - OK!')