def modis_sst(infileid,
outdir,
download_dir='/tmp',
vmin=None,
vmax=None,
contrast='relative',
ngcps=(21, 25),
resample_radius=5000.,
resample_sigma=2500.,
denoise_kernel='boxcar',
denoise_width=20,
open_iterations=1,
nprocs=1,
pngkml=False,
write_netcdf=False,
file_range=None):
"""
"""
dic = None
# Check file containing ranges
if file_range is not None:
if not os.path.isfile(file_range):
raise Exception('file_range {} not found'.format(file_range))
# Read a txt file which contains three columns: yearday,vmin,vmax
with open(file_range, 'r') as f:
dic = {}
for line in f:
(fdoy, fmin, fmax) = line.split(',')
dic[int(fdoy)] = (float(fmin), float(fmax))
# modissstfname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/A2011338122500.L2_LAC_SST'
# modis02fname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/MYD021KM.A2011338.1225.005.2011339235825.hdf'
# modis03fname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/MYD03.A2011338.1225.005.2011339233301.hdf'
# modis35l2fname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/MYD35_L2.A2011338.1225.005.2011340001234.hdf'
if contrast == 'med':
listbox = [[-6., 35., 2.75, 42.48], [2.74, 30, 42.2, 47.00]]
elif contrast == 'cwe':
listbox = [[-23., 35.2, -5.5, 42.88], [-23., 42.8, 2.20, 51.]]
elif contrast == 'nwe':
listbox = [[-23., 50.8, 32.7, 68.]]
elif contrast == 'gom':
listbox = [[-98., 18.0, -80.5, 30.5]]
elif contrast == 'agulhas':
listbox = [[10.8437, -45.7404, 39.9799, -25.3019]]
elif contrast == 'gs':
listbox = [[-81.52, 20, -30, 45]]
else:
listbox = None
# Search/Download data
print 'Search/Download data'
if re.match(r'^[AT][0-9]{13}$', infileid) is None:
raise Exception('Input for modis_sst is an ID '
'(e.g. A2011338122500 or T2014143234500)')
platform = infileid[0]
date = datetime.strptime(infileid[1:], '%Y%j%H%M%S')
modissstid = {'A': 'MODISAL2SST', 'T': 'MODISTL2SST'}[platform]
modissstfname = modis.search_and_download(modissstid, date, download_dir)
modis02id = {'A': 'MYD021KM', 'T': 'MOD021KM'}[platform]
modis02fname = modis.search_and_download(modis02id, date, download_dir)
modis03id = {'A': 'MYD03', 'T': 'MOD03'}[platform]
modis03fname = modis.search_and_download(modis03id, date, download_dir)
modis35l2id = {'A': 'MYD35_L2', 'T': 'MOD35_L2'}[platform]
modis35l2fname = modis.search_and_download(modis35l2id, date, download_dir)
# Read/Process data
print 'Read/Process data'
# Read from SST file
modissstfile = modis.MODISL2File(modissstfname)
# lon = modissstfile.read_lon()
# lat = modissstfile.read_lat()
sst = modissstfile.read_sst() + 273.15
attrs = modissstfile.read_attributes()
modissstfile.close()
# Read from radiances file
modis02file = modis.MODIS02File(modis02fname)
rad11 = modis02file.read_radiance(31)
modis02file.close()
bt11 = modis.modis_bright(rad11, 31, 1)
# Read from geolocation file
modis03file = modis.MODIS03File(modis03fname)
lon = modis03file.read_lon()
lat = modis03file.read_lat()
modis03file.close()
# Read from cloud mask file
modis35l2file = modis.MODIS35L2File(modis35l2fname)
cloudmask = modis35l2file.read_cloudmask(byte=0)
modis35l2file.close()
cloudy = (np.bitwise_and(cloudmask, 2) == 0) & \
(np.bitwise_and(cloudmask, 4) == 0)
land = np.bitwise_and(cloudmask, 128) == 128 # Desert or Land
# land = (np.bitwise_and(cloudmask, 128) == 128) | \
# (np.bitwise_and(cloudmask, 64) == 64) # Desert or Land or Coastal
if listbox is not None:
mask_box = np.zeros(np.shape(sst))
for i in range(np.shape(listbox)[0]):
index_in = np.where((lon >= listbox[i][0]) & (lat >= listbox[i][1])
& (lon <= listbox[i][2])
& (lat <= listbox[i][3]))
mask_box[index_in] = 1
mask = cloudy | land | ma.getmaskarray(sst) | ma.getmaskarray(bt11) | (
mask_box == 0)
else:
mask = cloudy | land | ma.getmaskarray(sst) | ma.getmaskarray(bt11)
if mask.all():
print 'No data'
sys.exit(0)
# GCPs for resampling and geotiff georeference
scansize = 10
dtime0 = datetime.utcnow()
gcps = resample.get_gcps_from_bowtie(lon, lat, scansize, ngcps=ngcps)
#gcps = resample.get_gcps_from_bowtie_old(lon, lat, scansize, ngcps=ngcps)
dtime = datetime.utcnow() - dtime0
print 'Get GCPs from bowtie swath : {}'.format(dtime)
gcplon, gcplat, gcpnpixel, gcpnline = gcps
rspysize = lon.shape[0]
geod = pyproj.Geod(ellps='WGS84')
mid = abs(gcpnline[:, 0] - 0.5).argmin()
xdists = geod.inv(gcplon[mid, :-1], gcplat[mid, :-1], gcplon[mid, 1:],
gcplat[mid, 1:])[2]
xdist = np.sum(xdists) / abs(gcpnpixel[mid, -1] - gcpnpixel[mid, 0])
rspxsize = np.round(xdist / 1000.).astype('int') + 1
gcpline = gcpnline * rspysize
gcppixel = gcpnpixel * rspxsize
# Resample with LinearNDInterpolator in output space
dtime0 = datetime.utcnow()
pix, lin = resample.get_points_from_gcps(gcplon,
gcplat,
gcppixel,
gcpline,
rspxsize,
rspysize,
1,
lon,
lat,
nprocs=nprocs) - 0.5
dtime = datetime.utcnow() - dtime0
print 'Get input coordinates in new grid : {}'.format(dtime)
# Test input grid in output space
# import matplotlib.pyplot as plt
# for iscan in range(lon.shape[0] / scansize):
# pixscan = pix[iscan * scansize: (iscan+1) * scansize, :]
# linscan = lin[iscan * scansize: (iscan+1) * scansize, :]
# # maskscan = mask[iscan * scansize: (iscan+1) * scansize, :]
# # pixscan = pixscan[~maskscan]
# # linscan = linscan[~maskscan]
# plt.plot(pixscan.flatten(), linscan.flatten(), '+')
# plt.show()
# import pdb ; pdb.set_trace()
# \Test input grid in output space
dtime0 = datetime.utcnow()
sst.data[mask] = np.nan
bt11.data[mask] = np.nan
val = np.dstack((sst.data, bt11.data))
rspval = resample.resample_bowtie_linear(pix,
lin,
val,
scansize,
rspxsize,
rspysize,
show=False)
rspsst = rspval[:, :, 0]
rspbt11 = rspval[:, :, 1]
rspmask = ma.getmaskarray(rspsst) | ma.getmaskarray(rspbt11)
dtime = datetime.utcnow() - dtime0
print 'Interpolate in new grid : {}'.format(dtime)
# Resample with pyresample in lon/lat space
# rsplin, rsppix = np.mgrid[0:rspysize, 0:rspxsize] + 0.5
# rsplon, rsplat = resample.get_points_from_gcps(gcplon, gcplat, gcppixel,
# gcpline, rspxsize, rspysize,
# 0, rsppix, rsplin, nprocs=nprocs)
# # Test resample grid
# import matplotlib.pyplot as plt
# plt.plot(lon.flatten(), lat.flatten(), '+b')
# plt.plot(rsplon.flatten(), rsplat.flatten(), '+g')
# plt.plot(gcplon.flatten(), gcplat.flatten(), 'xr')
# plt.show()
# import pdb ; pdb.set_trace()
# # \Test resample grid
# # Test radius / sigma
# resample_radius = 5000.
# resample_sigma = 2500.
# sst.mask = False
# #sst.mask = sst.mask | (sst.data < 273.15+5) | (sst.data > 273.15+30)
# rspsst = resample.resample_gauss(lon, lat, sst, rsplon, rsplat,
# resample_radius, resample_sigma,
# nprocs=nprocs, show=True)
# import pdb ; pdb.set_trace()
# # \Test radius / sigma
# valid = np.where(mask == False)
# rspsst = resample.resample_gauss(lon[valid], lat[valid], sst[valid],
# rsplon, rsplat,
# resample_radius, resample_sigma,
# fill_value=None, nprocs=nprocs,
# show=False)
# rspbt11 = resample.resample_gauss(lon[valid], lat[valid], bt11[valid],
# rsplon, rsplat,
# resample_radius, resample_sigma,
# fill_value=None, nprocs=nprocs,
# show=False)
# rspmask = resample.resample_nearest(lon, lat, mask,
# rsplon, rsplat,
# resample_radius,
# fill_value=True, nprocs=nprocs,
# show=False)
# rspmask = rspmask | ma.getmaskarray(rspsst) | ma.getmaskarray(rspbt11)
# Denoise sst and open mask
rspsst.mask = rspmask
rspbt11.mask = rspmask
finalsst = denoise_sst(rspsst,
rspbt11,
kernel=denoise_kernel,
width=denoise_width,
show=False)
#finalsst = rspsst
finalmask = ~binary_opening(
~rspmask, structure=np.ones((3, 3)), iterations=open_iterations)
#finalmask = rspmask
finalsst.mask = finalmask
# Contrast
if vmin == None:
if contrast == 'relative':
vmin = np.percentile(finalsst.compressed(), 0.5)
#elif contrast == 'agulhas':
# dayofyear = float(attrs['start_time'].timetuple().tm_yday)
# vmin = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. - 9.
# #par = [277.94999694824219, 42, 2.5500030517578125, -219]
# par = [278.09999084472656, 0.62831853071795862,
# 2.4000091552734375, 0.1570796326794896]
# vmin = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific txt file is provided for the range
elif dic is not None:
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
# Read a txt file which contains three columns: yearday,vmin,vmax
extrema = dic.get(
dayofyear, dic[min(dic.keys(),
key=lambda k: abs(k - dayofyear))])
vmin = extrema[0]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
if vmax == None:
if contrast == 'relative':
vmax = np.percentile(finalsst.compressed(), 99.5)
#elif contrast == 'agulhas':
# dayofyear = float(attrs['start_time'].timetuple().tm_yday)
# vmax = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. + 4.
# #par = [300.59999084472656, 21, 2.8499908447265625, -191]
# par = [300.59999084472656, 0.29919930034188508,
# 2.8499908447265625, 0.14959965017094254]
# vmax = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific text file is provided for the range
elif dic is not None:
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
extrema = dic.get(
dayofyear, dic[min(dic.keys(),
key=lambda k: abs(k - dayofyear))])
vmax = extrema[1]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
# Flip (geotiff in "swath sense")
finalsst = finalsst[::-1, ::-1]
gcppixel = rspxsize - gcppixel
gcpline = rspysize - gcpline
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(attrs['start_time'],
attrs['stop_time'],
units='ms')
metadata['product_name'] = 'SST_MODIS_denoised'
if contrast == 'relative':
metadata['name'] = os.path.splitext(os.path.basename(modissstfname))[0]
else:
metadata['name'] = '{}_{}'.format(
os.path.splitext(os.path.basename(modissstfname))[0], contrast)
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = [
modissstfname, modis02fname, modis03fname, modis35l2fname
]
metadata['source_provider'] = 'NASA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface temperature'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'radiometer'
metadata['sensor_name'] = 'MODIS'
metadata['sensor_platform'] = attrs['platform']
metadata['sensor_pass'] = attrs['pass']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
gcpheight = np.zeros(gcppixel.shape)
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcpheight,
gcppixel, gcpline)
band = []
indndv = np.where(ma.getmaskarray(finalsst) == True)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(finalsst.data, vmin, vmax, out=finalsst.data)
array = np.round((finalsst.data - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=vmax,
vmax_pal=vmax,
vmin=vmin,
vmin_pal=vmin)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'sea surface temperature',
'unittype': 'K',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'denoised_sst'
band[0]['long_name'] = 'denoised sea surface temperature'
band[0]['standard_name'] = 'sea_surface_temperature'
# ymid = abs(gcpline[:, 0] - rspysize / 2.).argmin()
# xdists = geod.inv(gcplon[ymid, :-1], gcplat[ymid, :-1],
# gcplon[ymid, 1:], gcplat[ymid, 1:])[2] / \
# np.abs(gcppixel[ymid, 1:] - gcppixel[ymid, :-1])
# xmid = abs(gcppixel[0, :] - rspxsize / 2.).argmin()
# ydists = geod.inv(gcplon[:-1, xmid], gcplat[:-1, xmid],
# gcplon[1:, xmid], gcplat[1:, xmid])[2] / \
# np.abs(gcpline[1:, xmid] - gcpline[:-1, xmid])
# print xdists.min(), xdists.max(), xdists.mean()
# # e.g. 999.763079208 999.763084628 999.763082543
# print ydists.min(), ydists.max(), ydists.mean()
# # e.g. 1006.4149472 1008.60679776 1007.5888004
metadata['spatial_resolution'] = 1000.
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def viirs_sst(infile, outdir, vmin=None, vmax=None, contrast='relative',
ngcps=(41, 32), denoise_kernel='boxcar', denoise_width=27,
open_iterations=1, nprocs=1,
pngkml=False, write_netcdf=False, file_range=None):
"""
"""
dic = None
# Check file containing ranges
if file_range is not None:
if not os.path.isfile(file_range):
raise Exception('file_range {} not found'.format(file_range))
# Read a txt file which contains three columns: yearday,vmin,vmax
with open(file_range, 'r') as f:
dic = {}
for line in f:
(fdoy, fmin, fmax) = line.split(',')
dic[int(fdoy)] = (float(fmin), float(fmax))
if contrast == 'med':
listbox = [[-6., 35., 2.75, 42.48],
[2.74, 30, 42.2, 47.00]]
elif contrast == 'cwe':
listbox = [[-23., 35.2, -5.5, 42.88],
[-23., 42.8, 2.20, 51.]]
elif contrast == 'nwe':
listbox = [[-23., 50.8, 32.7, 68.]]
elif contrast == 'gom':
listbox = [[-98., 18.0, -80.5, 30.5]]
elif contrast == 'agulhas':
listbox = [[10.8437, -45.7404, 39.9799, -25.3019]]
elif contrast == 'gs':
listbox = [[-81.52, 20, -30, 45]]
else:
listbox = None
# Read/Process data
print 'Read/Process data'
dataset = Dataset(infile)
start_time = datetime.strptime(dataset.start_time, '%Y%m%dT%H%M%SZ')
print start_time.day
print start_time.month
stop_time = datetime.strptime(dataset.stop_time, '%Y%m%dT%H%M%SZ')
lon = dataset.variables['lon'][:, :]
lat = dataset.variables['lat'][:, :]
sst = np.ma.array(dataset.variables['sea_surface_temperature'][0, :, :])
_bt11= dataset.variables['brightness_temperature_11um'][0, :, :]
bt11 = np.ma.array(_bt11)
quality_level = np.ma.array(dataset.variables['quality_level'][0, :, :])
'''
if file_shape is not None:
with open(file_shape, 'r') as fshape:
shape = shapely.wkt.load(fshape)
box = shape.bounds
index_in = np.where((lon >= box[0]) & (lat >= box[1])
& (lon <= box[2]) & (lat <= box[3]))
index_out = np.where((lon < box[0]) | (lat < box[1])
| (lon > box[2]) | (lat > box[3]))
sst[index_out] = np.nan
print(np.shape(index_in))
sys.exit(1)
for i, j in zip(index_in[0], index_in[1]):
p = Point(lon[i, j], lat[i, j])
if p.within(shape) is False:
sst[i, j] = np.nan
'''
if listbox is not None:
mask_box = np.zeros(np.shape(sst))
for i in range(np.shape(listbox)[0]):
index_in = np.where((lon >= listbox[i][0]) & (lat >= listbox[i][1])
& (lon <= listbox[i][2]) & (lat <= listbox[i][3]))
mask_box[index_in] = 1
mask = ma.getmaskarray(sst) | ma.getmaskarray(bt11) | \
(quality_level.data < 4) | (mask_box == 0)
else:
mask = ma.getmaskarray(sst) | ma.getmaskarray(bt11) | \
(quality_level.data < 4)
if mask.all():
print 'No data'
sys.exit(0)
# GCPs for resampling and geotiff georeference
scansize = 16
dtime0 = datetime.utcnow()
gcps = resample.get_gcps_from_bowtie(lon, lat, scansize, ngcps=ngcps)
dtime = datetime.utcnow() - dtime0
print 'Get GCPs from bowtie swath : {}'.format(dtime)
gcplon, gcplat, gcpnpixel, gcpnline = gcps
rspysize = lon.shape[0]
geod = pyproj.Geod(ellps='WGS84')
mid = abs(gcpnline[:, 0] - 0.5).argmin()
xdists = geod.inv(gcplon[mid, :-1], gcplat[mid, :-1],
gcplon[mid, 1:], gcplat[mid, 1:])[2]
xdist = np.sum(xdists) / abs(gcpnpixel[mid, -1] - gcpnpixel[mid, 0])
rspxsize = np.round(xdist / 750.).astype('int') + 1
gcpline = gcpnline * rspysize
gcppixel = gcpnpixel * rspxsize
# Resample with LinearNDInterpolator in output space
dtime0 = datetime.utcnow()
pix, lin = resample.get_points_from_gcps(gcplon, gcplat, gcppixel,
gcpline, rspxsize, rspysize,
1, lon, lat, nprocs=nprocs) - 0.5
dtime = datetime.utcnow() - dtime0
print 'Get input coordinates in new grid : {}'.format(dtime)
# Test input grid in output space
# import matplotlib.pyplot as plt
# for iscan in range(lon.shape[0] / scansize):
# pixscan = pix[iscan * scansize: (iscan+1) * scansize, :]
# linscan = lin[iscan * scansize: (iscan+1) * scansize, :]
# # maskscan = mask[iscan * scansize: (iscan+1) * scansize, :]
# # pixscan = pixscan[~maskscan]
# # linscan = linscan[~maskscan]
# plt.plot(pixscan.flatten(), linscan.flatten(), '+')
# plt.show()
# import pdb ; pdb.set_trace()
# \Test input grid in output space
dtime0 = datetime.utcnow()
sst.data[mask] = np.nan
bt11.data[mask] = np.nan
val = np.dstack((sst.data, bt11.data))
rspval = resample.resample_bowtie_linear(pix, lin, val, scansize,
rspxsize, rspysize, show=False)
rspsst = rspval[:, :, 0]
rspbt11 = rspval[:, :, 1]
rspmask = ma.getmaskarray(rspsst) | ma.getmaskarray(rspbt11)
dtime = datetime.utcnow() - dtime0
print 'Interpolate in new grid : {}'.format(dtime)
# Denoise sst and open mask
rspsst.mask = rspmask
rspbt11.mask = rspmask
finalsst = denoise_sst(rspsst, rspbt11, kernel=denoise_kernel,
width=denoise_width, show=False)
finalmask = ~binary_opening(~rspmask, structure=np.ones((3, 3)),
iterations=open_iterations)
finalsst.mask = finalmask
# Contrast
if vmin == None:
if contrast == 'relative':
vmin = np.percentile(finalsst.compressed(), 0.5)
#elif contrast == 'agulhas':
# dayofyear = float(start_time.timetuple().tm_yday)
# vmin = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. - 9.
# #par = [277.94999694824219, 42, 2.5500030517578125, -219]
# par = [278.09999084472656, 0.62831853071795862,
# 2.4000091552734375, 0.1570796326794896]
# vmin = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific txt file is provided for the range
elif dic is not None:
dayofyear = float(start_time.timetuple().tm_yday)
extrema = dic.get(dayofyear, dic[min(dic.keys(),
key=lambda k:abs(k - dayofyear))])
vmin = extrema[0]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
if vmax == None:
if contrast == 'relative':
vmax = np.percentile(finalsst.compressed(), 99.5)
#elif contrast == 'agulhas':
# dayofyear = float(start_time.timetuple().tm_yday)
# vmax = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. + 4.
# #par = [300.59999084472656, 21, 2.8499908447265625, -191]
# par = [300.59999084472656, 0.29919930034188508,
# 2.8499908447265625, 0.14959965017094254]
# vmax = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific text file is provided for the range
elif dic is not None:
dayofyear = float(start_time.timetuple().tm_yday)
extrema = dic.get(dayofyear, dic[min(dic.keys(),
key=lambda k:abs(k - dayofyear))])
vmax = extrema[1]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
# Flip (geotiff in "swath sense")
finalsst = finalsst[::-1, ::-1]
gcppixel = rspxsize - gcppixel
gcpline = rspysize - gcpline
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time,
units='ms')
metadata['product_name'] = 'SST_VIIRS_denoised'
if contrast == 'relative':
metadata['name'] = os.path.splitext(os.path.basename(infile))[0]
else:
metadata['name'] = '{}_{}'.format(os.path.splitext(os.path.basename(infile))[0], contrast)
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'NOAA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface temperature'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'radiometer'
metadata['sensor_name'] = 'VIIRS'
metadata['sensor_platform'] = 'Suomi-NPP'
#metadata['sensor_pass'] =
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
gcpheight = np.zeros(gcppixel.shape)
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcpheight,
gcppixel, gcpline)
band = []
indndv = np.where(ma.getmaskarray(finalsst) == True)
offset, scale = vmin, (vmax-vmin)/254.
np.clip(finalsst.data, vmin, vmax, out=finalsst.data)
array = np.round((finalsst.data - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=vmax, vmax_pal=vmax,
vmin=vmin, vmin_pal=vmin)
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'sea surface temperature', 'unittype':'K',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'colortable':colortable})
if write_netcdf == False:
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'denoised_sst'
band[0]['long_name'] = 'denoised sea surface temperature'
band[0]['standard_name'] = 'sea_surface_temperature'
# ymid = abs(gcpline[:, 0] - rspysize / 2.).argmin()
# xdists = geod.inv(gcplon[ymid, :-1], gcplat[ymid, :-1],
# gcplon[ymid, 1:], gcplat[ymid, 1:])[2] / \
# np.abs(gcppixel[ymid, 1:] - gcppixel[ymid, :-1])
# xmid = abs(gcppixel[0, :] - rspxsize / 2.).argmin()
# ydists = geod.inv(gcplon[:-1, xmid], gcplat[:-1, xmid],
# gcplon[1:, xmid], gcplat[1:, xmid])[2] / \
# np.abs(gcpline[1:, xmid] - gcpline[:-1, xmid])
# print xdists.min(), xdists.max(), xdists.mean()
# # e.g. 749.905437495 749.905892002 749.905827652
# print ydists.min(), ydists.max(), ydists.mean()
# # e.g. 737.638084996 741.195663083 739.157662785
metadata['spatial_resolution'] = 750.
stfmt.write_netcdf(ncfile, metadata, geolocation, band, 'swath',
ngcps=gcplon.shape)
def eodyn_current(infile,
outdir,
vmin=0.,
vmax=5.08,
vmin_pal=0.,
vmax_pal=2.,
write_netcdf=False):
"""
"""
# Read/Process data
print 'Read/Process data'
ncfile = NCFile(infile)
if 'id' in ncfile.read_global_attributes():
l4id = ncfile.read_global_attribute('id')
# l4id = 'e-Odyn' #ncfile.read_global_attribute('id')
elif re.match(r'^e-Odyn_.*\.nc', os.path.basename(infile)) is not None:
l4id = 'e-Odyn'
else:
raise Exception('Unknown GlobCurrent L4 file.')
# /TMP
ucur = ncfile.read_values(L4_MAPS[l4id]['uname'])[::, ::-1, 0]
ucur = np.transpose(ucur)
vcur = ncfile.read_values(L4_MAPS[l4id]['vname'])[::, ::-1, 0]
vcur = np.transpose(vcur)
masku = [ucur == -9999]
maskv = [vcur == -9999]
if l4id not in ['CourantGeostr']:
lon = ncfile.read_values('lon')[0:2].astype('float64')
lat = ncfile.read_values('lat')[-1:-3:-1].astype('float64')
for i in range(2): # avoid rounding errors
lon[i] = np.round(lon[i] * 10000) / 10000
lat[i] = np.round(lat[i] * 10000) / 10000
else:
lon = ncfile.read_values('lon')[:]
shift = -np.where(lon < 0)[0][0]
ucur = np.roll(ucur, shift, axis=1)
vcur = np.roll(vcur, shift, axis=1)
lon = lon[shift:shift + 2]
lat = ncfile.read_values('lat')[-1:-3:-1]
lon0, dlon, lat0, dlat = lon[0], lon[1] - lon[0], lat[0], lat[1] - lat[0]
#dtime_units = ncfile.read_field('time').units
#dtime = num2date(ncfile.read_values('time')[0], dtime_units)
timefmt = '%Y-%m-%dT%H:%M:%S.%fZ'
start_time = datetime.strptime(
ncfile.read_global_attribute('time_coverage_start'), timefmt)
stop_time = datetime.strptime(
ncfile.read_global_attribute('time_coverage_end'), timefmt)
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
# rundtime = ncfile.read_global_attribute('date_modified')
# rundtime = datetime.strptime(rundtime, '%Y%m%dT%H%M%SZ')
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
metadata['product_name'] = L4_MAPS[l4id]['productname']
metadata['name'] = os.path.splitext(os.path.basename(infile))[0]
metadata['datetime'] = dtime
metadata['time_range'] = time_range
#metadata['time_range'] = L4_MAPS[l4id]['timerange']
metadata['source_URI'] = infile
metadata['source_provider'] = 'e-Odyn'
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = ['current velocity', 'current direction']
# metadata['type'] = 'model'
# metadata['model_longitude_resolution'] = abs(dlon)
# metadata['model_latitude_resolution'] = abs(dlat)
# metadata['model_analysis_datetime'] = stfmt.format_time(rundtime)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['geotransform'] = [
lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat
]
band = []
mask = ucur.mask | vcur.mask
print(mask)
curvel = np.sqrt(ucur.data**2 + vcur.data**2)
curdir = np.mod(
np.arctan2(vcur.data, ucur.data) * 180. / np.pi + 360., 360.)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(curvel, vmin, vmax, out=curvel)
array = np.round((curvel - offset) / scale).astype('uint8')
array[mask] = 255
array[masku] = 255
array[maskv] = 255
print(array)
colortable = stfmt.format_colortable('matplotlib_jet',
vmin=vmin,
vmax=vmax,
vmin_pal=vmin_pal,
vmax_pal=vmax_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'current velocity',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
array = np.round(curdir / 360. * 254.).astype('uint8')
array[mask] = 255
array[masku] = 255
array[maskv] = 255
band.append({
'array': array,
'scale': 360. / 254.,
'offset': 0.,
'description': 'current direction',
'unittype': 'deg',
'nodatavalue': 255,
'parameter_range': [0, 360.]
})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
elif write_netcdf == True:
print 'Write netcdf'
# u/v -> bands
band = []
mask = ucur.mask | vcur.mask
vmin = -vmax
offset, scale = vmin, (vmax - vmin) / 254.
u = np.clip(ucur.data, vmin, vmax)
array = np.round((u - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'current u',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax]
})
v = np.clip(vcur.data, vmin, vmax)
array = np.round((v - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'current v',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax]
})
# Write
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
dgcpy=1.,
dgcpx=1.)
def sentinel3_slstr_bt(infile,
outdir,
vmin=None,
vmax=None,
min_percentile=2.0,
channels='ir',
file_range=None,
write_netcdf=False,
log_path=None,
lat_crop=80.0):
t0 = datetime.utcnow()
# Process nadir data
view = 'n'
fname = 'BT'
sltype = 'i'
if type(channels) is list or type(channels) is tuple:
bandnames = channels
product_name = 'Sentinel-3_SLSTR'
elif 'ir' == channels:
bandnames = ('S8', )
product_name = 'Sentinel-3_SLSTR_IR'
else:
raise Exception('channels must be either "ir", or a tuple of band')
# Read coordinates and compute gcps
(syntool_stats, metadata, geolocation, tie_lon, tie_lat, slice_lat0,
slice_lat1, __, __, ngcps,
month) = slstr.read_geometry(infile, bandnames, fname, sltype, view,
product_name, vmin, vmax, log_path)
# Compute masks
logger.info('Build masks')
t_start = datetime.utcnow()
quality_flags = slstr.read_mask(infile, sltype, view, slice_lat0,
slice_lat1)
raw_cloud_flags = slstr.read_cloud_mask(infile, sltype, view, slice_lat0,
slice_lat1)
contrast_mask, data_mask = build_mask_ir(channels, quality_flags,
raw_cloud_flags, tie_lat,
lat_crop)
t_stop = datetime.utcnow()
syntool_stats['mask_computation'] = (t_stop - t_start).total_seconds()
# Read band to compute histograms
logger.info('Construct bands')
t_start = datetime.utcnow()
bands = []
# Initialize min and max values
if vmin is None:
vmin = [None] * len(bandnames)
if vmax is None:
vmax = [None] * len(bandnames)
_vmin = list(vmin)
_vmax = list(vmax)
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
fieldname = slstr.get_field_name(fname, bandname, sltype, view)
band = slstr.read_band(infile, bandname, fieldname, slice_lat0,
slice_lat1)
logger.info('\tSet contrast')
valid_ratio_lower_threshold = 0.001 # 0.1%
# Select valid data to compute histograms
valid_data_mask = (band.mask | contrast_mask)
valid_data, extra_data_mask, updated_min = get_valid_data_ir(
tie_lon, tie_lat, file_range, band, valid_data_mask, bandname,
month)
if extra_data_mask is not None:
data_mask = (data_mask | extra_data_mask)
if updated_min is not None:
_vmin = [
updated_min,
]
_min = updated_min
# No need to produce an output if all data values are masked
if numpy.all(data_mask):
logger.warn('No valid value found for band {}'.format(bandname))
sys.exit(0)
# Retrieve minimum and maximum values from default or valid_data
# histograms
valid_ratio = float(valid_data.size) / float(band.data.size)
syntool_stats[bandname]['valid_ratio'] = valid_ratio
if valid_ratio_lower_threshold >= valid_ratio:
_min, _max = slstr.apply_default_min_max(default_minmax, bandname,
_vmin[band_index],
_vmax[band_index],
syntool_stats)
else:
_min, _max = slstr.fromband_min_max(valid_data,
bandname,
_vmin[band_index],
_vmax[band_index],
syntool_stats,
min_percentile=min_percentile,
max_percentile=99.99)
_vmin[band_index] = _min
_vmax[band_index] = _max
logger.info('\tContrast : vmin={} / vmax={}'.format(
_vmin[band_index], _vmax[band_index]))
min_values = [_vmin[band_index] for band_index in range(len(bandnames))]
max_values = [_vmax[band_index] for band_index in range(len(bandnames))]
t_stop = datetime.utcnow()
syntool_stats['minmax_computation'] = (t_stop - t_start).total_seconds()
syntool_stats['final_min'] = float(numpy.min(min_values))
syntool_stats['final_max'] = float(numpy.max(max_values))
_min = numpy.min(min_values)
_max = numpy.max(max_values)
scale = (_max - _min) / 254.
offset = _min
# Construct bands
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
fieldname = slstr.get_field_name(fname, bandname, sltype, view)
band = slstr.read_band(infile, bandname, fieldname, slice_lat0,
slice_lat1)
bnd = band.data
logger.info('\tBytescaling')
byte = bytescale(bnd, cmin=_min, cmax=_max, low=0, high=254)
description = '{} {} (log)'.format(bandname, fname)
if band.mask is not numpy.ma.nomask:
byte[band.mask] = 255
# mask night data for rgb and invalid data for ir (cloud, land,
# range value). Also mask data for extreme latitudes
byte[data_mask] = 255
band_range = [_vmin[band_index], _vmax[band_index]]
description = '{} {}'.format(bandname, fname) # no log for IR
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=_max,
vmax_pal=_max,
vmin=_min,
vmin_pal=_min)
bands.append({
'array': byte,
'plot': band.data,
'scale': scale,
'offset': offset,
'description': description,
'unittype': '',
'nodatavalue': 255,
'parameter_range': band_range,
'colortable': colortable
})
if write_netcdf:
bands[-1]['name'] = bandname
bands[-1]['long_name'] = bandname
bands[-1]['unittype'] = '1'
logger.info('Make sure nodata are at the same locations in all bands')
mask = numpy.any([_band['array'] == 255 for _band in bands], axis=0)
for band in bands:
band['array'][mask] = 255
t_stop = datetime.utcnow()
syntool_stats['bytescaling'] = (t_stop - t_start).total_seconds()
if write_netcdf:
metadata['spatial_resolution'] = 1000
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
bands,
'swath',
ngcps=ngcps)
else:
logger.info('Write geotiff')
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, bands)
logger.info(datetime.utcnow() - t0)
syntool_stats['total_time'] = (datetime.utcnow() - t0).total_seconds()
if log_path is not None:
import json
full_path = os.path.normpath(infile)
file_path = os.path.basename(full_path)
file_name, _ = os.path.splitext(file_path)
stats_path = os.path.join(log_path, '{}.json'.format(file_name))
with open(stats_path, 'w') as f:
json.dump(syntool_stats, f)
def ascat_l2b(infile,
outdir,
vmin=0.,
vmax=25.4,
vmin_pal=0.,
vmax_pal=50 * 0.514,
write_netcdf=False):
"""
"""
# Read/Process data
dset = Dataset(infile)
nrow = len(dset.dimensions['NUMROWS'])
ncell = len(dset.dimensions['NUMCELLS'])
if ncell != 82:
raise Exception('Expects NUMCELLS=82 (KNMI ASCAT L2B 12.5km).')
source = dset.source
if 'metop-a' in source.lower():
platform = 'Metop-A'
elif 'metop-b' in source.lower():
platform = 'Metop-B'
else:
raise Exception('Platform ?')
start_time = datetime.strptime(dset.start_date + dset.start_time,
'%Y-%m-%d%H:%M:%S')
stop_time = datetime.strptime(dset.stop_date + dset.stop_time,
'%Y-%m-%d%H:%M:%S')
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
datagroup = os.path.splitext(os.path.basename(infile))[0]
if write_netcdf == False:
# Create GeoTIFF file
_to_geotiff(infile, outdir, vmin, vmax, vmin_pal, vmax_pal, nrow,
ncell, dtime, time_range, datagroup, platform, dset)
else:
for i in range(2):
if i == 0: # left swath
swath_slice = [slice(0, nrow), slice(0, ncell / 2)]
dataset_name = '{}_left'.format(datagroup)
else: # right swath
swath_slice = [slice(0, nrow), slice(ncell / 2, ncell)]
dataset_name = '{}_right'.format(datagroup)
lat = dset.variables['lat'][swath_slice]
lon = dset.variables['lon'][swath_slice]
irow = 0
while irow < nrow:
lon0 = lon[irow, 0] - 180
lon[irow:, :] = np.mod(lon[irow:, :] - lon0, 360) + lon0
notcont = (lon[irow + 1:, :] < lon0 + 90) & (lon[irow:-1, :] > lon0 + 270) | \
(lon[irow + 1:, :] > lon0 + 270) & (lon[irow:-1, :] < lon0 + 90)
indnotcont = np.where(notcont.any(axis=1))[0]
if indnotcont.size == 0:
irow = nrow
else:
indnotcont = indnotcont.min()
if indnotcont == 0:
raise Exception('Unexpected longitudes.')
irow = irow + indnotcont
ind = np.where(np.abs(lon[1:, :] - lon[:-1, :]) > 180.)
if ind[0].size != 0:
raise Exception('Failed to make longitudes continuous.')
if lon[nrow / 2, ncell / 4] > 180:
lon -= 360
elif lon[nrow / 2, ncell / 4] < -180:
lon += 360
print(lon)
wind_speed = dset.variables['wind_speed'][swath_slice]
wind_dir = dset.variables['wind_dir'][swath_slice]
wind_dir = np.mod(90. - wind_dir, 360.)
dgcp = 16.
ngcps = (np.ceil(np.array(lon.shape) / dgcp) + 1.).astype('int32')
pix = np.linspace(0, lon.shape[1] - 1,
num=ngcps[1]).round().astype('int32')
lin = np.linspace(0, lon.shape[0] - 1,
num=ngcps[0]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
# Construct metadata/geolocation/band(s)
print('Construct metadata/geolocation/band(s)')
metadata = {}
metadata['product_name'] = '{}_ASCAT_L2B'.format(platform)
metadata['datagroup'] = datagroup
metadata['name'] = dataset_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(
datetime.utcnow())
metadata['parameter'] = ['wind speed', 'wind direction']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(
gcplon, gcplat, gcphei, gcppix, gcplin)
print('Write netcdf')
# u/v -> bands
band = []
u = wind_speed * np.cos(np.deg2rad(wind_dir))
v = wind_speed * np.sin(np.deg2rad(wind_dir))
mask = np.ma.getmaskarray(u) | np.ma.getmaskarray(v)
vmin = -vmax
offset, scale = vmin, (vmax - vmin) / 254.
clipped = np.clip(np.ma.getdata(u), vmin, vmax)
array = np.round((clipped - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind u',
'unittype': 'm s-1',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'name': 'u',
'standard_name': 'eastward_wind'
})
clipped = np.clip(np.ma.getdata(v), vmin, vmax)
array = np.round((clipped - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind v',
'unittype': 'm s-1',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'name': 'v',
'standard_name': 'northward_wind'
})
# Write
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
metadata['spatial_resolution'] = 12500.
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def sentinel2_rgb(
infile,
outdir,
# For output resolution
overview_index=None,
downsampling=2,
# For manual contrast
vmin=[None, None, None],
vmax=[None, None, None],
# For auto contrast
contrast_overview_index=2,
landmaskpath=None,
slope_threshold=-40.,
debug_fig_dir=None,
atmos_correction=0,
atmos_lut_path=None,
vmax_factor=None,
# For output type
write_netcdf=False):
"""
"""
# Identify stitching groups
print 'Identify stitching group(s)'
groups = safemsil1c_stitching_groups(infile)
projs = groups.keys()
for proj, urls in groups.iteritems():
print ' {} : {} granule(s)'.format(proj, len(urls))
datagroup = make_datagroup(groups)
# Set contrast
if None in vmin or None in vmax:
print 'Set contrast'
vmin, vmax = set_contrast(list(vmin),
list(vmax),
groups,
overview_index=contrast_overview_index,
landmaskpath=landmaskpath,
slope_threshold=slope_threshold,
debug_fig_dir=debug_fig_dir,
atmos_correction=atmos_correction,
atmos_lut_path=atmos_lut_path)
print 'vmin = {:0.4f} / {:0.4f} / {:0.4f}'.format(*vmin)
print 'vmax = {:0.4f} / {:0.4f} / {:0.4f}'.format(*vmax)
if vmax_factor is not None:
print 'Apply vmax_factor={}'.format(vmax_factor)
_vmax = []
for vmi, vma in zip(vmin, vmax):
_vmax.append(vmi + vmax_factor * (vma - vmi))
vmax = _vmax
print 'new vmax = {:0.4f} / {:0.4f} / {:0.4f}'.format(*vmax)
# Build geotiff or netcdf
print 'Build geotiff or netcdf'
bandnames = ['B04', 'B03', 'B02']
for proj in projs:
# Open stitched mapper
print ' {} : open mapper with overview {}'.format(
proj, overview_index)
t0 = datetime.utcnow()
mapper = SAFEMSIL1CStitchedFile(groups[proj],
native_resolution='10m',
overview_index=overview_index,
tight=True)
mapper.open()
print ' {}'.format(datetime.utcnow() - t0)
# Construct bands
bands = []
qvalue = mapper.read_global_attribute('quantification_value')
for iband, bandname in enumerate(bandnames):
fieldname = '{}_digital_number'.format(bandname)
print ' {} : read {}'.format(proj, fieldname)
t0 = datetime.utcnow()
band = mapper.read_values(fieldname)
print ' {}'.format(datetime.utcnow() - t0)
if downsampling != 1:
print ' {} : downsample by {}'.format(proj, downsampling)
t0 = datetime.utcnow()
shp = list(band.shape)
shp[0] -= np.mod(shp[0], downsampling)
shp[1] -= np.mod(shp[1], downsampling)
sli = [slice(0, shp[0]), slice(0, shp[1])]
rshp = (shp[0] / downsampling, downsampling,
shp[1] / downsampling, downsampling)
if not np.ma.is_masked(band):
mask = np.ma.nomask
else:
mask = band[sli].mask.reshape(rshp).\
sum(axis=3, dtype='uint8').\
sum(axis=1, dtype='uint8') > 0
band = np.ma.MaskedArray(band[sli].data.reshape(rshp).\
mean(axis=3, dtype='uint16').\
mean(axis=1, dtype='uint16'),
mask=mask)
del mask
print ' {}'.format(datetime.utcnow() - t0)
print ' {} : bytescale in [{}, {}]'.format(
proj, vmin[iband], vmax[iband])
t0 = datetime.utcnow()
vmin_dn = np.round(vmin[iband] * qvalue)
vmax_dn = np.round(vmax[iband] * qvalue)
byte = bytescale(band.data,
cmin=vmin_dn,
cmax=vmax_dn,
low=0,
high=254)
if band.mask is not np.ma.nomask:
byte[band.mask] = 255
del band
scale = (vmax[iband] - vmin[iband]) / 254.
offset = vmin[iband]
description = '{} TOA reflectance'.format(bandname)
bands.append({
'array': byte,
'scale': scale,
'offset': offset,
'description': description,
'unittype': '',
'nodatavalue': 255,
'parameter_range': [vmin[iband], vmax[iband]]
})
print ' {}'.format(datetime.utcnow() - t0)
# Make sure nodata are at the same locations in all bands
mask = np.any([band['array'] == 255 for band in bands], axis=0)
for band in bands:
band['array'][mask] = 255
# Construct metadata and geolocation
print ' {} : construct metadata and geolocation'.format(proj)
t0 = datetime.utcnow()
cs_code = mapper.read_global_attribute('horizontal_cs_code')
epsg_num = cs_code.lower().lstrip('epsg:')
dataname = '{}-{}'.format(datagroup, epsg_num)
start_time = mapper.get_start_time()
end_time = mapper.get_end_time()
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='ms')
sensor_pass = mapper.read_global_attribute(
'sensing_orbit_direction').lower()
metadata = {}
metadata['product_name'] = 'Sentinel-2_RGB'
metadata['name'] = dataname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'ESA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = [
'B04 TOA reflectance', 'B03 TOA reflectance', 'B02 TOA reflectance'
]
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'multi-spectral imager'
metadata['sensor_name'] = 'MSI'
metadata['sensor_platform'] = 'Sentinel-2'
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
srs = osr.SpatialReference()
srs.ImportFromEPSG(int(epsg_num))
ulx = mapper.read_global_attribute('ulx')
dx = mapper.read_global_attribute('xdim') * downsampling
uly = mapper.read_global_attribute('uly')
dy = mapper.read_global_attribute('ydim') * downsampling
geolocation = {}
geolocation['projection'] = srs.ExportToWkt()
geolocation['geotransform'] = [ulx, dx, 0, uly, 0, dy]
print ' {}'.format(datetime.utcnow() - t0)
# Write geotiff or netcdf
mapper.close()
if write_netcdf == False:
print ' {} : write geotiff'.format(proj)
t0 = datetime.utcnow()
tifffile = stfmt.format_tifffilename(outdir,
metadata,
create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, bands)
print ' {}'.format(datetime.utcnow() - t0)
elif write_netcdf == True:
print ' {} : write geotiff'.format(proj)
t0 = datetime.utcnow()
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
bands[0]['name'] = 'B04_TOA_reflectance'
bands[1]['name'] = 'B03_TOA_reflectance'
bands[2]['name'] = 'B02_TOA_reflectance'
resolution = min([abs(dy), abs(dx)])
metadata['spatial_resolution'] = resolution
dgcps_meter = 25000.
dgcps = (np.round(dgcps_meter / resolution).astype('int'), ) * 2
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
bands,
'grid_proj',
dgcps=dgcps)
print ' {}'.format(datetime.utcnow() - t0)
def viirs_chlora(infileid,
outdir,
download_dir='/tmp',
vmin=None,
vmax=None,
contrast='relative',
ngcps=(26, 32),
open_iterations=1,
nprocs=1,
pngkml=False,
write_netcdf=False):
"""
"""
if contrast == 'med':
listbox = [[-6., 35., 2.75, 42.48], [2.74, 30, 42.2, 47.00]]
elif contrast == 'cwe':
listbox = [[-23., 35.2, -5.5, 42.88], [-23., 42.8, 2.20, 51.]]
elif contrast == 'nwe':
listbox = [[-23., 50.8, 32.7, 68.]]
elif contrast == 'gom':
listbox = [[-98., 18.0, -80.5, 30.5]]
elif contrast == 'agulhas':
listbox = [[10.8437, -45.7404, 39.9799, -25.3019]]
elif contrast == 'gs':
listbox = [[-81.52, 20, -30, 45]]
else:
listbox = None
# Search/Download data
print 'Search/Download data'
if re.match(r'^V[0-9]{13}$', infileid) is None:
raise Exception('Input for viirs_chlora is an ID '
'(e.g. V2014093110000)')
product_id = 'VIIRSL2OC'
date = datetime.strptime(infileid[1:], '%Y%j%H%M%S')
viirsocfname = viirs.search_and_download(product_id, date, download_dir)
# Read/Process data
print 'Read/Process data'
# Read from OC file
viirsocfile = viirs.VIIRSL2File(viirsocfname)
lon = viirsocfile.read_lon()
lat = viirsocfile.read_lat()
chlora = viirsocfile.read_chlora()
attrs = viirsocfile.read_attributes()
viirsocfile.close()
if listbox is not None:
mask_box = np.zeros(np.shape(chlora.data))
for i in range(np.shape(listbox)[0]):
index_in = np.where((lon >= listbox[i][0]) & (lat >= listbox[i][1])
& (lon <= listbox[i][2])
& (lat <= listbox[i][3]))
mask_box[index_in] = 1
mask = (mask_box == 0) | ma.getmaskarray(chlora)
else:
mask = ma.getmaskarray(chlora)
if mask.all():
print 'No data'
sys.exit(0)
# GCPs for resampling and geotiff georeference
scansize = 16
dtime0 = datetime.utcnow()
gcps = resample.get_gcps_from_bowtie(lon, lat, scansize, ngcps=ngcps)
dtime = datetime.utcnow() - dtime0
print 'Get GCPs from bowtie swath : {}'.format(dtime)
gcplon, gcplat, gcpnpixel, gcpnline = gcps
rspysize = lon.shape[0]
geod = pyproj.Geod(ellps='WGS84')
mid = abs(gcpnline[:, 0] - 0.5).argmin()
xdists = geod.inv(gcplon[mid, :-1], gcplat[mid, :-1], gcplon[mid, 1:],
gcplat[mid, 1:])[2]
xdist = np.sum(xdists) / abs(gcpnpixel[mid, -1] - gcpnpixel[mid, 0])
rspxsize = np.round(xdist / 750.).astype('int') + 1
gcpline = gcpnline * rspysize
gcppixel = gcpnpixel * rspxsize
# Resample with LinearNDInterpolator in output space
dtime0 = datetime.utcnow()
pix, lin = resample.get_points_from_gcps(gcplon,
gcplat,
gcppixel,
gcpline,
rspxsize,
rspysize,
1,
lon,
lat,
nprocs=nprocs) - 0.5
dtime = datetime.utcnow() - dtime0
print 'Get input coordinates in new grid : {}'.format(dtime)
# Test input grid in output space
# import matplotlib.pyplot as plt
# for iscan in range(lon.shape[0] / scansize):
# pixscan = pix[iscan * scansize: (iscan+1) * scansize, :]
# linscan = lin[iscan * scansize: (iscan+1) * scansize, :]
# # maskscan = mask[iscan * scansize: (iscan+1) * scansize, :]
# # pixscan = pixscan[~maskscan]
# # linscan = linscan[~maskscan]
# plt.plot(pixscan.flatten(), linscan.flatten(), '+')
# plt.show()
# import pdb ; pdb.set_trace()
# \Test input grid in output space
dtime0 = datetime.utcnow()
chlora.data[mask] = np.nan
rspchlora = resample.resample_bowtie_linear(pix,
lin,
chlora.data,
scansize,
rspxsize,
rspysize,
show=False)
rspmask = ma.getmaskarray(rspchlora)
dtime = datetime.utcnow() - dtime0
print 'Interpolate in new grid : {}'.format(dtime)
# Take log and open mask
finalchlora = ma.log(rspchlora)
finalmask = ~binary_opening(
~rspmask, structure=np.ones((3, 3)), iterations=open_iterations)
finalchlora.mask = finalmask
# Contrast
if vmin == None:
if contrast == 'relative':
vmin = np.percentile(finalchlora.compressed(), 0.5)
elif contrast == 'agulhas':
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
vmin = -0.5 * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) - 3.
elif contrast == 'med' or contrast == 'nwe' or contrast == 'cwe':
vmin = np.percentile(finalchlora.compressed(), 2)
else:
raise Exception('Unknown contrast : {}'.format(contrast))
else:
if vmin != 0:
vmin = math.log(vmin)
else:
vmin = np.percentile(finalchlora.compressed(), 0.5)
if vmax == None:
if contrast == 'relative':
vmax = np.percentile(finalchlora.compressed(), 99.5)
elif contrast == 'agulhas':
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
vmax = 0.5 * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 3.
elif contrast == 'med':
vmax = np.percentile(finalchlora.compressed(), 98)
elif contrast == 'nwe':
vmax = np.percentile(finalchlora.compressed(), 98)
elif contrast == 'cwe':
vmax = np.percentile(finalchlora.compressed(), 98)
else:
raise Exception('Unknown contrast : {}'.format(contrast))
else:
if vmax != 0:
vmax = math.log(vmax)
else:
vmax = np.percentile(finalchlora.compressed(), 98)
# Flip (geotiff in "swath sense")
finalchlora = finalchlora[::-1, ::-1]
gcppixel = rspxsize - gcppixel
gcpline = rspysize - gcpline
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(attrs['start_time'],
attrs['stop_time'],
units='ms')
metadata['product_name'] = 'Chlorophyll_a_concentration_VIIRS'
if contrast == 'relative':
metadata['name'] = os.path.splitext(os.path.basename(viirsocfname))[0]
else:
metadata['name'] = '{}_{}'.format(
os.path.splitext(os.path.basename(viirsocfname))[0], contrast)
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = viirsocfname
metadata['source_provider'] = 'NOAA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'chlorophyll a concentration'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'radiometer'
metadata['sensor_name'] = 'VIIRS'
metadata['sensor_platform'] = 'Suomi-NPP'
metadata['sensor_pass'] = attrs['pass']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
gcpheight = np.zeros(gcppixel.shape)
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcpheight,
gcppixel, gcpline)
band = []
indndv = np.where(ma.getmaskarray(finalchlora) == True)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(finalchlora.data, vmin, vmax, out=finalchlora.data)
array = np.round((finalchlora.data - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('chla_jet',
vmax=vmax,
vmax_pal=vmax,
vmin=vmin,
vmin_pal=vmin)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'chlorophyll a concentration',
'unittype': 'log(mg/m3)',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
if write_netcdf == False:
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'chlor_a'
band[0]['long_name'] = 'Chlorophyll Concentration, OCI Algorithm'
band[0][
'standard_name'] = 'mass_concentration_chlorophyll_concentration_in_sea_water'
band[0]['unittype'] = 'mg m^-3 (log)'
# ymid = abs(gcpline[:, 0] - rspysize / 2.).argmin()
# xdists = geod.inv(gcplon[ymid, :-1], gcplat[ymid, :-1],
# gcplon[ymid, 1:], gcplat[ymid, 1:])[2] / \
# np.abs(gcppixel[ymid, 1:] - gcppixel[ymid, :-1])
# xmid = abs(gcppixel[0, :] - rspxsize / 2.).argmin()
# ydists = geod.inv(gcplon[:-1, xmid], gcplat[:-1, xmid],
# gcplon[1:, xmid], gcplat[1:, xmid])[2] / \
# np.abs(gcpline[1:, xmid] - gcpline[:-1, xmid])
# print xdists.min(), xdists.max(), xdists.mean()
# # e.g. 749.810419844 749.810438261 749.810429577
# print ydists.min(), ydists.max(), ydists.mean()
# # e.g. 737.874499629 739.856423757 738.87317625
metadata['spatial_resolution'] = 750.
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def odyssea_sst(infile,
outdir,
vmin=271.05,
vmax=309.15,
vmin_pal=273.,
vmax_pal=305.,
write_netcdf=False):
"""
"""
# Read/Process data
print 'Read/Process data'
odyssea = GHRSSTNCFile(infile)
sst = odyssea.read_values('analysed_sst')[::-1, :]
mask = odyssea.read_values('mask')[::-1, :]
#sea_ice_fraction = odyssea.read_values('sea_ice_fraction')[::-1, :]
# lon = odyssea.read_values('lon')
# dlon = lon[1] - lon[0]
# lon0 = lon[0] - dlon / 2
# lat = odyssea.read_values('lat')[::-1]
# dlat = lat[1] - lat[0]
# lat0 = lat[0] - dlat / 2
lon0 = odyssea.read_global_attribute('westernmost_longitude')
dlon = float(odyssea.read_global_attribute('geospatial_lon_resolution'))
lat0 = odyssea.read_global_attribute('northernmost_latitude')
dlat = -float(odyssea.read_global_attribute('geospatial_lat_resolution'))
dtime = odyssea.read_values('time')[0]
dtime_units = odyssea.read_field('time').units
dtime = num2date(dtime, dtime_units)
odyssea_id = odyssea.read_global_attribute('id')
if 'glob' in odyssea_id.lower():
product_name = 'ODYSSEA_SST'
elif 'saf' in odyssea_id.lower():
product_name = 'ODYSSEA_SAF_SST'
elif 'med' in odyssea_id.lower():
product_name = 'ODYSSEA_MED_SST'
#vmin_pal=283. ; vmax_pal=300.
elif 'nwe' in odyssea_id.lower():
product_name = 'ODYSSEA_NWE_SST'
elif 'bra' in odyssea_id.lower():
product_name = 'ODYSSEA_BRA_SST'
elif 'nseabaltic' in odyssea_id.lower():
product_name = 'DMI-OI_NSEABALTIC_SST'
else:
raise Exception('Unknown odyssea ID : {}'.format(odyssea_id))
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
metadata['product_name'] = product_name
metadata['name'] = os.path.splitext(os.path.basename(infile))[0]
metadata['datetime'] = stfmt.format_time(dtime)
metadata['time_range'] = ['-12h', '+12h']
metadata['source_URI'] = infile
metadata['source_provider'] = 'Ifremer'
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface temperature'
metadata['type'] = 'remote sensing'
metadata['longitude_resolution'] = abs(dlon)
metadata['latitude_resolution'] = abs(dlat)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['geotransform'] = [lon0, dlon, 0, lat0, 0, dlat]
band = []
#indndv = np.where((sst.mask == True) | (sea_ice_fraction > 0))
indndv = np.where((sst.mask == True) | (mask != 1))
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(sst, vmin, vmax, out=sst)
array = np.round((sst - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'sea surface temperature',
'unittype': 'K',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'analysed_sst'
band[0]['long_name'] = 'analysed sea surface temperature'
band[0]['standard_name'] = 'sea_surface_temperature'
metadata['spatial_resolution'] = min([abs(dlat), abs(dlon)]) * 111000.
dgcps = np.round(1. / np.abs(np.array([dlat, dlon]))).astype('int')
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'grid_lonlat',
dgcps=dgcps)
def ecmwf_model_wind(infile, outdir, max_forecast_hours=None,
vmin=0., vmax=25.4, vmin_pal=0., vmax_pal=50*0.514,
write_netcdf=False):
"""
"""
# Read/Process data
windfield = ECMWF0125NCFile(infile)
u10 = windfield.read_values('u10m')[0, ::-1, :]
v10 = windfield.read_values('v10m')[0, ::-1, :]
lon = windfield.read_values('lon')
dlon = lon[1]-lon[0]
lat = windfield.read_values('lat')[::-1]
dlat = lat[1]-lat[0]
land_mask = get_land_mask()[::-1, :]
# Replicate -180 deg at 180 deg for gdal_warp
# dim = u10.shape
# u10 = np.hstack((u10, u10[:, 0].reshape((dim[0], 1))))
# v10 = np.hstack((v10, v10[:, 0].reshape((dim[0], 1))))
# lon = np.hstack((lon, lon[0]+360.))
# land_mask = np.hstack((land_mask, land_mask[:, 0].reshape((dim[0], 1))))
# /Replicate -180 deg at 180 deg for gdal_warp
dtime = windfield.read_values('time')[0]
dtime_units = windfield.read_field('time').units
dtime = num2date(dtime, dtime_units)
rundtime = windfield.read_global_attribute('run_time')
rundtime = datetime.strptime(rundtime, '%Y-%m-%dT%H:%M:%SZ')
if max_forecast_hours is not None:
forecast_hours = (dtime - rundtime).total_seconds() / 3600.
if forecast_hours > max_forecast_hours:
raise Exception('Exceeds max_forecast_hours.')
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
metadata['product_name'] = 'ECMWF_model_wind'
#metadata['name'] = os.path.splitext(os.path.basename(infile))[0]
metadata['name'] = 'ECMWF_'+dtime.strftime('%Y%m%dT%HZ')
metadata['datetime'] = stfmt.format_time(dtime)
metadata['time_range'] = ['-90m', '+90m']
metadata['source_URI'] = infile
metadata['source_provider'] = 'ECMWF'
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
#metadata['parameter'] = ['zonal wind speed', 'meridional wind speed']
metadata['parameter'] = ['wind speed', 'wind direction']
metadata['type'] = 'model'
metadata['model_longitude_resolution'] = 0.125
metadata['model_latitude_resolution'] = 0.125
metadata['model_analysis_datetime'] = stfmt.format_time(rundtime)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['geotransform'] = [lon[0]-dlon/2., dlon, 0,
lat[0]-dlat/2., 0, dlat]
# band = []
# scale = 0.2
# offset = -25.4
# windspeed = np.sqrt(u10**2 + v10**2)
# winddirection = np.arctan2(v10, u10)
# np.clip(windspeed, 0, abs(offset), out=windspeed)
# u10 = np.cos(winddirection)*windspeed
# array = np.round((u10-offset)/scale).astype('uint8')
# band.append({'array':array, 'scale':scale, 'offset':offset,
# 'description':'zonal wind speed', 'unittype':'m/s',
# 'nodatavalue':255, 'parameter_range':[-25.4, 25.4]})
# v10 = np.sin(winddirection)*windspeed
# array = np.round((v10-offset)/scale).astype('uint8')
# band.append({'array':array, 'scale':scale, 'offset':offset,
# 'description':'meridional wind speed', 'unittype':'m/s',
# 'nodatavalue':255, 'parameter_range':[-25.4, 25.4]})
band = []
indndv = np.where(land_mask == 1)
windspeed = np.sqrt(u10**2 + v10**2)
winddirection = np.mod(np.arctan2(v10, u10)*180./np.pi+360., 360.)
offset, scale = vmin, (vmax-vmin)/254.
np.clip(windspeed, vmin, vmax, out=windspeed)
array = np.round((windspeed - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('noaa_wind', vmax=vmax, vmax_pal=vmax_pal,
vmin=vmin, vmin_pal=vmin_pal)
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'wind speed', 'unittype':'m/s',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'colortable':colortable})
array = np.round(winddirection/360.*254.).astype('uint8')
array[indndv] = 255
band.append({'array':array, 'scale':360./254., 'offset':0.,
'description':'wind direction', 'unittype':'deg',
'nodatavalue':255, 'parameter_range':[0, 360.]})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
elif write_netcdf == True:
print 'Write netcdf'
# u/v -> bands
band = []
indndv = np.where(land_mask == 1)
vmin = -vmax
offset, scale = vmin, (vmax-vmin)/254.
np.clip(u10, vmin, vmax, out=u10)
array = np.round((u10 - offset) / scale).astype('uint8')
array[indndv] = 255
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'wind u', 'unittype':'m s-1',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'name':'u10m', 'long_name':'u component of horizontal wind',
'standard_name':'eastward_wind'})
np.clip(v10, vmin, vmax, out=v10)
array = np.round((v10 - offset) / scale).astype('uint8')
array[indndv] = 255
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'wind v', 'unittype':'m s-1',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'name':'v10m', 'long_name':'v component of horizontal wind',
'standard_name':'northward_wind'})
# Write
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
metadata['spatial_resolution'] = 0.125 * 111000.
dgcps = np.round(1. / np.array([0.125, 0.125])).astype('int')
stfmt.write_netcdf(ncfile, metadata, geolocation, band, 'grid_lonlat',
dgcps=dgcps)
def sentinel3_slstr_rad(infile,
outdir,
vmin=None,
vmax=None,
max_sunglint=150,
min_percentile=2.0,
channels='false_rgb',
write_netcdf=False,
lut_path=None,
log_path=None,
lat_crop=80.0):
t0 = datetime.utcnow()
# Process nadir data
view = 'n'
fname = 'radiance'
sltype = 'a'
if type(channels) is list or type(channels) is tuple:
bandnames = channels
product_name = 'Sentinel-3_SLSTR'
elif 'false_rgb' == channels:
bandnames = ('S3', 'S2', 'S1')
product_name = 'Sentinel-3_SLSTR_false_RGB'
else:
raise Exception('channels must be either "false_rgb",'
' or a tupple of bands')
# convert band into column number in the LUT
band_columns = [int(x[1:]) + 3 - 1 for x in bandnames]
# Read coordinates and compute gcps
(syntool_stats, metadata, geolocation, tie_lon, tie_lat, slice_lat0,
slice_lat1, nrow_all, ncell_all, ngcps,
__) = slstr.read_geometry(infile, bandnames, fname, sltype, view,
product_name, vmin, vmax, log_path)
# Compute atmospheric radiance TOA correction
L_toa = None
if lut_path is not None:
t_start = datetime.utcnow()
# Compute atmospheric correction for the whole granule
L_toa = atmospheric_correction(lut_path, infile, view, sltype,
band_columns, bandnames, nrow_all,
ncell_all)
# Extract the slices of atmospheric correction that match the ones of
# the bands after the removal of the high latitude columns.
for iband in range(len(L_toa)):
L_toa[iband] = L_toa[iband][slice_lat0, slice_lat1]
# An erroneous LUT could produce nan values for the atmospheric
# correction. This is not acceptable.
if numpy.any(~numpy.isfinite(L_toa[iband])):
raise Exception('Infinite or nan value found in the '
'atmospheric correction, please check that the'
'LUT has valid values for the angles contained'
'in the geometrie_t{}.nc file'.format(view))
t_stop = datetime.utcnow()
syntool_stats['lut_computation'] = (t_stop - t_start).total_seconds()
# Compute masks
logger.info('Build masks')
t_start = datetime.utcnow()
quality_flags = slstr.read_mask(infile, sltype, view, slice_lat0,
slice_lat1)
try:
contrast_mask, data_mask = build_mask_rgb(channels, quality_flags,
tie_lat, lat_crop)
except OnlyNightData:
logger.warn('No day data in granule.')
sys.exit(0)
t_stop = datetime.utcnow()
syntool_stats['mask_computation'] = (t_stop - t_start).total_seconds()
# Read band to compute histograms
logger.info('Construct bands')
t_start = datetime.utcnow()
bands = []
# Initialize min and max values
if vmin is None:
vmin = [None] * len(bandnames)
if vmax is None:
vmax = [None] * len(bandnames)
_vmin = list(vmin)
_vmax = list(vmax)
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
fieldname = slstr.get_field_name(fname, bandname, sltype, view)
band = slstr.read_band(infile, bandname, fieldname, slice_lat0,
slice_lat1)
# Apply atmospheric correction
if L_toa is not None:
band -= L_toa[band_index][:, :]
# Mask null and negative values: they are inferior or equal to
# atmospheric correction and should probably have been flagged as
# clouds.
mask_negative = (band <= 0.0)
logger.info('\tSet contrast')
valid_ratio_lower_threshold = 0.001 # 0.1%
# Select valid data to compute histograms
valid_data_mask = (band.mask | contrast_mask | mask_negative)
valid_data = get_valid_data_rgb(band, valid_data_mask, max_sunglint)
# No need to produce an output if all data values are masked
if numpy.all(data_mask):
logger.warn('No valid value found for band {}'.format(bandname))
sys.exit(0)
# Retrieve minimum and maximum values from default or valid_data
# histograms
valid_ratio = float(valid_data.size) / float(band.data.size)
syntool_stats[bandname]['valid_ratio'] = valid_ratio
if valid_ratio_lower_threshold >= valid_ratio:
_min, _max = slstr.apply_default_min_max(default_minmax, bandname,
_vmin[band_index],
_vmax[band_index],
syntool_stats)
else:
_min, _max = slstr.fromband_min_max(valid_data,
bandname,
_vmin[band_index],
_vmax[band_index],
syntool_stats,
min_percentile=min_percentile,
max_percentile=99.99)
# take default min and max if min or max are too high (lake,
# inland sea, clouds)
_max = min(_max, max_sunglint)
if (_min > 100):
_min = default_minmax[bandname][0]
_max = default_minmax[bandname][1]
_vmin[band_index] = _min
_vmax[band_index] = _max
logger.info('\tContrast : vmin={} / vmax={}'.format(
_vmin[band_index], _vmax[band_index]))
min_values = [_vmin[band_index] for band_index in range(len(bandnames))]
max_values = [_vmax[band_index] for band_index in range(len(bandnames))]
t_stop = datetime.utcnow()
syntool_stats['minmax_computation'] = (t_stop - t_start).total_seconds()
syntool_stats['final_min'] = float(numpy.min(min_values))
syntool_stats['final_max'] = float(numpy.max(max_values))
_min = numpy.min(min_values)
_max = numpy.max(max_values)
_min = numpy.log(_min)
_max = numpy.log(_max)
scale = (_max - _min) / 254.
offset = _min
# Construct bands
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
fieldname = slstr.get_field_name(fname, bandname, sltype, view)
band = slstr.read_band(infile, bandname, fieldname, slice_lat0,
slice_lat1)
# Apply atmospheric correction
if L_toa is not None:
band -= L_toa[band_index][:, :]
# Mask null and negative values: they are inferior or equal to
# atmospheric correction and should probably have been flagged.
mask_negative = (band <= 0.0)
# Compute the logarithm only for radiance values that are higher than
# the atmospheric correction.
bnd = band.data
bnd = numpy.log(band.data, where=(~mask_negative))
logger.info('\tBytescaling')
byte = bytescale(bnd, cmin=_min, cmax=_max, low=0, high=254)
description = '{} {} (log)'.format(bandname, fname)
if band.mask is not numpy.ma.nomask:
byte[band.mask] = 255
# Pixels with a radiance equal or inferior to atmospheric correction
# are clipped to the minimal value.
if 0 < mask_negative.size:
byte[numpy.where(mask_negative == True)] = 0 # noqa
# mask night data for rgb and invalid data for ir (cloud, land,
# range value). Also mask data for extreme latitudes
byte[data_mask] = 255
band_range = [_vmin[band_index], _vmax[band_index]]
bands.append({
'array': byte,
'plot': band.data[~mask_negative],
'scale': scale,
'offset': offset,
'description': description,
'unittype': '',
'nodatavalue': 255,
'parameter_range': band_range
})
if write_netcdf:
bands[-1]['name'] = bandname
bands[-1]['long_name'] = bandname
bands[-1]['unittype'] = '1'
logger.info('Make sure nodata are at the same locations in all bands')
mask = numpy.any([_band['array'] == 255 for _band in bands], axis=0)
for band in bands:
band['array'][mask] = 255
t_stop = datetime.utcnow()
syntool_stats['bytescaling'] = (t_stop - t_start).total_seconds()
if write_netcdf:
metadata['spatial_resolution'] = 500
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
bands,
'swath',
ngcps=ngcps)
else:
logger.info('Write geotiff')
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, bands)
logger.info(datetime.utcnow() - t0)
syntool_stats['total_time'] = (datetime.utcnow() - t0).total_seconds()
if log_path is not None:
import json
full_path = os.path.normpath(infile)
file_path = os.path.basename(full_path)
file_name, _ = os.path.splitext(file_path)
stats_path = os.path.join(log_path, '{}.json'.format(file_name))
with open(stats_path, 'w') as f:
json.dump(syntool_stats, f)
def ghrsst_seviri(infile,
outdir,
vmin=271.05,
vmax=309.15,
vmin_pal=273.,
vmax_pal=305.,
write_netcdf=False):
"""
"""
# Read/Process data
print 'Read/Process data'
seviri = GHRSSTNCFile(infile)
sst = seviri.read_values('sea_surface_temperature')[::-1, :]
mask = seviri.read_values('quality_level')[::-1, :]
#sea_ice_fraction = seviri.read_values('sea_ice_fraction')[::-1, :]
# lon = seviri.read_values('lon')
# dlon = lon[1] - lon[0]
# lon0 = lon[0] - dlon / 2
# lat = seviri.read_values('lat')[::-1]
# dlat = lat[1] - lat[0]
# lat0 = lat[0] - dlat / 2
lon0 = seviri.read_global_attribute('westernmost_longitude')
dlon = float(seviri.read_global_attribute('geospatial_lon_resolution'))
lat0 = seviri.read_global_attribute('northernmost_latitude')
dlat = -float(seviri.read_global_attribute('geospatial_lat_resolution'))
dtime = seviri.read_values('time')[0]
dtime_units = seviri.read_field('time').units
dtime = num2date(dtime, dtime_units)
start_time = datetime.strptime(seviri.read_global_attribute('start_time'),
'%Y%m%dT%H%M%SZ')
stop_time = datetime.strptime(seviri.read_global_attribute('stop_time'),
'%Y%m%dT%H%M%SZ')
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
metadata['product_name'] = 'SEVIRI_SST'
metadata['name'] = os.path.splitext(os.path.basename(infile))[0]
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'Ifremer'
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface temperature'
metadata['type'] = 'remote sensing'
metadata['longitude_resolution'] = abs(dlon)
metadata['latitude_resolution'] = abs(dlat)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['geotransform'] = [lon0, dlon, 0, lat0, 0, dlat]
band = []
#indndv = np.where((sst.mask == True) | (sea_ice_fraction > 0))
indndv = np.where((sst.mask == True) | (mask <= 3))
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(sst, vmin, vmax, out=sst)
array = np.round((sst - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'sea surface temperature',
'unittype': 'K',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'sea_surface_temperature'
band[0]['long_name'] = 'sea surface subskin temperature'
band[0]['standard_name'] = 'sea_surface_subskin_temperature'
metadata['spatial_resolution'] = min([abs(dlat), abs(dlon)]) * 111000.
dgcps = np.round(1. / np.abs(np.array([dlat, dlon]))).astype('int')
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'grid_lonlat',
dgcps=dgcps)
def sar_roughness(infile,
outdir,
pngkml=False,
contrast=None,
vmin=None,
vmax=None,
landmaskpath=None,
write_netcdf=False,
gcp2height=0):
"""
"""
# Read/Process data
print 'Read/Process data'
sarmp = SAFEGeoTiffFile(infile)
sarim = SARImage(sarmp)
mission = sarim.get_info('mission')
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
elif mission == 'S1B':
sensor_name = 'Sentinel-1B'
sensor_platform = 'Sentinel-1B'
source_provider = 'ESA'
else:
raise Exception('Unknown mission')
timefmt = '%Y-%m-%dT%H:%M:%S.%f'
start_time = datetime.strptime(sarim.get_info('start_time'), timefmt)
stop_time = datetime.strptime(sarim.get_info('stop_time'), timefmt)
sensor_pass = sarim.get_info('pass')
sensor_mode = sarim.get_info('mode')
sensor_swath = sarim.get_info('swath')
sensor_polarisation = sarim.get_info('polarisation')
product = sarim.get_info('product')
if product == 'GRD':
spacing = [2, 2]
elif product == 'SLC':
if sensor_mode == 'WV':
mspacing = (15, 15)
elif re.match(r'^S[1-6]$', sensor_mode) != None:
mspacing = (15, 15)
elif sensor_mode == 'IW':
raise Exception('sar_roughness for IW SLC ?')
elif sensor_mode == 'EW':
raise Exception('sar_roughness for EW SLC ?')
else:
raise Exception('Unkown S1 mode : {}'.format(sensor_mode))
spacing = np.round(sarim.meters2pixels(mspacing))
else:
raise Exception('Unkown S1 product : {}'.format(product))
mspacing = sarim.pixels2meters(spacing)
datagroup = sarim.get_info('safe_name').replace('.SAFE', '')
pid = datagroup.split('_')[-1]
dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
ssr = np.sqrt(sarim.get_data('roughness', spacing=spacing))
########## TMP calibration constant ##########
# if sensor_mode == 'WV':
# caldir = '/home/cercache/project/mpc-sentinel1/analysis/s1_data_analysis/L1/WV/S1A_WV_SLC__1S/cal_cste'
# if sensor_polarisation == 'HH':
# if sensor_swath == 'WV1':
# caltmp = (55.80+56.91)/2.
# calname = 'cal_cste_hh_wv1.pkl'
# elif sensor_swath == 'WV2':
# caltmp = (40.65+40.32)/2.
# calname = 'cal_cste_hh_wv2.pkl'
# elif sensor_polarisation == 'VV':
# if sensor_swath == 'WV1':
# caltmp = 58.24
# calname = 'cal_cste_vv_wv1.pkl'
# elif sensor_swath == 'WV2':
# caltmp = 49.02
# calname = 'cal_cste_vv_wv2.pkl'
# calpath = os.path.join(caldir, calname)
# if os.path.exists(calpath) == True:
# caltmp = get_caltmp(calpath, start_time)
# elif re.match(r'^S[1-6]$', sensor_mode) != None:
# if start_time < datetime(2014, 7, 16, 0, 0, 0):
# if sensor_mode == 'S6':
# raise Exception('S6 calibration missing')
# sm2cal = {'S1':58., 'S2':56., 'S3':52., 'S4':52., 'S5':49.}
# else:
# # from commissioning phase report
# sm2cal = {'S1':3., 'S2':5., 'S3':-1.5, 'S4':4., 'S5':1., 'S6':4.75}
# caltmp = sm2cal[sensor_mode]
# elif sensor_mode == 'IW':
# if start_time < datetime(2014, 7, 16, 0, 0, 0):
# caltmp = 109.
# else:
# caltmp = 3. # from commissioning phase report
# elif sensor_mode == 'EW':
# if start_time < datetime(2014, 7, 16, 0, 0, 0):
# caltmp = 94.
# else:
# caltmp = -1. # <- -2. # from commissioning phase report
# else:
# raise Exception('Which tmp calibration constant for this mode ?')
# print '--> caltmp=%f' % caltmp
# ssr *= np.sqrt(10 ** (caltmp / 10.))
########## /TMP calibration constant ##########
dim = ssr.shape
# Set contrast
if vmin == None or vmax == None:
if contrast == None:
if sensor_mode == 'WV':
contrast = 'relative'
else:
contrast = 'sea'
if contrast == 'relative':
if sensor_mode == 'WV':
noborder = [
slice(int(dim[0] * .05), int(dim[0] * .95)),
slice(int(dim[1] * .05), int(dim[1] * .95))
]
else:
noborder = [
slice(int(dim[0] * .05), int(dim[0] * .95)),
slice(int(dim[1] * .1), int(dim[1] * .9))
]
values = ssr[noborder]
if landmaskpath != None and os.path.exists(landmaskpath):
lmspacing = np.round(sarim.meters2pixels(111.32 / 120 * 1000))
lmspacing -= np.mod(lmspacing, spacing)
lon = sarim.get_data('lon', spacing=lmspacing)
lat = sarim.get_data('lat', spacing=lmspacing)
lmdim = (lon.shape[0] + 1, lon.shape[1] + 1)
landmask = np.ones(lmdim, dtype=bool)
landmask[:-1, :-1] = get_landmask(lon, lat, landmaskpath)
lmfac = lmspacing / spacing
landmask = np.repeat(landmask, lmfac[0], axis=0)
landmask = np.repeat(landmask, lmfac[1], axis=1)
seaindex = np.where(landmask[noborder] == False)
if seaindex[0].size >= ssr.size * 0.01:
values = values[seaindex]
if vmin == None:
vmin = scoreatpercentile(values, 0.1)
if vmax == None:
vmax = scoreatpercentile(values, 99.9)
elif contrast == 'sea':
if sensor_polarisation in ['HH', 'VV']:
if vmin == None:
vmin = 0.
if vmax == None:
vmax = 2.
else:
if vmin == None:
vmin = 1.
if vmax == None:
vmax = 3.
elif contrast == 'ice':
if sensor_polarisation in ['HH', 'VV']:
if vmin == None:
vmin = 0.
if vmax == None:
vmax = 3.5
else:
if vmin == None:
vmin = 1.
if vmax == None:
vmax = 5.
else:
raise Exception('Unknown contrast name.')
print '--> vmin=%f vmax=%f' % (vmin, vmax)
ssr = ssr[::-1, :] # keep SAR orientation for geotiff
geoloc = sarim.get_info('geolocation_grid')
gcplin = (dim[0] * spacing[0] - 1 - geoloc['line'] + 0.5) / spacing[0]
gcppix = (geoloc['pixel'] + 0.5) / spacing[1]
gcplon = geoloc['longitude']
gcplat = geoloc['latitude']
gcphei = geoloc['height']
if gcp2height is not None:
geod = pyproj.Geod(ellps='WGS84')
gcpforw, gcpback, _ = geod.inv(gcplon[:, :-1], gcplat[:, :-1],
gcplon[:, 1:], gcplat[:, 1:])
gcpforw = np.hstack((gcpforw, gcpforw[:, [-1]]))
gcpback = np.hstack((gcpback[:, [0]], gcpback))
gcpinc = geoloc['incidence_angle']
mvdist = (gcp2height - gcphei) / np.tan(np.deg2rad(gcpinc))
mvforw = gcpforw
indneg = np.where(mvdist < 0)
mvdist[indneg] = -mvdist[indneg]
mvforw[indneg] = gcpback[indneg]
_gcplon, _gcplat, _ = geod.fwd(gcplon, gcplat, mvforw, mvdist)
gcplon = _gcplon
gcplat = _gcplat
gcphei.fill(gcp2height)
if gcplon.min() < -135 and gcplon.max() > 135:
gcplon[np.where(gcplon < 0)] += 360.
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
if sensor_polarisation in ['HH', 'VV']:
metadata['product_name'] = 'SAR_roughness'
else:
metadata['product_name'] = 'SAR_roughness_crosspol'
metadata['name'] = dataname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface roughness'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = sarim._mapper._handler.GetGCPProjection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
band = []
scale = (vmax - vmin) / 254.
offset = vmin
indzero = np.where(ssr == 0)
array = np.clip(np.round((ssr - offset) / scale), 0, 254).astype('uint8')
array[indzero] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'sea surface roughness',
'unittype': '',
'nodatavalue': 255,
'parameter_range': [vmin, vmax]
})
# Write
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'sea_surface_roughness'
band[0]['long_name'] = 'sea surface roughness'
band[0]['unittype'] = '1'
metadata['spatial_resolution'] = mspacing.min()
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def sentinel3_olci(infile,
outdir,
vmin=None,
vmax=None,
slope_threshold=-0.00001,
channels='nir',
write_netcdf=False,
lut_path=None,
log_path=None,
lat_crop=85.0):
""""""
t0 = datetime.utcnow()
if type(channels) is list or type(channels) is tuple:
bandnames = channels
product_name = 'Sentinel-3_OLCI'
elif 'nir' == channels:
bandnames = ('Oa17', )
product_name = 'Sentinel-3_OLCI_NIR'
elif 'true_rgb' == channels:
bandnames = ('Oa09', 'Oa06', 'Oa04')
product_name = 'Sentinel-3_OLCI_true_RGB'
elif 'false_rgb' == channels:
bandnames = ('Oa17', 'Oa06', 'Oa04')
product_name = 'Sentinel-3_OLCI_false_RGB'
else:
raise Exception('channels must be either "nir", "true_rgb" '
'or "false_rgb"')
syntool_stats = {}
for bandname in bandnames:
syntool_stats[bandname] = {}
if log_path is not None and not os.path.exists(log_path):
try:
os.makedirs(log_path)
except OSError:
_, e, _ = sys.exc_info()
if e.errno != errno.EEXIST:
raise
# convert band into column number in the LUT
band_columns = [int(x[2:]) + 3 - 1 for x in bandnames]
if vmin is None:
vmin = [None] * len(bandnames)
if vmax is None:
vmax = [None] * len(bandnames)
full_path = os.path.normpath(infile)
file_path = os.path.basename(full_path)
file_name, _ = os.path.splitext(file_path)
geo_path = os.path.join(infile, 'geo_coordinates.nc')
time_path = os.path.join(infile, 'time_coordinates.nc')
quality_path = os.path.join(infile, 'qualityFlags.nc')
# Extract geo coordinates information
geo_handler = netCDF4.Dataset(geo_path, 'r')
nrow = geo_handler.dimensions['rows'].size
nrow_all = nrow
ncell = geo_handler.dimensions['columns'].size
ncell_all = ncell
lon = geo_handler.variables['longitude'][:]
tie_lon = numpy.ma.array(lon)
lat = geo_handler.variables['latitude'][:]
tie_lat = numpy.ma.array(lat)
geo_handler.close()
# Handle longitude continuity
dlon = lon[1:, :] - lon[:-1, :]
if 180.0 <= numpy.max(numpy.abs(dlon)):
lon[lon < 0.0] = lon[lon < 0.0] + 360.0
# Extract time coordinates information
time_handler = netCDF4.Dataset(time_path, 'r')
start_timestamp = time_handler.variables['time_stamp'][0]
end_timestamp = time_handler.variables['time_stamp'][-1]
timestamp_units = time_handler.variables['time_stamp'].units
time_handler.close()
# Format time information
start_time = netCDF4.num2date(start_timestamp, timestamp_units)
end_time = netCDF4.num2date(end_timestamp, timestamp_units)
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='ms')
parameters = ['{} TOA radiance'.format(bnd) for bnd in bandnames]
metadata = {}
metadata['product_name'] = product_name
metadata['name'] = file_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'ESA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = parameters
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'medium-resolution imaging spectrometer'
metadata['sensor_name'] = 'OLCI'
metadata['sensor_platform'] = 'Sentinel-3'
# Crop high latitude to avoid projection issues
LAT_MAX = 89.0
ind_valid_cols = numpy.where(numpy.abs(tie_lat).max(axis=0) <= LAT_MAX)[0]
slice_lat0 = slice(None)
slice_lat1 = slice(numpy.min(ind_valid_cols),
numpy.max(ind_valid_cols) + 1)
tie_lat = tie_lat[slice_lat0, slice_lat1]
tie_lon = tie_lon[slice_lat0, slice_lat1]
nrow, ncell = tie_lat.shape
# Handle longitude continuity
dlon = tie_lon[1:, :] - tie_lon[:-1, :]
if 180.0 <= numpy.max(numpy.abs(dlon)):
lon0 = tie_lon[0, 0] + 180.0
tie_lon[:, :] = numpy.mod(tie_lon[:, :] - lon0, 360.0) + lon0
# Compute GCPs
tie_row = numpy.linspace(0, nrow - 1, num=tie_lon.shape[0])
tie_cell = numpy.linspace(0, ncell - 1, num=tie_lon.shape[1])
tie_facrow = (nrow - 1.) / (tie_lon.shape[0] - 1.)
tie_faccell = (ncell - 1.) / (tie_lon.shape[1] - 1.)
gcp_fac = 128
gcp_fac = numpy.maximum(gcp_fac, numpy.maximum(tie_faccell, tie_facrow))
gcp_nrow = numpy.ceil((nrow - 1.) / gcp_fac).astype('int') + 1
gcp_ncell = numpy.ceil((ncell - 1.) / gcp_fac).astype('int') + 1
tie_indrow = numpy.round(
numpy.linspace(0, tie_lon.shape[0] - 1, num=gcp_nrow)).astype('int')
tie_indcell = numpy.round(
numpy.linspace(0, tie_lon.shape[1] - 1, num=gcp_ncell)).astype('int')
gcp_lon = tie_lon[tie_indrow.reshape((-1, 1)),
tie_indcell.reshape((1, -1))]
gcp_lat = tie_lat[tie_indrow.reshape((-1, 1)),
tie_indcell.reshape((1, -1))]
gcp_row = numpy.tile(tie_row[tie_indrow].reshape((-1, 1)) + 0.5,
(1, gcp_ncell))
gcp_cell = numpy.tile(tie_cell[tie_indcell].reshape((1, -1)) + 0.5,
(gcp_nrow, 1))
gcp_hei = numpy.zeros(gcp_lon.shape)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcp_lon, gcp_lat, gcp_hei,
gcp_cell, gcp_row)
syntool_stats['lon_min'] = float(numpy.min(tie_lon))
syntool_stats['lon_max'] = float(numpy.max(tie_lon))
syntool_stats['lat_min'] = float(numpy.min(tie_lat))
syntool_stats['lat_max'] = float(numpy.max(tie_lat))
# Compute atmospheric radiance TOA correction
L_toa = None
if lut_path is not None:
t_start = datetime.utcnow()
# Compute atmospheric correction for the whole granule
L_toa = atmospheric_correction(lut_path, infile, band_columns,
nrow_all, ncell_all)
# Extract the slices of atmospheric correction that match the ones of
# the bands after the removal of the high latitude columns.
for iband in range(len(L_toa)):
L_toa[iband] = L_toa[iband][slice_lat0, slice_lat1]
# An erroneous LUT could produce nan values for the atmospheric
# correction. This is not acceptable.
if numpy.any(~numpy.isfinite(L_toa[iband])):
raise Exception('Infinite or nan value found in the '
'atmospheric correction, please check that the'
'LUT has valid values for the angles contained'
'in the instrument_data.nc file')
t_stop = datetime.utcnow()
syntool_stats['lut_computation'] = (t_stop - t_start).total_seconds()
logger.info('Construct bands')
t_start = datetime.utcnow()
bands = []
dset = netCDF4.Dataset(quality_path, 'r')
quality_flags = dset.variables['quality_flags'][slice_lat0, slice_lat1]
dset.close()
bits = {
'saturated@Oa21': 0,
'saturated@Oa20': 1,
'saturated@Oa19': 2,
'saturated@Oa18': 3,
'saturated@Oa17': 4,
'saturated@Oa16': 5,
'saturated@Oa15': 6,
'saturated@Oa14': 7,
'saturated@Oa13': 8,
'saturated@Oa12': 9,
'saturated@Oa11': 10,
'saturated@Oa10': 11,
'saturated@Oa09': 12,
'saturated@Oa08': 13,
'saturated@Oa07': 14,
'saturated@Oa06': 15,
'saturated@Oa05': 16,
'saturated@Oa04': 17,
'saturated@Oa03': 18,
'saturated@Oa02': 19,
'saturated@Oa01': 20,
'dubious': 21,
'sun-glint_risk': 22,
'duplicated': 23,
'cosmetic': 24,
'invalid': 25,
'straylight_risk': 26,
'bright': 27,
'tidal_region': 28,
'fresh_inland_water': 29,
'coastline': 30,
'land': 31
}
off_flags = numpy.uint32(0)
off_flags = off_flags + numpy.uint32(1 << bits['dubious'])
off_flags = off_flags + numpy.uint32(1 << bits['invalid'])
off_flags = off_flags + numpy.uint32(1 << bits['straylight_risk'])
off_flags = off_flags + numpy.uint32(1 << bits['bright'])
off_flags = off_flags + numpy.uint32(1 << bits['tidal_region'])
# off_flags = off_flags + numpy.uint32(1 << bits['fresh_inland_water'])
off_flags = off_flags + numpy.uint32(1 << bits['coastline'])
off_flags = off_flags + numpy.uint32(1 << bits['land'])
# Filter out values where any of the bands is flagged as saturated
for bandname in bandnames:
bit_name = 'saturated@{}'.format(bandname)
off_flags = off_flags + numpy.uint32(1 << bits[bit_name])
lat_mask = (numpy.abs(tie_lat) > lat_crop)
data_mask = numpy.zeros(quality_flags.shape, dtype='bool')
data_mask = (data_mask | lat_mask)
if numpy.all(data_mask):
logger.warn('No data to extract.')
sys.exit(0)
contrast_mask = numpy.zeros(quality_flags.shape, dtype='bool')
contrast_mask = (contrast_mask | lat_mask)
contrast_mask = (contrast_mask |
(numpy.bitwise_and(quality_flags, off_flags) > 0))
t_stop = datetime.utcnow()
syntool_stats['mask_computation'] = (t_stop - t_start).total_seconds()
t_start = datetime.utcnow()
_vmin = list(vmin)
_vmax = list(vmax)
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
logger.info('\tConstruct {} band'.format(bandname))
fieldname = '{}_radiance'.format(bandname)
file_path = os.path.join(infile, '{}.nc'.format(fieldname))
f_handler = netCDF4.Dataset(file_path, 'r')
band = f_handler.variables[fieldname][:]
band = numpy.ma.array(band)
f_handler.close()
# Apply atmospheric correction
band = band[slice_lat0, slice_lat1]
if L_toa is not None:
band -= L_toa[band_index][:, :]
# Mask null and negative values: they are inferior or equal to
# atmospheric correction and should probably have be flagged as clouds.
mask_negative = (band <= 0.0)
logger.info('\tSet contrast')
valid_ratio_lower_threshold = 0.001 # 0.1%
valid_data_mask = (band.mask | contrast_mask | mask_negative)
valid_data = band.data[~valid_data_mask]
valid_ratio = float(valid_data.size) / float(band.data.size)
syntool_stats[bandname]['valid_ratio'] = valid_ratio
if valid_ratio_lower_threshold >= valid_ratio:
logger.warn('No valid values for {}'.format(bandname))
logger.warn('Using default min/max.')
# Use arbitrary extrema on land
if _vmin[band_index] is None:
_min = default_minmax[bandname][0]
_vmin[band_index] = _min
syntool_stats[bandname]['default_min'] = float(_min)
if _vmax[band_index] is None:
_max = default_minmax[bandname][1]
_vmax[band_index] = _max
syntool_stats[bandname]['default_max'] = float(_max)
else:
# TODO: add clipping for NIR
# _min = numpy.clip(_min, 1.5, 10.0)
# _max = numpy.clip(_max, 30.0, 60.0)
if _vmin[band_index] is None:
_min = numpy.percentile(valid_data, .5)
_vmin[band_index] = _min
syntool_stats[bandname]['p0050'] = float(_min)
syntool_stats[bandname]['min'] = float(numpy.min(valid_data))
if _vmax[band_index] is None:
_max = numpy.percentile(valid_data, 99.99)
_vmax[band_index] = _max
p99 = numpy.percentile(valid_data, 99.0)
syntool_stats[bandname]['p9900'] = float(p99)
syntool_stats[bandname]['p9999'] = float(_max)
syntool_stats[bandname]['max'] = float(numpy.max(valid_data))
logger.info('\tContrast : vmin={} / vmax={}'.format(
_vmin[band_index], _vmax[band_index]))
min_values = [_vmin[band_index] for band_index in range(len(bandnames))]
max_values = [_vmax[band_index] for band_index in range(len(bandnames))]
t_stop = datetime.utcnow()
syntool_stats['minmax_computation'] = (t_stop - t_start).total_seconds()
syntool_stats['final_min'] = float(numpy.min(min_values))
syntool_stats['final_max'] = float(numpy.max(max_values))
_min = numpy.log(numpy.min(min_values))
_max = numpy.log(numpy.max(max_values))
scale = (_max - _min) / 254.
offset = _min
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
logger.info('\tConstruct {} band'.format(bandname))
fieldname = '{}_radiance'.format(bandname)
file_path = os.path.join(infile, '{}.nc'.format(fieldname))
f_handler = netCDF4.Dataset(file_path, 'r')
band = f_handler.variables[fieldname][:]
band = numpy.ma.array(band)
f_handler.close()
# Apply atmospheric correction
band = band[slice_lat0, slice_lat1]
if L_toa is not None:
band -= L_toa[band_index][:, :]
# Mask null and negative values: they are inferior or equal to
# atmospheric correction and should probably have be flagged as clouds.
mask_negative = (band <= 0.0)
# Compute the logarithm only for radiance values that are higher than
# the atmospheric correction.
bnd = numpy.log(band.data, where=(~mask_negative))
logger.info('\tBytescaling')
byte = bytescale(bnd, cmin=_min, cmax=_max, low=0, high=254)
description = '{} radiance (log)'.format(bandname)
if band.mask is not numpy.ma.nomask:
byte[band.mask] = 255
# mask data for extreme latitudes
byte[data_mask] = 255
# Pixels with a radiance equal or inferior to atmospheric correction
# are clipped to the minimal value.
if 0 < mask_negative.size:
byte[numpy.where(mask_negative == True)] = 0 # noqa
band_range = [_vmin[band_index], _vmax[band_index]]
bands.append({
'array': byte,
'scale': scale,
'offset': offset,
'description': description,
'unittype': '',
'nodatavalue': 255,
'parameter_range': band_range
})
if write_netcdf:
bands[-1]['name'] = bandname
bands[-1]['long_name'] = bandname
bands[-1]['unittype'] = '1'
logger.info('Make sure nodata are at the same locations in all bands')
mask = numpy.any([_band['array'] == 255 for _band in bands], axis=0)
for band in bands:
band['array'][mask] = 255
t_stop = datetime.utcnow()
syntool_stats['bytescaling'] = (t_stop - t_start).total_seconds()
if write_netcdf:
metadata['spatial_resolution'] = 300
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
bands,
'swath',
ngcps=gcp_lon.shape)
else:
logger.info('Write geotiff')
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, bands)
logger.info(datetime.utcnow() - t0)
syntool_stats['total_time'] = (datetime.utcnow() - t0).total_seconds()
if log_path is not None:
import json
stats_path = os.path.join(log_path, '{}.json'.format(file_name))
with open(stats_path, 'w') as f:
json.dump(syntool_stats, f)
def oscar_current(infile,
outdir,
date=None,
vmin=0.,
vmax=5.08,
vmin_pal=0.,
vmax_pal=2.,
write_netcdf=True):
""" """
if date is None:
raise Exception('oscar_current conversion needs a date !')
# Read/Process data
print 'Read/Process data'
ncfile = Dataset(infile)
time = ncfile.variables['time']
time_index = np.where(num2date(time[:], time.units) == date)[0]
if time_index.size != 1:
raise Exception('Date not found in oscar file !')
time_index = time_index[0]
dtime = num2date(time[time_index], time.units)
lat = ncfile.variables['latitude'][:]
lon = ncfile.variables['longitude'][:]
lon_index = np.where(lon < lon[0] + 360.)[0]
lon = lon[lon_index]
ucur = ncfile.variables['u'][time_index, 0, :, lon_index]
vcur = ncfile.variables['v'][time_index, 0, :, lon_index]
# ucur = ncfile.variables['um'][time_index, 0, :, lon_index]
# vcur = ncfile.variables['vm'][time_index, 0, :, lon_index]
if isinstance(ucur, np.ma.MaskedArray) == False:
ucur = np.ma.masked_invalid(ucur)
if isinstance(vcur, np.ma.MaskedArray) == False:
vcur = np.ma.masked_invalid(vcur)
gt180_index = np.where(lon > 180)
lon[gt180_index] -= 360
shift = lon.size - gt180_index[0][0]
lon = np.roll(lon, shift, axis=0)
ucur = np.roll(ucur, shift, axis=1)
vcur = np.roll(vcur, shift, axis=1)
lon = np.insert(lon, 0, -180., axis=0)
ucur = np.ma.concatenate((ucur[:, [-1]], ucur), axis=1)
vcur = np.ma.concatenate((vcur[:, [-1]], vcur), axis=1)
#ucur = np.insert(ucur, 0, ucur[:, -1], axis=1)
#vcur = np.insert(vcur, 0, vcur[:, -1], axis=1)
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
metadata['product_name'] = 'OSCAR_current'
metadata['name'] = os.path.splitext(os.path.basename(infile))[0] + '_' +\
dtime.strftime('%Y%m%dT%H%M%S')
metadata['datetime'] = stfmt.format_time(dtime)
metadata['time_range'] = ['-60h', '+60h']
metadata['source_URI'] = infile
metadata['source_provider'] = 'NOAA'
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = ['current velocity', 'current direction']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
dlon = lon[1] - lon[0]
dlat = lat[1] - lat[0]
geolocation['geotransform'] = [
lon[0] - dlon / 2., dlon, 0, lat[0] - dlat / 2., 0, dlat
]
band = []
mask = np.ma.getmaskarray(ucur) | np.ma.getmaskarray(vcur)
curvel = np.sqrt(ucur.data**2 + vcur.data**2)
curdir = np.mod(
np.arctan2(vcur.data, ucur.data) * 180. / np.pi + 360., 360.)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(curvel, vmin, vmax, out=curvel)
array = np.round((curvel - offset) / scale).astype('uint8')
array[mask] = 255
colortable = stfmt.format_colortable('matplotlib_jet',
vmin=vmin,
vmax=vmax,
vmin_pal=vmin_pal,
vmax_pal=vmax_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'current velocity',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
array = np.round(curdir / 360. * 254.).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': 360. / 254.,
'offset': 0.,
'description': 'current direction',
'unittype': 'deg',
'nodatavalue': 255,
'parameter_range': [0, 360.]
})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
elif write_netcdf == True:
print 'Write netcdf'
# u/v -> bands
band = []
mask = np.ma.getmaskarray(ucur) | np.ma.getmaskarray(vcur)
vmin = -vmax
offset, scale = vmin, (vmax - vmin) / 254.
u = np.clip(ucur.data, vmin, vmax)
array = np.round((u - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'current u',
'unittype': 'm s-1',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'name': 'u',
'long_name': 'Ocean Surface Zonal Currents'
})
v = np.clip(vcur.data, vmin, vmax)
array = np.round((v - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'current v',
'unittype': 'm s-1',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'name': 'v',
'long_name': 'Ocean Surface Meridional Currents'
})
# Write
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
metadata['spatial_resolution'] = min([abs(dlat), abs(dlon)]) * 111000.
dgcps = np.round(1. / np.abs(np.array([dlat, dlon]))).astype('int')
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'grid_lonlat',
dgcps=dgcps)