def sar_doppler(infile, outdir):
"""
"""
# tmp
#infile = '/local/home/fab/data/sar/ASA/agulhas/ASA_WSM_1PNPDE20110518_210602_000002143102_00330_48189_1274/SAR_doppler.nc'
# infile = '/local/home/fab/data/sar/ASA/agulhas/ASA_WSM_1PNPDE20110824_211403_000002143106_00014_49597_2093/SAR_doppler.nc'
# outdir = '/local/home/data/syntool_inputs'
# /tmp
# Read/Process data
print 'Read/Process data'
sardop = NCFile(infile)
product_ref = sardop.read_global_attribute('SOURCE_PRODUCT_REF')
start_time = sardop.read_global_attribute('SOURCE_START_DATE')
start_time = datetime.strptime(start_time, '%Y%m%d%H%M%S.%f')
duration = sardop.read_global_attribute('SOURCE_ACQ_DURATION')
stop_time = start_time + timedelta(seconds=duration)
polarisation = sardop.read_global_attribute('SOURCE_POLARIZATION')
lon = sardop.read_values('longitude')[::-1, :]
lat = sardop.read_values('latitude')[::-1, :]
#dopano = sardop.read_values('dopanomaly')[::-1, :]
radvel = sardop.read_values('radial_vel')[::-1, :]
validity = sardop.read_values('validity')[::-1, :]
track_angle = sardop.read_global_attribute('SOURCE_TRACK_ANGLE')
if track_angle < 0:
radvel *= -1
shp = lon.shape
nlines = np.ceil(shp[0] / 4.) + 1
lines = np.round(np.linspace(0, shp[0] - 1, num=nlines)).astype('int32')
npixels = np.ceil(shp[1] / 4) + 1
pixels = np.round(np.linspace(0, shp[1] - 1, num=npixels)).astype('int32')
gcplin = np.tile(lines.reshape(nlines, 1), (1, npixels))
gcppix = np.tile(pixels.reshape(1, npixels), (nlines, 1))
gcplon = lon[gcplin, gcppix]
gcplat = lat[gcplin, gcppix]
gcphei = np.zeros((nlines, npixels))
gcppix = gcppix + 0.5
gcplin = gcplin + 0.5
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
metadata['product_name'] = 'SAR_doppler'
metadata['name'] = product_ref
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'ESA'
metadata['processing_center'] = 'CLS'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'radial horizontal velocities'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = 'ASAR'
metadata['sensor_platform'] = 'ENVISAT'
metadata['sensor_mode'] = 'WSM'
#metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = polarisation
#metadata['sensor_pass'] = sensor_pass
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection(geogcs='WGS84')
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
# band = []
# scale = (vmax-vmin)/254.
# offset = vmin
# indzero = np.where(validity == 0)
# array = np.clip(np.round((radvel-offset)/scale), 0, 254).astype('uint8')
# array[indzero] = 255
# band.append({'array':array, 'scale':scale, 'offset':offset,
# 'description':'radial horizontal velocities', 'unittype':'m/s',
# 'nodatavalue':255, 'parameter_range':[vmin, vmax]})
band = []
cmap = doppler_colormap()
norm = Normalize(vmin=-2.5, vmax=2.5)
rgb = cmap(norm(radvel))
indnodata = np.where(validity == 0)
for ich in range(3):
channel = np.round(rgb[:, :, ich] * 255).astype('uint8')
channel[indnodata] = 0
band.append({'array': channel, 'nodatavalue': 0})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
def read_geometry(infile, bandnames, fname, sltype, view, product_name, vmin,
vmax, log_path):
""" Read coordinates from geometry SLSTR file """
# Initiate log file
syntool_stats = {}
for bandname in bandnames:
syntool_stats[bandname] = {}
if log_path is not None and not os.path.exists(log_path):
try:
os.makedirs(log_path)
except OSError:
_, e, _ = sys.exc_info()
if e.errno != errno.EEXIST:
raise
if vmin is None:
vmin = [None] * len(bandnames)
if vmax is None:
vmax = [None] * len(bandnames)
full_path = os.path.normpath(infile)
file_path = os.path.basename(full_path)
file_name, _ = os.path.splitext(file_path)
geo_filename = 'geodetic_{}{}.nc'.format(sltype, view)
geo_path = os.path.join(infile, geo_filename)
lat_varname = 'latitude_{}{}'.format(sltype, view)
lon_varname = 'longitude_{}{}'.format(sltype, view)
# Extract geo coordinates information
geo_handler = netCDF4.Dataset(geo_path, 'r')
nrow = geo_handler.dimensions['rows'].size
nrow_all = nrow
ncell = geo_handler.dimensions['columns'].size
ncell_all = ncell
lon = geo_handler.variables[lon_varname][:]
tie_lon = numpy.ma.array(lon)
tie_lon._sharedmask = False
lat = geo_handler.variables[lat_varname][:]
tie_lat = numpy.ma.array(lat)
start_time_str = geo_handler.start_time
stop_time_str = geo_handler.stop_time
geo_handler.close()
# Handle longitude range
dlon = tie_lon[1:, :] - tie_lon[:-1, :]
if 180.0 <= numpy.max(numpy.abs(dlon)):
tie_lon[tie_lon < 0.0] = tie_lon[tie_lon < 0.0] + 360.0
# Extract time coordinates information
# Format time information
start_time = datetime.strptime(start_time_str, "%Y-%m-%dT%H:%M:%S.%fZ")
end_time = datetime.strptime(stop_time_str, "%Y-%m-%dT%H:%M:%S.%fZ")
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='ms')
month = start_time.month
parameters = ['{} TOA {}'.format(bnd, fname) for bnd in bandnames]
metadata = {}
metadata['product_name'] = '{}'.format(product_name)
metadata['name'] = file_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'ESA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = parameters
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'dual scan temperature radiometer'
metadata['sensor_name'] = 'SLSTR'
metadata['sensor_platform'] = 'Sentinel-3'
# Crop high latitude to avoid projection issues
LAT_MAX = 89.0
ind_valid_cols = numpy.where(numpy.abs(tie_lat).max(axis=0) <= LAT_MAX)[0]
slice_lat0 = slice(None)
slice_lat1 = slice(numpy.min(ind_valid_cols),
numpy.max(ind_valid_cols) + 1)
tie_lat = tie_lat[slice_lat0, slice_lat1]
tie_lon = tie_lon[slice_lat0, slice_lat1]
nrow, ncell = tie_lat.shape
# Handle longitude continuity
dlon = tie_lon[1:, :] - tie_lon[:-1, :]
if 180.0 <= numpy.max(numpy.abs(dlon)):
lon0 = tie_lon[0, 0] + 180.0
tie_lon._sharedmask = False
tie_lon[:, :] = numpy.mod(tie_lon[:, :] - lon0, 360.0) + lon0
# Compute GCPs
tie_row = numpy.linspace(0, nrow - 1, num=tie_lon.shape[0])
tie_cell = numpy.linspace(0, ncell - 1, num=tie_lon.shape[1])
tie_facrow = (nrow - 1.) / (tie_lon.shape[0] - 1.)
tie_faccell = (ncell - 1.) / (tie_lon.shape[1] - 1.)
gcp_fac = 128
gcp_fac = numpy.maximum(gcp_fac, numpy.maximum(tie_faccell, tie_facrow))
gcp_nrow = numpy.ceil((nrow - 1.) / gcp_fac).astype('int') + 1
gcp_ncell = numpy.ceil((ncell - 1.) / gcp_fac).astype('int') + 1
tie_indrow = numpy.round(
numpy.linspace(0, tie_lon.shape[0] - 1, num=gcp_nrow)).astype('int')
tie_indcell = numpy.round(
numpy.linspace(0, tie_lon.shape[1] - 1, num=gcp_ncell)).astype('int')
gcp_lon = tie_lon[tie_indrow.reshape((-1, 1)),
tie_indcell.reshape((1, -1))]
gcp_lat = tie_lat[tie_indrow.reshape((-1, 1)),
tie_indcell.reshape((1, -1))]
gcp_row = numpy.tile(tie_row[tie_indrow].reshape((-1, 1)) + 0.5,
(1, gcp_ncell))
gcp_cell = numpy.tile(tie_cell[tie_indcell].reshape((1, -1)) + 0.5,
(gcp_nrow, 1))
gcp_hei = numpy.zeros(gcp_lon.shape)
ngcps = gcp_lon.shape
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcp_lon, gcp_lat, gcp_hei,
gcp_cell, gcp_row)
syntool_stats['lon_min'] = float(numpy.min(tie_lon))
syntool_stats['lon_max'] = float(numpy.max(tie_lon))
syntool_stats['lat_min'] = float(numpy.min(tie_lat))
syntool_stats['lat_max'] = float(numpy.max(tie_lat))
return (syntool_stats, metadata, geolocation, tie_lon, tie_lat, slice_lat0,
slice_lat1, nrow_all, ncell_all, ngcps, month)
def viirs_chlora(infileid,
outdir,
download_dir='/tmp',
vmin=None,
vmax=None,
contrast='relative',
ngcps=(26, 32),
open_iterations=1,
nprocs=1,
pngkml=False,
write_netcdf=False):
"""
"""
if contrast == 'med':
listbox = [[-6., 35., 2.75, 42.48], [2.74, 30, 42.2, 47.00]]
elif contrast == 'cwe':
listbox = [[-23., 35.2, -5.5, 42.88], [-23., 42.8, 2.20, 51.]]
elif contrast == 'nwe':
listbox = [[-23., 50.8, 32.7, 68.]]
elif contrast == 'gom':
listbox = [[-98., 18.0, -80.5, 30.5]]
elif contrast == 'agulhas':
listbox = [[10.8437, -45.7404, 39.9799, -25.3019]]
elif contrast == 'gs':
listbox = [[-81.52, 20, -30, 45]]
else:
listbox = None
# Search/Download data
print 'Search/Download data'
if re.match(r'^V[0-9]{13}$', infileid) is None:
raise Exception('Input for viirs_chlora is an ID '
'(e.g. V2014093110000)')
product_id = 'VIIRSL2OC'
date = datetime.strptime(infileid[1:], '%Y%j%H%M%S')
viirsocfname = viirs.search_and_download(product_id, date, download_dir)
# Read/Process data
print 'Read/Process data'
# Read from OC file
viirsocfile = viirs.VIIRSL2File(viirsocfname)
lon = viirsocfile.read_lon()
lat = viirsocfile.read_lat()
chlora = viirsocfile.read_chlora()
attrs = viirsocfile.read_attributes()
viirsocfile.close()
if listbox is not None:
mask_box = np.zeros(np.shape(chlora.data))
for i in range(np.shape(listbox)[0]):
index_in = np.where((lon >= listbox[i][0]) & (lat >= listbox[i][1])
& (lon <= listbox[i][2])
& (lat <= listbox[i][3]))
mask_box[index_in] = 1
mask = (mask_box == 0) | ma.getmaskarray(chlora)
else:
mask = ma.getmaskarray(chlora)
if mask.all():
print 'No data'
sys.exit(0)
# GCPs for resampling and geotiff georeference
scansize = 16
dtime0 = datetime.utcnow()
gcps = resample.get_gcps_from_bowtie(lon, lat, scansize, ngcps=ngcps)
dtime = datetime.utcnow() - dtime0
print 'Get GCPs from bowtie swath : {}'.format(dtime)
gcplon, gcplat, gcpnpixel, gcpnline = gcps
rspysize = lon.shape[0]
geod = pyproj.Geod(ellps='WGS84')
mid = abs(gcpnline[:, 0] - 0.5).argmin()
xdists = geod.inv(gcplon[mid, :-1], gcplat[mid, :-1], gcplon[mid, 1:],
gcplat[mid, 1:])[2]
xdist = np.sum(xdists) / abs(gcpnpixel[mid, -1] - gcpnpixel[mid, 0])
rspxsize = np.round(xdist / 750.).astype('int') + 1
gcpline = gcpnline * rspysize
gcppixel = gcpnpixel * rspxsize
# Resample with LinearNDInterpolator in output space
dtime0 = datetime.utcnow()
pix, lin = resample.get_points_from_gcps(gcplon,
gcplat,
gcppixel,
gcpline,
rspxsize,
rspysize,
1,
lon,
lat,
nprocs=nprocs) - 0.5
dtime = datetime.utcnow() - dtime0
print 'Get input coordinates in new grid : {}'.format(dtime)
# Test input grid in output space
# import matplotlib.pyplot as plt
# for iscan in range(lon.shape[0] / scansize):
# pixscan = pix[iscan * scansize: (iscan+1) * scansize, :]
# linscan = lin[iscan * scansize: (iscan+1) * scansize, :]
# # maskscan = mask[iscan * scansize: (iscan+1) * scansize, :]
# # pixscan = pixscan[~maskscan]
# # linscan = linscan[~maskscan]
# plt.plot(pixscan.flatten(), linscan.flatten(), '+')
# plt.show()
# import pdb ; pdb.set_trace()
# \Test input grid in output space
dtime0 = datetime.utcnow()
chlora.data[mask] = np.nan
rspchlora = resample.resample_bowtie_linear(pix,
lin,
chlora.data,
scansize,
rspxsize,
rspysize,
show=False)
rspmask = ma.getmaskarray(rspchlora)
dtime = datetime.utcnow() - dtime0
print 'Interpolate in new grid : {}'.format(dtime)
# Take log and open mask
finalchlora = ma.log(rspchlora)
finalmask = ~binary_opening(
~rspmask, structure=np.ones((3, 3)), iterations=open_iterations)
finalchlora.mask = finalmask
# Contrast
if vmin == None:
if contrast == 'relative':
vmin = np.percentile(finalchlora.compressed(), 0.5)
elif contrast == 'agulhas':
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
vmin = -0.5 * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) - 3.
elif contrast == 'med' or contrast == 'nwe' or contrast == 'cwe':
vmin = np.percentile(finalchlora.compressed(), 2)
else:
raise Exception('Unknown contrast : {}'.format(contrast))
else:
if vmin != 0:
vmin = math.log(vmin)
else:
vmin = np.percentile(finalchlora.compressed(), 0.5)
if vmax == None:
if contrast == 'relative':
vmax = np.percentile(finalchlora.compressed(), 99.5)
elif contrast == 'agulhas':
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
vmax = 0.5 * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 3.
elif contrast == 'med':
vmax = np.percentile(finalchlora.compressed(), 98)
elif contrast == 'nwe':
vmax = np.percentile(finalchlora.compressed(), 98)
elif contrast == 'cwe':
vmax = np.percentile(finalchlora.compressed(), 98)
else:
raise Exception('Unknown contrast : {}'.format(contrast))
else:
if vmax != 0:
vmax = math.log(vmax)
else:
vmax = np.percentile(finalchlora.compressed(), 98)
# Flip (geotiff in "swath sense")
finalchlora = finalchlora[::-1, ::-1]
gcppixel = rspxsize - gcppixel
gcpline = rspysize - gcpline
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(attrs['start_time'],
attrs['stop_time'],
units='ms')
metadata['product_name'] = 'Chlorophyll_a_concentration_VIIRS'
if contrast == 'relative':
metadata['name'] = os.path.splitext(os.path.basename(viirsocfname))[0]
else:
metadata['name'] = '{}_{}'.format(
os.path.splitext(os.path.basename(viirsocfname))[0], contrast)
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = viirsocfname
metadata['source_provider'] = 'NOAA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'chlorophyll a concentration'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'radiometer'
metadata['sensor_name'] = 'VIIRS'
metadata['sensor_platform'] = 'Suomi-NPP'
metadata['sensor_pass'] = attrs['pass']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
gcpheight = np.zeros(gcppixel.shape)
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcpheight,
gcppixel, gcpline)
band = []
indndv = np.where(ma.getmaskarray(finalchlora) == True)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(finalchlora.data, vmin, vmax, out=finalchlora.data)
array = np.round((finalchlora.data - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('chla_jet',
vmax=vmax,
vmax_pal=vmax,
vmin=vmin,
vmin_pal=vmin)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'chlorophyll a concentration',
'unittype': 'log(mg/m3)',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
if write_netcdf == False:
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'chlor_a'
band[0]['long_name'] = 'Chlorophyll Concentration, OCI Algorithm'
band[0][
'standard_name'] = 'mass_concentration_chlorophyll_concentration_in_sea_water'
band[0]['unittype'] = 'mg m^-3 (log)'
# ymid = abs(gcpline[:, 0] - rspysize / 2.).argmin()
# xdists = geod.inv(gcplon[ymid, :-1], gcplat[ymid, :-1],
# gcplon[ymid, 1:], gcplat[ymid, 1:])[2] / \
# np.abs(gcppixel[ymid, 1:] - gcppixel[ymid, :-1])
# xmid = abs(gcppixel[0, :] - rspxsize / 2.).argmin()
# ydists = geod.inv(gcplon[:-1, xmid], gcplat[:-1, xmid],
# gcplon[1:, xmid], gcplat[1:, xmid])[2] / \
# np.abs(gcpline[1:, xmid] - gcpline[:-1, xmid])
# print xdists.min(), xdists.max(), xdists.mean()
# # e.g. 749.810419844 749.810438261 749.810429577
# print ydists.min(), ydists.max(), ydists.mean()
# # e.g. 737.874499629 739.856423757 738.87317625
metadata['spatial_resolution'] = 750.
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def sar_doppler_exp(infile,
outdir,
pngkml=False,
vmin=-2.5,
vmax=2.5,
vmin_pal=-2.5,
vmax_pal=2.5):
"""
"""
# Read/Process data
print 'Read/Process data'
sardop = Dataset(infile)
mission = sardop.MISSIONNAME
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
else:
raise Exception('S1A mission expected.')
doptime = sardop.variables['rvlZeroDopplerTime'][:]
start_time = datetime.strptime(''.join(list(doptime[0, 0, :])),
'%Y-%m-%dT%H:%M:%S.%f')
stop_time = datetime.strptime(''.join(list(doptime[-1, -1, :])),
'%Y-%m-%dT%H:%M:%S.%f')
heading = sardop.variables['rvlHeading'][:]
if np.sin((90 - heading.mean()) * np.pi / 180) > 0:
sensor_pass = '******'
else:
sensor_pass = '******'
# safe_name = os.path.basename(os.path.dirname(os.path.dirname(infile)))
# sensor_mode = safe_name.split('_')[1]
# if sensor_mode not in ['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'IW', 'EW']:
# raise Exception('S[1-6]/IW/EW modes expected.')
# sensor_swath = os.path.basename(infile).split('-')[1].upper()
# sensor_polarisation = sardop.read_global_attribute('polarisation')
# datagroup = safe_name.replace('.SAFE', '')
# pid = datagroup.split('_')[-1]
# dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
dataname = os.path.splitext(os.path.basename(infile))[0]
sensor_mode = dataname.split('_')[1]
sensor_swath = sensor_mode
sensor_polarisation = sardop.POLARISATION
radvel = sardop.variables['rvlRadVel'][:]
sweepangle = sardop.variables['rvlSweepAngle'][:]
radvel = descalloping(radvel, sweepangle)
radvel = smooth(radvel)
inc = sardop.variables['rvlIncidenceAngle'][:]
radvel /= np.sin(np.deg2rad(inc))
#landflag = sardop.variables['rvlLandFlag'][:]
lon = sardop.variables['rvlLon'][:]
lat = sardop.variables['rvlLat'][:]
if sensor_pass == 'Ascending':
radvel *= -1
ngcps = np.ceil(np.array(lon.shape) / 10.) + 1
pix = np.linspace(0, lon.shape[1] - 1,
num=ngcps[1]).round().astype('int32')
lin = np.linspace(0, lon.shape[0] - 1,
num=ngcps[0]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
if gcplon.min() < -135 and gcplon.max() > 135:
gcplon[np.where(gcplon < 0)] += 360.
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
metadata['product_name'] = 'SAR_doppler_exp'
metadata['name'] = dataname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'radial horizontal velocities'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
# metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
band = []
#indndv = np.where(landflag != 0)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(radvel, vmin, vmax, out=radvel)
array = np.round((radvel - offset) / scale).astype('uint8')
#array[indndv] = 255
colortable = stfmt.format_colortable('doppler',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'radial horizontal velocities',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
def sar_wind(infile,
outdir,
pngkml=False,
valid_percent_min=1.,
vmin=0.,
vmax=25.4,
vmin_pal=0.,
vmax_pal=50 * 0.514):
"""
"""
# Read/Process data
print 'Read/Process data'
sarwind = SAFEOCNNCFile(infile, product='WIND')
mission = sarwind.read_global_attribute('missionName')
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
elif mission == 'S1B':
sensor_name = 'Sentinel-1B'
sensor_platform = 'Sentinel-1B'
source_provider = 'ESA'
else:
raise Exception('S1A/S1B missions expected.')
start_time = sarwind.get_start_time()
stop_time = sarwind.get_end_time()
heading = sarwind.read_values('owiHeading')
if np.sin((90 - heading[0, 0]) * np.pi / 180) > 0:
sensor_pass = '******'
else:
sensor_pass = '******'
safe_name = os.path.basename(os.path.dirname(os.path.dirname(infile)))
sensor_mode = safe_name.split('_')[1]
if sensor_mode not in ['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'IW', 'EW']:
raise Exception('S[1-6]/IW/EW modes expected.')
sensor_swath = os.path.basename(infile).split('-')[1].upper()
sensor_polarisation = sarwind.read_global_attribute('polarisation')
datagroup = safe_name.replace('.SAFE', '')
pid = datagroup.split('_')[-1]
dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
windspeed = sarwind.read_values('owiWindSpeed')
if windspeed.shape == (1, 1):
raise Exception('owiRaSize and owiAzSize equals 1 !')
winddirection = sarwind.read_values('owiWindDirection')
landflag = sarwind.read_values('owiLandFlag')
inversionquality = sarwind.read_values('owiInversionQuality')
windquality = sarwind.read_values('owiWindQuality')
#pbright = sarwind.read_values('owiPBright')
lon = sarwind.read_values('lon')
lat = sarwind.read_values('lat')
if np.ma.is_masked(lon) or np.ma.is_masked(lat):
raise Exception('Some lon and/or lat is masked.')
if np.all(lon == 0) or np.all(lat == 0):
raise Exception('All lon and/or lat set to 0.')
ngcps = np.ceil(np.array(lon.shape) / 10.).astype('int') + 1
pix = np.linspace(0, lon.shape[1] - 1,
num=ngcps[1]).round().astype('int32')
lin = np.linspace(0, lon.shape[0] - 1,
num=ngcps[0]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
## Make sure lon are continuous (no jump because of IDL crossing)
## (if IDL crossing, by convention we make lon to be around 180deg)
# if gcplon.min() < -135 and gcplon.max() > 135:
# gcplon[np.where(gcplon < 0)] += 360.
gcplonmid = gcplon[ngcps[0] / 2, ngcps[1] / 2]
gcplon = np.mod(gcplon - (gcplonmid - 180.), 360.) + gcplonmid - 180.
gcplonmin = gcplon.min()
gcplon = gcplon - np.floor((gcplonmin + 180.) / 360.) * 360.
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
metadata['product_name'] = 'SAR_wind'
metadata['name'] = dataname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = ''
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = ['wind speed', 'wind direction']
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
band = []
#mask = landflag != 0
mask = (landflag != 0) | \
((windspeed == 0) & (winddirection == 180)) | \
((windspeed == 0) & (windquality == 3)) | \
((windspeed == 0) & (inversionquality == 2))
mask = np.ma.getdata(mask) # we don't want to sum on a masked mask
valid_percent = np.sum(~mask) / float(mask.size) * 100
if valid_percent <= valid_percent_min:
raise Exception(
'Not enough valid data: {:0.3f}%'.format(valid_percent))
# if np.all(mask):
# raise Exception('Data is fully masked !')
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(windspeed, vmin, vmax, out=windspeed)
array = np.round((windspeed - offset) / scale).astype('uint8')
array[mask] = 255
colortable = stfmt.format_colortable('noaa_wind',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind speed',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
winddirection = np.mod(90. - winddirection + 180., 360.)
array = np.round(winddirection / 360. * 254.).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': 360. / 254.,
'offset': 0.,
'description': 'wind direction',
'unittype': 'deg',
'nodatavalue': 255,
'parameter_range': [0, 360.]
})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
def viirs_sst(infile, outdir, vmin=None, vmax=None, contrast='relative',
ngcps=(41, 32), denoise_kernel='boxcar', denoise_width=27,
open_iterations=1, nprocs=1,
pngkml=False, write_netcdf=False, file_range=None):
"""
"""
dic = None
# Check file containing ranges
if file_range is not None:
if not os.path.isfile(file_range):
raise Exception('file_range {} not found'.format(file_range))
# Read a txt file which contains three columns: yearday,vmin,vmax
with open(file_range, 'r') as f:
dic = {}
for line in f:
(fdoy, fmin, fmax) = line.split(',')
dic[int(fdoy)] = (float(fmin), float(fmax))
if contrast == 'med':
listbox = [[-6., 35., 2.75, 42.48],
[2.74, 30, 42.2, 47.00]]
elif contrast == 'cwe':
listbox = [[-23., 35.2, -5.5, 42.88],
[-23., 42.8, 2.20, 51.]]
elif contrast == 'nwe':
listbox = [[-23., 50.8, 32.7, 68.]]
elif contrast == 'gom':
listbox = [[-98., 18.0, -80.5, 30.5]]
elif contrast == 'agulhas':
listbox = [[10.8437, -45.7404, 39.9799, -25.3019]]
elif contrast == 'gs':
listbox = [[-81.52, 20, -30, 45]]
else:
listbox = None
# Read/Process data
print 'Read/Process data'
dataset = Dataset(infile)
start_time = datetime.strptime(dataset.start_time, '%Y%m%dT%H%M%SZ')
print start_time.day
print start_time.month
stop_time = datetime.strptime(dataset.stop_time, '%Y%m%dT%H%M%SZ')
lon = dataset.variables['lon'][:, :]
lat = dataset.variables['lat'][:, :]
sst = np.ma.array(dataset.variables['sea_surface_temperature'][0, :, :])
_bt11= dataset.variables['brightness_temperature_11um'][0, :, :]
bt11 = np.ma.array(_bt11)
quality_level = np.ma.array(dataset.variables['quality_level'][0, :, :])
'''
if file_shape is not None:
with open(file_shape, 'r') as fshape:
shape = shapely.wkt.load(fshape)
box = shape.bounds
index_in = np.where((lon >= box[0]) & (lat >= box[1])
& (lon <= box[2]) & (lat <= box[3]))
index_out = np.where((lon < box[0]) | (lat < box[1])
| (lon > box[2]) | (lat > box[3]))
sst[index_out] = np.nan
print(np.shape(index_in))
sys.exit(1)
for i, j in zip(index_in[0], index_in[1]):
p = Point(lon[i, j], lat[i, j])
if p.within(shape) is False:
sst[i, j] = np.nan
'''
if listbox is not None:
mask_box = np.zeros(np.shape(sst))
for i in range(np.shape(listbox)[0]):
index_in = np.where((lon >= listbox[i][0]) & (lat >= listbox[i][1])
& (lon <= listbox[i][2]) & (lat <= listbox[i][3]))
mask_box[index_in] = 1
mask = ma.getmaskarray(sst) | ma.getmaskarray(bt11) | \
(quality_level.data < 4) | (mask_box == 0)
else:
mask = ma.getmaskarray(sst) | ma.getmaskarray(bt11) | \
(quality_level.data < 4)
if mask.all():
print 'No data'
sys.exit(0)
# GCPs for resampling and geotiff georeference
scansize = 16
dtime0 = datetime.utcnow()
gcps = resample.get_gcps_from_bowtie(lon, lat, scansize, ngcps=ngcps)
dtime = datetime.utcnow() - dtime0
print 'Get GCPs from bowtie swath : {}'.format(dtime)
gcplon, gcplat, gcpnpixel, gcpnline = gcps
rspysize = lon.shape[0]
geod = pyproj.Geod(ellps='WGS84')
mid = abs(gcpnline[:, 0] - 0.5).argmin()
xdists = geod.inv(gcplon[mid, :-1], gcplat[mid, :-1],
gcplon[mid, 1:], gcplat[mid, 1:])[2]
xdist = np.sum(xdists) / abs(gcpnpixel[mid, -1] - gcpnpixel[mid, 0])
rspxsize = np.round(xdist / 750.).astype('int') + 1
gcpline = gcpnline * rspysize
gcppixel = gcpnpixel * rspxsize
# Resample with LinearNDInterpolator in output space
dtime0 = datetime.utcnow()
pix, lin = resample.get_points_from_gcps(gcplon, gcplat, gcppixel,
gcpline, rspxsize, rspysize,
1, lon, lat, nprocs=nprocs) - 0.5
dtime = datetime.utcnow() - dtime0
print 'Get input coordinates in new grid : {}'.format(dtime)
# Test input grid in output space
# import matplotlib.pyplot as plt
# for iscan in range(lon.shape[0] / scansize):
# pixscan = pix[iscan * scansize: (iscan+1) * scansize, :]
# linscan = lin[iscan * scansize: (iscan+1) * scansize, :]
# # maskscan = mask[iscan * scansize: (iscan+1) * scansize, :]
# # pixscan = pixscan[~maskscan]
# # linscan = linscan[~maskscan]
# plt.plot(pixscan.flatten(), linscan.flatten(), '+')
# plt.show()
# import pdb ; pdb.set_trace()
# \Test input grid in output space
dtime0 = datetime.utcnow()
sst.data[mask] = np.nan
bt11.data[mask] = np.nan
val = np.dstack((sst.data, bt11.data))
rspval = resample.resample_bowtie_linear(pix, lin, val, scansize,
rspxsize, rspysize, show=False)
rspsst = rspval[:, :, 0]
rspbt11 = rspval[:, :, 1]
rspmask = ma.getmaskarray(rspsst) | ma.getmaskarray(rspbt11)
dtime = datetime.utcnow() - dtime0
print 'Interpolate in new grid : {}'.format(dtime)
# Denoise sst and open mask
rspsst.mask = rspmask
rspbt11.mask = rspmask
finalsst = denoise_sst(rspsst, rspbt11, kernel=denoise_kernel,
width=denoise_width, show=False)
finalmask = ~binary_opening(~rspmask, structure=np.ones((3, 3)),
iterations=open_iterations)
finalsst.mask = finalmask
# Contrast
if vmin == None:
if contrast == 'relative':
vmin = np.percentile(finalsst.compressed(), 0.5)
#elif contrast == 'agulhas':
# dayofyear = float(start_time.timetuple().tm_yday)
# vmin = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. - 9.
# #par = [277.94999694824219, 42, 2.5500030517578125, -219]
# par = [278.09999084472656, 0.62831853071795862,
# 2.4000091552734375, 0.1570796326794896]
# vmin = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific txt file is provided for the range
elif dic is not None:
dayofyear = float(start_time.timetuple().tm_yday)
extrema = dic.get(dayofyear, dic[min(dic.keys(),
key=lambda k:abs(k - dayofyear))])
vmin = extrema[0]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
if vmax == None:
if contrast == 'relative':
vmax = np.percentile(finalsst.compressed(), 99.5)
#elif contrast == 'agulhas':
# dayofyear = float(start_time.timetuple().tm_yday)
# vmax = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. + 4.
# #par = [300.59999084472656, 21, 2.8499908447265625, -191]
# par = [300.59999084472656, 0.29919930034188508,
# 2.8499908447265625, 0.14959965017094254]
# vmax = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific text file is provided for the range
elif dic is not None:
dayofyear = float(start_time.timetuple().tm_yday)
extrema = dic.get(dayofyear, dic[min(dic.keys(),
key=lambda k:abs(k - dayofyear))])
vmax = extrema[1]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
# Flip (geotiff in "swath sense")
finalsst = finalsst[::-1, ::-1]
gcppixel = rspxsize - gcppixel
gcpline = rspysize - gcpline
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time,
units='ms')
metadata['product_name'] = 'SST_VIIRS_denoised'
if contrast == 'relative':
metadata['name'] = os.path.splitext(os.path.basename(infile))[0]
else:
metadata['name'] = '{}_{}'.format(os.path.splitext(os.path.basename(infile))[0], contrast)
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'NOAA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface temperature'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'radiometer'
metadata['sensor_name'] = 'VIIRS'
metadata['sensor_platform'] = 'Suomi-NPP'
#metadata['sensor_pass'] =
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
gcpheight = np.zeros(gcppixel.shape)
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcpheight,
gcppixel, gcpline)
band = []
indndv = np.where(ma.getmaskarray(finalsst) == True)
offset, scale = vmin, (vmax-vmin)/254.
np.clip(finalsst.data, vmin, vmax, out=finalsst.data)
array = np.round((finalsst.data - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=vmax, vmax_pal=vmax,
vmin=vmin, vmin_pal=vmin)
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'sea surface temperature', 'unittype':'K',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'colortable':colortable})
if write_netcdf == False:
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'denoised_sst'
band[0]['long_name'] = 'denoised sea surface temperature'
band[0]['standard_name'] = 'sea_surface_temperature'
# ymid = abs(gcpline[:, 0] - rspysize / 2.).argmin()
# xdists = geod.inv(gcplon[ymid, :-1], gcplat[ymid, :-1],
# gcplon[ymid, 1:], gcplat[ymid, 1:])[2] / \
# np.abs(gcppixel[ymid, 1:] - gcppixel[ymid, :-1])
# xmid = abs(gcppixel[0, :] - rspxsize / 2.).argmin()
# ydists = geod.inv(gcplon[:-1, xmid], gcplat[:-1, xmid],
# gcplon[1:, xmid], gcplat[1:, xmid])[2] / \
# np.abs(gcpline[1:, xmid] - gcpline[:-1, xmid])
# print xdists.min(), xdists.max(), xdists.mean()
# # e.g. 749.905437495 749.905892002 749.905827652
# print ydists.min(), ydists.max(), ydists.mean()
# # e.g. 737.638084996 741.195663083 739.157662785
metadata['spatial_resolution'] = 750.
stfmt.write_netcdf(ncfile, metadata, geolocation, band, 'swath',
ngcps=gcplon.shape)
def sar_xspec(infile, outdir, pngkml=False, vmax_re=None, vmax_im=None,
make_rgb=True, ncolors=74):
"""
"""
# Read/Process data
print 'Read/Process data'
sarim = sarimage(infile)
mission = sarim.get_info('mission')
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
else:
raise Exception('S1A mission expected.')
product = sarim.get_info('product')
if product != 'SLC':
raise Exception('SLC expected.')
timefmt = '%Y-%m-%dT%H:%M:%S.%f'
start_time = datetime.strptime(sarim.get_info('start_time'), timefmt)
stop_time = datetime.strptime(sarim.get_info('stop_time'), timefmt)
sensor_pass = sarim.get_info('pass')
sensor_mode = sarim.get_info('mode')
sensor_swath = sarim.get_info('swath')
sensor_polarisation = sarim.get_info('polarisation')
datagroup = sarim.get_info('safe_name').replace('.SAFE', '')
pid = datagroup.split('_')[-1]
dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
# Compute SAR Xspec and make figures
if sensor_mode == 'WV':
azi_periodo_size = 1024
azi_dist, ran_dist = 20000., 20000. # ignored in WV case
xspec_size = (512, 512)
fontsize = 'small'
elif re.match(r'^S[1-6]$', sensor_mode) != None:
azi_periodo_size = 1024
azi_dist, ran_dist = 10000., 10000.
xspec_size = (512, 512) #(256, 256)
fontsize = 'small' #'x-small'
elif sensor_mode in ['IW', 'EW']:
azi_periodo_size = 512
azi_dist, ran_dist = 10000., 10000.
xspec_size = (512, 512) #(256, 256)
fontsize = 'small' #'x-small'
else:
raise Exception('Which settings for this mode ?')
sarxspec = sarimage2sarxspec_loop(sarim, azi_dist=azi_dist, ran_dist=ran_dist,
azi_periodo_size=azi_periodo_size)
cmap_re, cmap_im = get_cmaps(ncolors=ncolors)
fig_re = make_sarxspec_fig(sarxspec, part='real', tau=1, kmax=2*np.pi/75,
kmin=2*np.pi/400, xspec_size=xspec_size,
uniq_vmax=True, north_oriented=True,
klim=[2*np.pi/400, 2*np.pi/200, 2*np.pi/100],
north_arrow=False, index_pos=None, vmax_pos='tr',
nvar_pos=None, fontsize=fontsize,
vmax=vmax_re, cmap=cmap_re)
if sensor_mode == 'WV':
ax = fig_re.gca()
imnum = sarim.get_info('image_number')
imnumtxt = '#{:03d}'.format(imnum)
ax.text(0.51, 0.99, imnumtxt, transform=ax.transAxes,
ha='left', va='top', fontsize=fontsize)
if make_rgb == True:
rgb_re = fig2rgb(fig_re)
#print nuniqcolors(rgb_re)
else:
img_re, pal_re = fig2imgpal(fig_re, cmap_re)
plt.close(fig_re)
fig_im = make_sarxspec_fig(sarxspec, part='imag', tau=1, kmax=2*np.pi/75,
kmin=2*np.pi/400, xspec_size=xspec_size,
uniq_vmax=True, north_oriented=True,
klim=[2*np.pi/400, 2*np.pi/200, 2*np.pi/100],
north_arrow=False, index_pos=None, vmax_pos='tr',
nvar_pos=None, fontsize=fontsize,
vmax=vmax_im, cmap=cmap_im)
if sensor_mode == 'WV':
ax = fig_im.gca()
imnum = sarim.get_info('image_number')
imnumtxt = '#{:03d}'.format(imnum)
ax.text(0.51, 0.99, imnumtxt, transform=ax.transAxes,
ha='left', va='top', fontsize=fontsize)
if make_rgb == True:
rgb_im = fig2rgb(fig_im)
#print nuniqcolors(rgb_im)
else:
img_im, pal_im = fig2imgpal(fig_im, cmap_im)
plt.close(fig_im)
if make_rgb == True:
nlin, npix = rgb_re.shape[0:2]
else:
nlin, npix = img_re.shape
# Handle GCPS
# geoloc = sarim.get_info('geolocation_grid')
# pix = np.array([0, geoloc['npixels']-1, geoloc['npixels']-1, 0])
# lin = np.array([0, 0, geoloc['nlines']-1, geoloc['nlines']-1])
# gcplon = geoloc['longitude'][lin, pix]
# gcplat = geoloc['latitude'][lin, pix]
# gcphei = np.zeros(4)
# gcppix = np.array([0, 512, 512, 0])
# gcplin = np.array([0, 0, 512, 512])
#############################################
# geoloc = sarim.get_info('geolocation_grid')
# gcplon = geoloc['longitude']
# gcplat = geoloc['latitude']
# gcphei = np.zeros(gcplon.shape)
# geod = pyproj.Geod(ellps='WGS84')
# nglin, ngpix = geoloc['nlines'], geoloc['npixels']
# ra_geo_spacing = geod.inv(gcplon[nglin/2, 0:-1], gcplat[nglin/2, 0:-1],
# gcplon[nglin/2, 1:], gcplat[nglin/2, 1:])[2]
# ra_geo_dist = np.hstack((0., ra_geo_spacing.cumsum()))
# ra_geo_ndist = ra_geo_dist/ra_geo_dist[-1]
# gcppix = np.tile((ra_geo_ndist*npix).reshape((1, -1)), (nglin, 1))
# az_geo_spacing = geod.inv(gcplon[0:-1, ngpix/2], gcplat[0:-1, ngpix/2],
# gcplon[1:, ngpix/2], gcplat[1:, ngpix/2])[2]
# az_geo_dist = np.hstack((0., az_geo_spacing.cumsum()))
# az_geo_ndist = az_geo_dist/az_geo_dist[-1]
# gcplin = np.tile((az_geo_ndist*nlin).reshape((-1, 1)), (1, ngpix))
# import pdb ; pdb.set_trace()
#############################################
#import pdb ; pdb.set_trace()
ext_min = sarxspec[0][0].get_info('extent')[0:2]
ext_max = sarxspec[-1][-1].get_info('extent')[2:4]
# geoloc = sarim.get_info('geolocation_grid')
# nglin, ngpix = geoloc['nlines'], geoloc['npixels']
nglin, ngpix = len(sarxspec)+1, len(sarxspec[0])+1
pix = np.hstack((np.round(np.linspace(ext_min[1], ext_max[1], num=ngpix)),
np.ones(nglin)*ext_max[1],
np.round(np.linspace(ext_max[1], ext_min[1], num=ngpix)),
np.ones(nglin)*ext_min[1]))
lin = np.hstack((np.ones(ngpix)*ext_min[0],
np.round(np.linspace(ext_min[0], ext_max[0], num=nglin)),
np.ones(ngpix)*ext_max[0],
np.round(np.linspace(ext_max[0], ext_min[0], num=nglin))))
lon, lat = np.zeros(pix.size), np.zeros(pix.size)
for ipt in range(pix.size):
ext = [lin[ipt], pix[ipt], lin[ipt], pix[ipt]]
lon[ipt] = sarim.get_data('lon', extent=ext, spacing=1)
lat[ipt] = sarim.get_data('lat', extent=ext, spacing=1)
ndist = np.zeros(pix.size)
lim = [0, ngpix, ngpix+nglin, 2*ngpix+nglin, 2*ngpix+2*nglin]
geod = pyproj.Geod(ellps='WGS84')
for iside in range(4):
pt0, pt1 = lim[iside], lim[iside+1]-1
ddist = geod.inv(lon[pt0:pt1], lat[pt0:pt1], lon[pt0+1:pt1+1],
lat[pt0+1:pt1+1])[2]
dist = ddist.cumsum()
ndist[pt0:pt1+1] = np.hstack((0., dist))/dist.max()
gcppix = np.hstack((ndist[lim[0]:lim[1]-1]*npix, np.ones(nglin-1)*npix,
(1-ndist[lim[2]:lim[3]-1])*npix, np.zeros(nglin-1)))
gcplin = np.hstack((np.zeros(ngpix-1), ndist[lim[1]:lim[2]-1]*nlin,
np.ones(ngpix-1)*nlin, (1-ndist[lim[3]:lim[4]-1])*nlin))
gcplon = np.hstack((lon[lim[0]:lim[1]-1], lon[lim[1]:lim[2]-1],
lon[lim[2]:lim[3]-1], lon[lim[3]:lim[4]-1]))
gcplat = np.hstack((lat[lim[0]:lim[1]-1], lat[lim[1]:lim[2]-1],
lat[lim[2]:lim[3]-1], lat[lim[3]:lim[4]-1]))
gcphei = np.zeros(gcplon.size)
#import pdb ; pdb.set_trace()
if gcplon.min() < -135 and gcplon.max() > 135:
gcplon[np.where(gcplon < 0)] += 360.
#############################################
if sensor_pass == 'Descending':
gcppix = npix-gcppix
gcplin = nlin-gcplin
gcplin = nlin-gcplin # because fig will be read and wrote from top to bottom
# Loop on part and write
for part in ['real', 'imag']:
print part
if part == 'real':
product = 'SAR_cross-spectrum_real'
nameext = '-xspec_re'
if make_rgb == True:
rgb = rgb_re
else:
img = img_re
pal = pal_re
elif part == 'imag':
product = 'SAR_cross-spectrum_imaginary'
nameext = '-xspec_im'
if make_rgb == True:
rgb = rgb_im
else:
img = img_im
pal = pal_im
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time,
units='ms')
metadata['product_name'] = product
metadata['name'] = dataname + nameext
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = ''
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = sarim._mapper._handler.GetGCPProjection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei,
gcppix, gcplin)
band = []
if make_rgb == True:
band.append({'array': rgb[:, :, 0]})
band.append({'array': rgb[:, :, 1]})
band.append({'array': rgb[:, :, 2]})
else:
band.append({'array': img, 'nodatavalue': 255,
'colortable': palette2colortable(pal)})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
def read_orbit(infile,
outdir,
vmin=-0.2,
vmax=0.2,
vmin_pal=-0.2,
vmax_pal=0.2,
dist_gcp=5,
process_var=('ssha', 'swh', 'ws', 'sigma0'),
keep_empty=False,
log_path=None):
"""
"""
t0 = datetime.utcnow()
syntool_stats = {}
if log_path is not None and not os.path.exists(log_path):
try:
os.makedirs(log_path)
except OSError:
_, e, _ = sys.exc_info()
if e.errno != errno.EEXIST:
raise
listvar = []
height = False
wave = False
if 'ssha' in process_var:
height = True
if 'swh' in process_var:
wave = True
if 'ws' in process_var:
wind = True
if 'sigma0' in process_var:
sigma = True
if height is False and wave is False and wind is False and sigma is False:
sys.exit('please specify at least on variable to convert')
L2id = '1Hz' #'Ku'
# Read variables
dset = Dataset(os.path.join(infile, 'standard_measurement.nc'), 'r')
### For debug purposes
ind_tmp = 0
lon_0 = dset.variables[L2_MAPS[L2id]['lonname']][ind_tmp:]
lon_0[lon_0 > 180] = lon_0[lon_0 > 180] - 360
lat_0 = dset.variables[L2_MAPS[L2id]['latname']][ind_tmp:]
mask_surface = dset.variables[L2_MAPS[L2id]['surface']][ind_tmp:]
if height is True:
ssha_0 = dset.variables[L2_MAPS[L2id]['hname']][ind_tmp:]
ssha_fill_value = dset.variables[L2_MAPS[L2id]['hname']]._FillValue
ssha_0[mask_surface > 1] = numpy.nan
if wave is True:
swh_0 = dset.variables[L2_MAPS[L2id]['wname']][ind_tmp:]
swh_fill_value = dset.variables[L2_MAPS[L2id]['wname']]._FillValue
swh_0[mask_surface > 1] = numpy.nan
if wind is True:
ws_0 = dset.variables[L2_MAPS[L2id]['winame']][ind_tmp:]
ws_fill_value = dset.variables[L2_MAPS[L2id]['winame']]._FillValue
ws_0[mask_surface > 1] = numpy.nan
if sigma is True:
sigma0_0 = dset.variables[L2_MAPS[L2id]['sname']][ind_tmp:]
sigma0_fill_value = dset.variables[L2_MAPS[L2id]['sname']]._FillValue
sigma0_0[mask_surface > 1] = numpy.nan
# var_0 /= dset.variables[L2_MAPS[L2id]['hname']].scale_factor
time_0 = dset.variables[L2_MAPS[L2id]['timename']][ind_tmp:]
time_units = dset.variables[L2_MAPS[L2id]['timename']].units
dset.close()
# Trick to deal with continuity in longitude
dlon = abs(lon_0[1:] - lon_0[:-1])
lref = lon_0[numpy.shape(lon_0)[0] / 2]
lon_0 = numpy.mod(lon_0 - (lref - 180), 360) + (lref - 180)
lon_0 = numpy.rad2deg(numpy.unwrap(numpy.deg2rad(lon_0)))
# Interpolate on land
dtime = time_0[1:] - time_0[:-1]
dlon = abs(lon_0[1:] - lon_0[:-1])
# delta = numpy.median(dtime)
if len(time_0) > 3:
delta = stats.mode(dtime)[0][0]
else:
logger.warn('orbit too short')
return
t_start = datetime.utcnow()
ndelta = numpy.round(dtime / delta).astype('int')
if L2id == '1Hz':
dtime_threshold = 3
else:
dtime_threshold = 3 / 20
ind_dtime = numpy.where(ndelta >= dtime_threshold)[0]
if len(ind_dtime) == 0:
time = time_0
lon = lon_0
lat = lat_0
if height is True:
ssha = ssha_0
if wave is True:
swh = swh_0
if wind is True:
ws = ws_0
if sigma is True:
sigma0 = sigma0_0
else:
time = []
lon = []
lat = []
time.append(time_0[:ind_dtime[0] + 1])
lon.append(lon_0[:ind_dtime[0] + 1])
lat.append(lat_0[:ind_dtime[0] + 1])
func_lon = interpolate.interp1d(time_0, lon_0, kind='cubic')
func_lat = interpolate.interp1d(time_0, lat_0, kind='cubic')
if height is True:
ssha = []
ssha.append(ssha_0[:ind_dtime[0] + 1])
if wave is True:
swh = []
swh.append(swh_0[:ind_dtime[0] + 1])
if wind is True:
ws = []
ws.append(ws_0[:ind_dtime[0] + 1])
if sigma is True:
sigma0 = []
sigma0.append(sigma0_0[:ind_dtime[0] + 1])
for i in range(len(ind_dtime)):
time_fill = numpy.linspace(
time_0[ind_dtime[i]],
time_0[ind_dtime[i] + 1],
num=(
(time_0[ind_dtime[i] + 1] - time_0[ind_dtime[i]]) / delta +
1))
time.append(time_fill[1:-1])
lon_tmp = func_lon(time_fill[1:-1])
lat_tmp = func_lat(time_fill[1:-1])
lon.append(lon_tmp)
lat.append(lat_tmp)
if height is True:
ssha_fill = numpy.full(numpy.shape(time_fill[1:-1]), numpy.nan)
ssha.append(ssha_fill)
if wave is True:
swh_fill = numpy.full(numpy.shape(time_fill[1:-1]), numpy.nan)
swh.append(swh_fill)
if wind is True:
ws_fill = numpy.full(numpy.shape(time_fill[1:-1]), numpy.nan)
ws.append(ws_fill)
if sigma is True:
sigma0_fill = numpy.full(numpy.shape(time_fill[1:-1]),
numpy.nan)
sigma0.append(sigma0_fill)
if i != (len(ind_dtime) - 1):
slice_ind = slice(ind_dtime[i] + 1, ind_dtime[i + 1] + 1)
time.append(time_0[slice_ind])
lon.append(lon_0[slice_ind])
lat.append(lat_0[slice_ind])
if height is True:
ssha.append(ssha_0[slice_ind])
if wave is True:
swh.append(swh_0[slice_ind])
if wind is True:
ws.append(ws_0[slice_ind])
if sigma is True:
sigma0.append(sigma0_0[slice_ind])
else:
time.append(time_0[ind_dtime[i] + 1:])
lon.append(lon_0[ind_dtime[i] + 1:])
lat.append(lat_0[ind_dtime[i] + 1:])
if height is True:
ssha.append(ssha_0[ind_dtime[i] + 1:])
if wave is True:
swh.append(swh_0[ind_dtime[i] + 1:])
if wind is True:
ws.append(ws_0[ind_dtime[i] + 1:])
if sigma is True:
sigma0.append(sigma0_0[ind_dtime[i] + 1:])
time = numpy.concatenate(time, axis=0)
lon = numpy.concatenate(lon, axis=0)
lat = numpy.concatenate(lat, axis=0)
if height is True:
ssha = numpy.concatenate(ssha, axis=0)
ssha[ssha == ssha_fill_value] = numpy.nan
if wave is True:
swh = numpy.concatenate(swh, axis=0)
swh[swh == swh_fill_value] = numpy.nan
if wind is True:
ws = numpy.concatenate(ws, axis=0)
ws[ws == ws_fill_value] = numpy.nan
if sigma is True:
sigma0 = numpy.concatenate(sigma0, axis=0)
sigma0[sigma0 == sigma0_fill_value] = numpy.nan
t_stop = datetime.utcnow()
syntool_stats['gapfill_computation'] = (t_stop - t_start).total_seconds()
lon = lon - numpy.floor((numpy.min(lon) + 180.) / 360.) * 360.
ntime = numpy.shape(time)[0]
if height is True:
nan_mask = numpy.isnan(ssha)
nan_ind = numpy.where(nan_mask)
ssha[nan_ind] = 0
mask_gap_ssha = (nan_mask | (abs(ssha) > 50))
ssha[nan_ind] = numpy.nan # Restore nan values
if not mask_gap_ssha.all():
_min = numpy.nanmin(ssha[numpy.where(~mask_gap_ssha)])
_max = numpy.nanmax(ssha[numpy.where(~mask_gap_ssha)])
syntool_stats['ssha'] = {'min': _min, 'max': _max}
listvar.append({
'var': ssha,
'mask_gap': mask_gap_ssha,
'range': [-0.4, 0.4],
'parameter': 'SSHA',
'palette': 'matplotlib_jet',
'range_pal': [-0.4, 0.4]
})
if wave is True:
nan_mask = numpy.isnan(swh)
nan_ind = numpy.where(nan_mask)
swh[nan_ind] = 0
mask_gap_swh = (nan_mask | (abs(swh) > 50))
swh[nan_ind] = numpy.nan # Restore nan values
if not mask_gap_swh.all():
_min = numpy.nanmin(swh[numpy.where(~mask_gap_swh)])
_max = numpy.nanmax(swh[numpy.where(~mask_gap_swh)])
syntool_stats['swh'] = {'min': _min, 'max': _max}
listvar.append({
'var': swh,
'mask_gap': mask_gap_swh,
'range': [0., 8.],
'parameter': 'SWH',
'palette': 'matplotlib_jet',
'range_pal': [0., 8.]
})
if wind is True:
nan_mask = numpy.isnan(ws)
nan_ind = numpy.where(nan_mask)
ws[nan_ind] = 0
mask_gap_ws = (nan_mask | (abs(ws) > 200))
ws[nan_ind] = numpy.nan # Restore nan values
if not mask_gap_ws.all():
_min = numpy.nanmin(ws[numpy.where(~mask_gap_ws)])
_max = numpy.nanmax(ws[numpy.where(~mask_gap_ws)])
syntool_stats['ws'] = {'min': _min, 'max': _max}
listvar.append({
'var': ws,
'mask_gap': mask_gap_ws,
'range': [0., 25.],
'parameter': 'Ws',
'palette': 'noaa_wind',
'range_pal': [0., 25.]
})
if sigma is True:
nan_mask = numpy.isnan(sigma0)
nan_ind = numpy.where(nan_mask)
sigma0[nan_ind] = 0
mask_gap_sigma0 = (nan_mask | (sigma0 > 100))
sigma0[nan_ind] = numpy.nan # Restore nan values
if not mask_gap_sigma0.all():
_min = numpy.nanmin(sigma0[numpy.where(~mask_gap_sigma0)])
_max = numpy.nanmax(sigma0[numpy.where(~mask_gap_sigma0)])
syntool_stats['sigma0'] = {'min': _min, 'max': _max}
listvar.append({
'var': sigma0,
'mask_gap': mask_gap_sigma0,
'range': [5., 25.],
'parameter': 'Sigma0',
'palette': 'matplotlib_jet',
'range_pal': [5., 25.]
})
start_time = num2date(time[0], time_units)
end_time = num2date(time[-1], time_units)
# NOTE : lon/lat must be continuous even if crossing dateline
# (ie. no [-180,180] clipping)
# Make GCPs (mimic a swath of arbitrary width in lon/lat, here ~5km)
# gcps = tools_for_gcp.make_gcps_v1(lon, lat, dist_gcp=dist_gcp)
gcps = tools_for_gcp.make_gcps_v2(lon, lat, dist_gcp=dist_gcp)
file_name, _ = os.path.splitext(os.path.basename(os.path.normpath(infile)))
for i in range(len(listvar)):
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='s')
metadata['product_name'] = (L2_MAPS[L2id]['productname'] + '_' +
listvar[i]['parameter'])
metadata['name'] = file_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(*gcps)
band = []
vmin = listvar[i]['range'][0]
vmax = listvar[i]['range'][1]
vmin_pal = listvar[i]['range_pal'][0]
vmax_pal = listvar[i]['range_pal'][1]
scale = (vmax - vmin) / 254.
offset = vmin
# Avoid warnings caused by NaN values
#ssha[numpy.where(~(numpy.isfinite(mask_gap)))] = 255
var = listvar[i]['var']
mask_gap = listvar[i]['mask_gap']
var[numpy.where(mask_gap)] = 255
array = numpy.clip(numpy.round((var - offset) / scale), 0,
254).astype('uint8')
array[numpy.where(mask_gap)] = 255
array = array[:, numpy.newaxis]
if not keep_empty and numpy.all(array == 255):
parameter = listvar[i]['parameter']
logger.warn('No valid values in this dataset for ' \
'{}'.format(parameter))
logger.warn('Skipped.')
continue
colortable = stfmt.format_colortable(listvar[i]['palette'],
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': listvar[i]['parameter'],
'unittype': 'm',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Check GCPs
# tools_for_gcp.check_gcps(tifffile, lon, lat, gcps[0], gcps[1])
syntool_stats['total_time'] = (datetime.utcnow() - t0).total_seconds()
if log_path is not None:
import json
stats_path = os.path.join(log_path, '{}.json'.format(file_name))
with open(stats_path, 'w') as f:
json.dump(syntool_stats, f)
def read_orbit(infile,
outdir,
vmin=-0.2,
vmax=0.2,
vmin_pal=-0.2,
vmax_pal=0.2,
dist_gcp=None,
write_netcdf=False):
"""
"""
if (re.match(r'^dt_global_j2_adt_vfec_.*\.nc', os.path.basename(infile))
is not None):
L3id = 'J2_ADT'
# vmin = -0.2; vmax = 0.2 ; vmin_pal = -0.2 ; vmax_pal = 0.2
elif (re.match(r'^dt_global_al_adt_vfec.*\.nc', os.path.basename(infile))
is not None):
L3id = 'AL_ADT'
# vmin = -0.2 ; vmax = 0.2 ; vmin_pal = -0.2 ; vmax_pal = 0.2
elif (re.match(r'^dt_global_j2_sla_vfec.*\.nc', os.path.basename(infile))
is not None):
L3id = 'J2_SLA'
# vmin = -0.2 ; vmax = 0.2 ; vmin_pal = -0.2 ; vmax_pal = 0.2
elif (re.match(r'^dt_global_al_sla_vfec.*\.nc', os.path.basename(infile))
is not None):
L3id = 'AL_SLA'
# vmin = -0.2 ; vmax = 0.2 ; vmin_pal = -0.2 ; vmax_pal = 0.2
# Read
dset = Dataset(infile)
lon_all = dset.variables['longitude'][:]
lat_all = dset.variables['latitude'][:]
var_all = dset.variables[L3_MAPS[L3id]['hname']][:]
time_all = dset.variables['time'][:]
ipass_all = dset.variables['track'][:]
cycle_all = dset.variables['cycle'][:]
time_units = dset.variables['time'].units
var_fill_value = dset.variables[L3_MAPS[L3id]['hname']]._FillValue
var_all[var_all == var_fill_value] = np.nan
dset.close()
# Detect passes index
dipass = ipass_all[1:] - ipass_all[:-1]
ind_ipass = np.where(dipass != 0)[0]
ind_ipass += 1
ind_ipass = np.hstack([0, ind_ipass, len(ipass_all)])
# Loop on all passes and compute geotiff for each pass
for i in range(len(ind_ipass) - 1):
lon_0 = lon_all[ind_ipass[i]:ind_ipass[i + 1]]
# lon_0 = np.mod(lon_0 + 180, 360) - 180
lref = lon_0[np.shape(lon_0)[0] / 2]
lon_0 = np.mod(lon_0 - (lref - 180), 360) + (lref - 180)
#lon_0 = np.mod(lon_0 - np.min(lon_0), 360) - lref # - 180
lon_0 = np.rad2deg(np.unwrap(np.deg2rad(lon_0)))
lat_0 = lat_all[ind_ipass[i]:ind_ipass[i + 1]]
var_0 = var_all[ind_ipass[i]:ind_ipass[i + 1]]
time_0 = time_all[ind_ipass[i]:ind_ipass[i + 1]]
ipass_0 = ipass_all[ind_ipass[i]:ind_ipass[i + 1]]
cycle_0 = cycle_all[ind_ipass[i]:ind_ipass[i + 1]]
dtime = time_0[1:] - time_0[:-1]
dlon = abs(lon_0[1:] - lon_0[:-1])
if (np.array(dlon) > 180).any():
lon_0 = np.mod(lon_0, 360)
if len(time_0) > 3:
delta = stats.mode(dtime)[0][0]
else:
continue
ndelta = np.round(dtime / delta).astype('int')
ind_dtime = np.where(ndelta >= 2)[0]
if ind_dtime.size != 0:
time = time_0[:ind_dtime[0] + 1]
var = var_0[:ind_dtime[0] + 1]
else:
time = time_0
var = var_0
for i in range(len(ind_dtime)):
# time_fill = np.linspace(time_0[ind_dtime[i]],
# time_0[ind_dtime[i] + 1],
# num=(time_0[ind_dtime[i] + 1]
# - time_0[ind_dtime[i]]) / delta)
time_fill = np.linspace(time_0[ind_dtime[i]],
time_0[ind_dtime[i] + 1],
num=ndelta[ind_dtime[i]],
endpoint=False)
var_fill = np.zeros(np.shape(time_fill[1:])) * np.nan
# time = time.append(time_fill[1:])
time = np.hstack([time, time_fill[1:]])
# var = var.append(var_fill)
var = np.hstack([var, var_fill])
if i != (len(ind_dtime) - 1):
time = np.hstack(
[time, time_0[ind_dtime[i] + 1:ind_dtime[i + 1] + 1]])
var = np.hstack(
[var, var_0[ind_dtime[i] + 1:ind_dtime[i + 1] + 1]])
else:
time = np.hstack([time, time_0[ind_dtime[i] + 1:]])
var = np.hstack([var, var_0[ind_dtime[i] + 1:]])
# time = np.concatenate(time)
# var = np.concatenate(var)
func = interpolate.interp1d(time_0, lon_0, kind='quadratic')
lon = func(time)
func = interpolate.interp1d(time_0, lat_0, kind='quadratic')
lat = func(time)
ssha = var
mask_gap = np.isnan(ssha)
lon = lon - np.floor((np.min(lon) + 180.) / 360.) * 360.
time = np.float64(time)
start_time = num2date(time[0], time_units)
end_time = num2date(time[-1], time_units)
time = num2date(time, units=time_units)
time_num = time
for t in range(np.shape(time)[0]):
time_num[t] = date2num(
time[t],
units='microseconds since 1970-01-01 00:00:00.000000Z')
time_num[t] *= 10**(-6)
#time = (num2date(time, time_units)
# - calendar.timegm(cnes_day.timetuple()))
# for t in range(np.shape(time)[0]):
# time[t] = date2num(num2date(time[t], units) - cnes_day, unit=)
#time = time.total_seconds()
#time = (time + calendar.timegm(julian_day.timetuple())
# - calendar.timegm(cnes_day.timetuple()))
# NOTE : lon/lat must be continuous even if crossing dateline
# (ie. no [-180,180] clipping)
# Make GCPs (mimic a swath of arbitrary width in lon/lat, here ~5km)
# gcps = tools_for_gcp.make_gcps_v1(lon, lat, dist_gcp=dist_gcp)
gcps = tools_for_gcp.make_gcps_v2(lon, lat, dist_gcp=dist_gcp)
# Write geotiff
# NOTE : product_name to be changed, set here for test
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='s')
metadata['product_name'] = L3_MAPS[L3id]['productname']
dname = (os.path.splitext(os.path.basename(infile))[0] + '_c' +
str(int(cycle_0[0])).zfill(4) + '_p' +
str(int(ipass_0[0])).zfill(3))
metadata['name'] = dname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['begin_time'] = start_time.strftime(TIMEFMT)
metadata['end_time'] = end_time.strftime(TIMEFMT)
metadata['source_URI'] = infile
metadata['source_provider'] = 'AVISO'
metadata['processing_center'] = ''
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datatime'] = stfmt.format_time(datetime.utcnow())
metadata['type'] = 'along_track'
metadata['cycle'] = int(cycle_0[0])
metadata['pass'] = int(ipass_0[0])
metadata['spatial_resolution'] = np.float32(7000)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(*gcps)
band = []
mask = np.ma.getmaskarray(ssha)
if write_netcdf:
vmin = np.nanmin(ssha)
vmin_pal = vmin
vmax = np.nanmax(ssha)
vmax_pal = vmax
#print('bla')
scale = (vmax - vmin) / 254.
offset = vmin
mask = np.ma.getmaskarray(ssha)
array = np.clip(np.round((ssha - offset) / scale), 0,
254).astype('uint8')
array[mask] = 255
array[mask_gap] = 255
if write_netcdf is False:
array = array[:, np.newaxis]
colortable = stfmt.format_colortable('matplotlib_jet',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': L3_MAPS[L3id]['parameter'],
'name': L3_MAPS[L3id]['hname'],
'unittype': 'm',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
if write_netcdf is False:
tifffile = stfmt.format_tifffilename(outdir,
metadata,
create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
else:
geolocation['geotransform'] = [lon, lat]
geolocation['time'] = time_num[:]
netcdffile = stfmt.format_ncfilename(outdir,
metadata,
create_dir=True)
write_netcdf_1d(netcdffile,
metadata,
geolocation,
band,
model='along_track',
dgcps=1)
def read_orbit(infile,
outdir,
vmin=-0.2,
vmax=0.2,
vmin_pal=-0.2,
vmax_pal=0.2,
dist_gcp=25.5,
keep_empty=False):
"""
"""
if (re.match('.*SLAXT_filt_NorthEastAtlantic*.', os.path.basename(infile))
is not None):
L2id = 'X_TRACK_NEA_SLA'
elif (re.match('.*SLAXT_filt_GoMCaribbean*.', os.path.basename(infile))
is not None):
L2id = 'X_TRACK_GOM_SLA'
elif (re.match('.*SLAXT_filt_MediterraneanSea*.', os.path.basename(infile))
is not None):
L2id = 'X_TRACK_MED_SLA'
else:
logger.warn('Unknown id for file {}'.format(os.path.basename(infile)))
# vmin = -0.2 ; vmax = 0.2 ; vmin_pal = -0.2 ; vmax_pal = 0.2
# Read variables
dset = Dataset(infile)
### For debug purposes
ind_tmp = 0
lon_0 = dset.variables[L2_MAPS[L2id]['lonname']][ind_tmp:]
lon_0[lon_0 > 180] = lon_0[lon_0 > 180] - 360
lat_0 = dset.variables[L2_MAPS[L2id]['latname']][ind_tmp:]
ssha_0 = dset.variables[L2_MAPS[L2id]['hname']][ind_tmp:]
ssha_fill_value = dset.variables[L2_MAPS[L2id]['hname']]._FillValue
# var_0 /= dset.variables[L2_MAPS[L2id]['hname']].scale_factor
time_0 = dset.variables['time'][ind_tmp:]
time_units = dset.variables['time'].units
dset.close()
# Trick to deal with continuity in longitude
dlon = abs(lon_0[1:] - lon_0[:-1])
lref = lon_0[np.shape(lon_0)[0] / 2]
lon_0 = np.mod(lon_0 - (lref - 180), 360) + (lref - 180)
lon_0 = np.rad2deg(np.unwrap(np.deg2rad(lon_0)))
# Interpolate on land
dtime = time_0[1:] - time_0[:-1]
dlon = abs(lon_0[1:] - lon_0[:-1])
# delta = np.median(dtime)
if len(time_0) > 3:
delta = stats.mode(dtime)[0][0]
else:
sys.exit('orbit too short')
ndelta = np.round(dtime / delta).astype('int')
ind_dtime = np.where(ndelta >= 5)[0]
if len(ind_dtime) == 0:
time = time_0
ssha = ssha_0
lon = lon_0
lat = lat_0
else:
time = time_0[:ind_dtime[0] + 1]
ssha = ssha_0[:ind_dtime[0] + 1]
for i in range(len(ind_dtime)):
# time_fill = np.linspace(time_0[ind_dtime[i]], time_0[ind_dtime[i] + 1],
# num=(time_0[ind_dtime[i]+1]
# - time_0[ind_dtime[i]])/delta)
time_fill = np.linspace(time_0[ind_dtime[i]],
time_0[ind_dtime[i] + 1],
num=ndelta[ind_dtime[i]],
endpoint=False)
ssha_fill = np.zeros(np.shape(time_fill[1:])) * np.nan
ssha = np.hstack([ssha, ssha_fill])
time = np.hstack([time, time_fill[1:]])
if i != (len(ind_dtime) - 1):
time = np.hstack(
[time, time_0[ind_dtime[i] + 1:ind_dtime[i + 1] + 1]])
ssha = np.hstack(
[ssha, ssha_0[ind_dtime[i] + 1:ind_dtime[i + 1] + 1]])
else:
time = np.hstack([time, time_0[ind_dtime[i] + 1:]])
ssha = np.hstack([ssha, ssha_0[ind_dtime[i] + 1:]])
func = interpolate.interp1d(time_0, lon_0, kind='linear')
lon = func(time)
func = interpolate.interp1d(time_0, lat_0, kind='linear')
lat = func(time)
ssha[ssha == ssha_fill_value] = np.nan
lon = lon - np.floor((np.min(lon) + 180.) / 360.) * 360.
ntime = np.shape(time)[0]
mask_gap_ssha = np.isnan(ssha)
start_time = num2date(time[0], time_units)
end_time = num2date(time[-1], time_units)
# NOTE : lon/lat must be continuous even if crossing dateline
# (ie. no [-180,180] clipping)
# Make GCPs (mimic a swath of arbitrary width in lon/lat, here ~5km)
# gcps = tools_for_gcp.make_gcps_v1(lon, lat, dist_gcp=dist_gcp)
gcps = tools_for_gcp.make_gcps_v2(lon, lat, dist_gcp=dist_gcp)
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='s')
metadata['product_name'] = (L2_MAPS[L2id]['productname'])
metadata['name'] = (os.path.splitext(os.path.basename(infile))[0])
metadata['datetime'] = dtime
metadata['time_range'] = time_range
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(*gcps)
band = []
#vmin = listvar[i]['range'][0]
#vmax = listvar[i]['range'][1]
#vmin_pal = listvar[i]['range_pal'][0]
#vmax_pal = listvar[i]['range_pal'][1]
scale = (vmax - vmin) / 254.
offset = vmin
# Avoid warnings caused by NaN values
#ssha[np.where(~(np.isfinite(mask_gap)))] = 255
var = ssha
var[np.where(mask_gap_ssha)] = 255
array = np.clip(np.round((var - offset) / scale), 0, 254).astype('uint8')
array[np.where(mask_gap_ssha)] = 255
array = array[:, np.newaxis]
if not keep_empty and np.all(array == 255):
parameter = L2_MAPS[L2id]['parameter']
logger.warn('No valid values in this dataset for ' \
'{}'.format(parameter))
logger.warn('Skipped.')
colortable = stfmt.format_colortable('matplotlib_jet',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': L2_MAPS[L2id]['parameter'],
'unittype': 'm',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
def smosstorm_ascat_wind(infile, outdir,
vmin=0., vmax=50.8, vmin_pal=0., vmax_pal=50.):
"""
"""
# Read/Process data
dataset = Dataset(infile)
lon = dataset.variables['lon'][:]
valid = np.where(ma.getmaskarray(lon) == False)
slices = [slice(valid[0].min(), valid[0].max() + 1),
slice(valid[1].min(), valid[1].max() + 1)]
lon = lon[slices]
if lon.shape[1] % 2 != 0:
raise Exception('Number of cells should be even.')
swath_ncell = lon.shape[1] / 2
lat = dataset.variables['lat'][slices]
wind_speed = dataset.variables['wind_speed'][slices]
wind_dir = dataset.variables['wind_dir'][slices]
time = dataset.variables['time'][slices]
time_units = dataset.variables['time'].units
start_time = num2date(time.min(), time_units)
stop_time = num2date(time.max(), time_units)
datagroup = '_'.join(os.path.splitext(os.path.basename(infile))[0].split('_')[0:3])
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time,
units='ms')
metadata['product_name'] = 'SMOSSTORM_ASCAT_wind'
metadata['datagroup'] = datagroup
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'EUMETSAT'
metadata['processing_center'] = 'KNMI'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = ['wind speed', 'wind direction']
for i in range(2):
if i == 0:
metadata['name'] = datagroup + '_left'
else:
metadata['name'] = datagroup + '_right'
swath_slice = slice(i * swath_ncell, (i + 1) * swath_ncell)
swath_lon = lon[:, swath_slice]
swath_lat = lat[:, swath_slice]
swath_wind_speed = wind_speed[:, swath_slice]
swath_wind_dir = wind_dir[:, swath_slice]
dgcp = 16. # spacing is about 12.5km
ngcps = np.ceil(np.array(swath_lon.shape) / dgcp) + 1.
pix = np.linspace(0, swath_lon.shape[1] - 1, num=ngcps[1]).round().astype('int32')
lin = np.linspace(0, swath_lon.shape[0] - 1, num=ngcps[0]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
gcplon = swath_lon[lin2d, pix2d]
gcplon[np.where(gcplon >= 180)] -= 360
if gcplon.max() - gcplon.min() > 180:
gcplon[np.where(gcplon < 0)] += 360.
gcplat = swath_lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei,
gcppix, gcplin)
band = []
indndv = np.where(ma.getmaskarray(swath_wind_speed))
offset, scale = vmin, (vmax-vmin)/254.
clipped = np.clip(ma.getdata(swath_wind_speed), vmin, vmax)
array = np.round((clipped - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('matplotlib_jet',
vmax=vmax, vmax_pal=vmax_pal,
vmin=vmin, vmin_pal=vmin_pal)
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'wind speed', 'unittype':'m/s',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'colortable':colortable})
indndv = np.where(ma.getmaskarray(swath_wind_dir))
clipped = np.clip(ma.getdata(swath_wind_dir), 0, 360)
array = np.round(clipped / 360. * 254.).astype('uint8')
array[indndv] = 255
band.append({'array':array, 'scale':360./254., 'offset':0.,
'description':'wind direction', 'unittype':'deg',
'nodatavalue':255, 'parameter_range':[0, 360.]})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
def process_slices(slices, dset, datagroup, beam, ref_dt, vmin, vmax, vmin_pal,
vmax_pal):
""""""
geod = Geod(ellps='WGS84')
# Inner, middle and outer beam widths
beam_widths = (94.0, 120.0, 156.0,)
beam_width = beam_widths[beam]
for k, chunk_rows in enumerate(slices):
_chunk_lon = dset.variables['beam_clon'][chunk_rows, beam]
chunk_size = numpy.shape(_chunk_lon)[0]
if 0 >= chunk_size:
# Empty chunk, skip
continue
chunk_lon0 = _chunk_lon[0] - 180.0
chunk_lon = numpy.mod(_chunk_lon - chunk_lon0, 360.0) + chunk_lon0
chunk_lat = dset.variables['beam_clat'][chunk_rows, beam]
values = dset.variables['SSS'][chunk_rows, beam]
values = numpy.ma.masked_where(values==-999., values)
# Build GCPs
dgcp = 32.
if numpy.max(numpy.abs(chunk_lat)) > 75.0:
dgcp = 4.
ngcplin = numpy.ceil(chunk_lon.size / dgcp).astype('int32')
_gcp_alongtrack = numpy.linspace(0, chunk_lon.size - 1, num=ngcplin)
_gcp_indices = numpy.round(_gcp_alongtrack).astype('int32')
_gcppix = numpy.array([-1.0, 0.5, 2.0])
ngcppix = _gcppix.size
gcppix = numpy.tile(_gcppix[numpy.newaxis, :],
(ngcplin, 1))
gcpind = numpy.tile(_gcp_indices[:, numpy.newaxis],
(1, ngcppix)).astype('int32')
gcplin = gcpind + 0.5
# Compute swath direction
_ind0 = numpy.minimum(gcpind, chunk_lon.size - 2)
_ind1 = _ind0 + 1
ind_same = numpy.where((chunk_lon[_ind0] == chunk_lon[_ind1]) &
(chunk_lat[_ind0] == chunk_lat[_ind1]))
for ig_line, ig_pixel in zip(ind_same[0], ind_same[1]):
if _ind1[ig_line, ig_pixel] < chunk_lon.size -1:
_ind1[ig_line, ig_pixel] += 1
else:
_ind0[ig_line, ig_pixel] -= 1
lat_diff = chunk_lat[_ind1] - chunk_lat[_ind0]
lon_diff = chunk_lon[_ind1] - chunk_lon[_ind0]
swath_dir = numpy.arctan2(lat_diff, lon_diff)
# Compute GCPs geographical coordinates from the location of the beam
# center and its width
gcphei = numpy.zeros(gcppix.shape)
gcplon = numpy.zeros(gcppix.shape)
gcplat = numpy.zeros(gcppix.shape)
for gcp_i in range(len(_gcp_indices)):
ind = _gcp_indices[gcp_i]
central_lon = chunk_lon[ind]
central_lat = chunk_lat[ind]
across_dir = -1 * numpy.rad2deg(numpy.mod(swath_dir[gcp_i][1],
2 * numpy.pi))
lon_a, lat_a, _ = geod.fwd(central_lon, central_lat,
across_dir,
1000.0 * 1.5 * beam_width)
lon_b, lat_b, _ = geod.fwd(central_lon, central_lat,
180.0 + across_dir,
1000.0 * 1.5 * beam_width)
gcplon[gcp_i][0] = lon_b
gcplon[gcp_i][1] = central_lon
gcplon[gcp_i][2] = lon_a
gcplat[gcp_i][0] = lat_b
gcplat[gcp_i][1] = central_lat
gcplat[gcp_i][2] = lat_a
# Fix longitudinal continuity
half_ind = numpy.floor(len(_gcp_indices) * 0.5).astype('int32')
gcplon0 = gcplon[half_ind, 1] - 180.0
gcplon = numpy.mod(gcplon - gcplon0, 360.0) + gcplon0
# Construct metadata/geolocation/band(s)
sec = dset.variables['sec'][chunk_rows]
start_dt = ref_dt + datetime.timedelta(seconds=sec[0])
stop_dt = ref_dt + datetime.timedelta(seconds=sec[-1])
dtime, time_range = stfmt.format_time_and_range(start_dt, stop_dt,
units='h')
metadata = {}
metadata['name'] = '{}_{}'.format(datagroup, k)
metadata['time_range'] = time_range
metadata['datetime'] = dtime
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei,
gcppix, gcplin)
# Build mask
land_frac = dset.variables['land_frac'][chunk_rows, beam]
ice_frac = dset.variables['ice_frac'][chunk_rows, beam]
scat_land_frac = dset.variables['scat_land_frac'][chunk_rows, beam]
scat_ice_frac = dset.variables['scat_ice_frac'][chunk_rows, beam]
mask = (values.mask |
(land_frac > 0.1) |
(ice_frac > 0.1)) # |
# (scat_land_frac > 0.001) |
# (scat_ice_frac > 0.001))
# Pack data
band = []
offset, scale = vmin, (vmax - vmin) / 254.
numpy.clip(values.data, vmin, vmax, out=values.data)
array = numpy.round((values.data - offset) / scale).astype('uint8')
array[numpy.where(mask)] = 255
colortable = stfmt.format_colortable('matplotlib_jet',
vmin=vmin, vmax=vmax,
vmin_pal=vmin_pal,
vmax_pal=vmax_pal)
array = array[:, numpy.newaxis]
band.append({'array':array,
'scale':scale,
'offset':offset,
'description':'sea surface salinity',
'unittype':'PSS',
'nodatavalue':255,
'parameter_range':[vmin, vmax],
'colortable':colortable})
yield metadata, geolocation, band
def sentinel3_olci(infile,
outdir,
vmin=None,
vmax=None,
slope_threshold=-0.00001,
channels='nir',
write_netcdf=False,
lut_path=None,
log_path=None,
lat_crop=85.0):
""""""
t0 = datetime.utcnow()
if type(channels) is list or type(channels) is tuple:
bandnames = channels
product_name = 'Sentinel-3_OLCI'
elif 'nir' == channels:
bandnames = ('Oa17', )
product_name = 'Sentinel-3_OLCI_NIR'
elif 'true_rgb' == channels:
bandnames = ('Oa09', 'Oa06', 'Oa04')
product_name = 'Sentinel-3_OLCI_true_RGB'
elif 'false_rgb' == channels:
bandnames = ('Oa17', 'Oa06', 'Oa04')
product_name = 'Sentinel-3_OLCI_false_RGB'
else:
raise Exception('channels must be either "nir", "true_rgb" '
'or "false_rgb"')
syntool_stats = {}
for bandname in bandnames:
syntool_stats[bandname] = {}
if log_path is not None and not os.path.exists(log_path):
try:
os.makedirs(log_path)
except OSError:
_, e, _ = sys.exc_info()
if e.errno != errno.EEXIST:
raise
# convert band into column number in the LUT
band_columns = [int(x[2:]) + 3 - 1 for x in bandnames]
if vmin is None:
vmin = [None] * len(bandnames)
if vmax is None:
vmax = [None] * len(bandnames)
full_path = os.path.normpath(infile)
file_path = os.path.basename(full_path)
file_name, _ = os.path.splitext(file_path)
geo_path = os.path.join(infile, 'geo_coordinates.nc')
time_path = os.path.join(infile, 'time_coordinates.nc')
quality_path = os.path.join(infile, 'qualityFlags.nc')
# Extract geo coordinates information
geo_handler = netCDF4.Dataset(geo_path, 'r')
nrow = geo_handler.dimensions['rows'].size
nrow_all = nrow
ncell = geo_handler.dimensions['columns'].size
ncell_all = ncell
lon = geo_handler.variables['longitude'][:]
tie_lon = numpy.ma.array(lon)
lat = geo_handler.variables['latitude'][:]
tie_lat = numpy.ma.array(lat)
geo_handler.close()
# Handle longitude continuity
dlon = lon[1:, :] - lon[:-1, :]
if 180.0 <= numpy.max(numpy.abs(dlon)):
lon[lon < 0.0] = lon[lon < 0.0] + 360.0
# Extract time coordinates information
time_handler = netCDF4.Dataset(time_path, 'r')
start_timestamp = time_handler.variables['time_stamp'][0]
end_timestamp = time_handler.variables['time_stamp'][-1]
timestamp_units = time_handler.variables['time_stamp'].units
time_handler.close()
# Format time information
start_time = netCDF4.num2date(start_timestamp, timestamp_units)
end_time = netCDF4.num2date(end_timestamp, timestamp_units)
(dtime, time_range) = stfmt.format_time_and_range(start_time,
end_time,
units='ms')
parameters = ['{} TOA radiance'.format(bnd) for bnd in bandnames]
metadata = {}
metadata['product_name'] = product_name
metadata['name'] = file_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'ESA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = parameters
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'medium-resolution imaging spectrometer'
metadata['sensor_name'] = 'OLCI'
metadata['sensor_platform'] = 'Sentinel-3'
# Crop high latitude to avoid projection issues
LAT_MAX = 89.0
ind_valid_cols = numpy.where(numpy.abs(tie_lat).max(axis=0) <= LAT_MAX)[0]
slice_lat0 = slice(None)
slice_lat1 = slice(numpy.min(ind_valid_cols),
numpy.max(ind_valid_cols) + 1)
tie_lat = tie_lat[slice_lat0, slice_lat1]
tie_lon = tie_lon[slice_lat0, slice_lat1]
nrow, ncell = tie_lat.shape
# Handle longitude continuity
dlon = tie_lon[1:, :] - tie_lon[:-1, :]
if 180.0 <= numpy.max(numpy.abs(dlon)):
lon0 = tie_lon[0, 0] + 180.0
tie_lon[:, :] = numpy.mod(tie_lon[:, :] - lon0, 360.0) + lon0
# Compute GCPs
tie_row = numpy.linspace(0, nrow - 1, num=tie_lon.shape[0])
tie_cell = numpy.linspace(0, ncell - 1, num=tie_lon.shape[1])
tie_facrow = (nrow - 1.) / (tie_lon.shape[0] - 1.)
tie_faccell = (ncell - 1.) / (tie_lon.shape[1] - 1.)
gcp_fac = 128
gcp_fac = numpy.maximum(gcp_fac, numpy.maximum(tie_faccell, tie_facrow))
gcp_nrow = numpy.ceil((nrow - 1.) / gcp_fac).astype('int') + 1
gcp_ncell = numpy.ceil((ncell - 1.) / gcp_fac).astype('int') + 1
tie_indrow = numpy.round(
numpy.linspace(0, tie_lon.shape[0] - 1, num=gcp_nrow)).astype('int')
tie_indcell = numpy.round(
numpy.linspace(0, tie_lon.shape[1] - 1, num=gcp_ncell)).astype('int')
gcp_lon = tie_lon[tie_indrow.reshape((-1, 1)),
tie_indcell.reshape((1, -1))]
gcp_lat = tie_lat[tie_indrow.reshape((-1, 1)),
tie_indcell.reshape((1, -1))]
gcp_row = numpy.tile(tie_row[tie_indrow].reshape((-1, 1)) + 0.5,
(1, gcp_ncell))
gcp_cell = numpy.tile(tie_cell[tie_indcell].reshape((1, -1)) + 0.5,
(gcp_nrow, 1))
gcp_hei = numpy.zeros(gcp_lon.shape)
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcp_lon, gcp_lat, gcp_hei,
gcp_cell, gcp_row)
syntool_stats['lon_min'] = float(numpy.min(tie_lon))
syntool_stats['lon_max'] = float(numpy.max(tie_lon))
syntool_stats['lat_min'] = float(numpy.min(tie_lat))
syntool_stats['lat_max'] = float(numpy.max(tie_lat))
# Compute atmospheric radiance TOA correction
L_toa = None
if lut_path is not None:
t_start = datetime.utcnow()
# Compute atmospheric correction for the whole granule
L_toa = atmospheric_correction(lut_path, infile, band_columns,
nrow_all, ncell_all)
# Extract the slices of atmospheric correction that match the ones of
# the bands after the removal of the high latitude columns.
for iband in range(len(L_toa)):
L_toa[iband] = L_toa[iband][slice_lat0, slice_lat1]
# An erroneous LUT could produce nan values for the atmospheric
# correction. This is not acceptable.
if numpy.any(~numpy.isfinite(L_toa[iband])):
raise Exception('Infinite or nan value found in the '
'atmospheric correction, please check that the'
'LUT has valid values for the angles contained'
'in the instrument_data.nc file')
t_stop = datetime.utcnow()
syntool_stats['lut_computation'] = (t_stop - t_start).total_seconds()
logger.info('Construct bands')
t_start = datetime.utcnow()
bands = []
dset = netCDF4.Dataset(quality_path, 'r')
quality_flags = dset.variables['quality_flags'][slice_lat0, slice_lat1]
dset.close()
bits = {
'saturated@Oa21': 0,
'saturated@Oa20': 1,
'saturated@Oa19': 2,
'saturated@Oa18': 3,
'saturated@Oa17': 4,
'saturated@Oa16': 5,
'saturated@Oa15': 6,
'saturated@Oa14': 7,
'saturated@Oa13': 8,
'saturated@Oa12': 9,
'saturated@Oa11': 10,
'saturated@Oa10': 11,
'saturated@Oa09': 12,
'saturated@Oa08': 13,
'saturated@Oa07': 14,
'saturated@Oa06': 15,
'saturated@Oa05': 16,
'saturated@Oa04': 17,
'saturated@Oa03': 18,
'saturated@Oa02': 19,
'saturated@Oa01': 20,
'dubious': 21,
'sun-glint_risk': 22,
'duplicated': 23,
'cosmetic': 24,
'invalid': 25,
'straylight_risk': 26,
'bright': 27,
'tidal_region': 28,
'fresh_inland_water': 29,
'coastline': 30,
'land': 31
}
off_flags = numpy.uint32(0)
off_flags = off_flags + numpy.uint32(1 << bits['dubious'])
off_flags = off_flags + numpy.uint32(1 << bits['invalid'])
off_flags = off_flags + numpy.uint32(1 << bits['straylight_risk'])
off_flags = off_flags + numpy.uint32(1 << bits['bright'])
off_flags = off_flags + numpy.uint32(1 << bits['tidal_region'])
# off_flags = off_flags + numpy.uint32(1 << bits['fresh_inland_water'])
off_flags = off_flags + numpy.uint32(1 << bits['coastline'])
off_flags = off_flags + numpy.uint32(1 << bits['land'])
# Filter out values where any of the bands is flagged as saturated
for bandname in bandnames:
bit_name = 'saturated@{}'.format(bandname)
off_flags = off_flags + numpy.uint32(1 << bits[bit_name])
lat_mask = (numpy.abs(tie_lat) > lat_crop)
data_mask = numpy.zeros(quality_flags.shape, dtype='bool')
data_mask = (data_mask | lat_mask)
if numpy.all(data_mask):
logger.warn('No data to extract.')
sys.exit(0)
contrast_mask = numpy.zeros(quality_flags.shape, dtype='bool')
contrast_mask = (contrast_mask | lat_mask)
contrast_mask = (contrast_mask |
(numpy.bitwise_and(quality_flags, off_flags) > 0))
t_stop = datetime.utcnow()
syntool_stats['mask_computation'] = (t_stop - t_start).total_seconds()
t_start = datetime.utcnow()
_vmin = list(vmin)
_vmax = list(vmax)
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
logger.info('\tConstruct {} band'.format(bandname))
fieldname = '{}_radiance'.format(bandname)
file_path = os.path.join(infile, '{}.nc'.format(fieldname))
f_handler = netCDF4.Dataset(file_path, 'r')
band = f_handler.variables[fieldname][:]
band = numpy.ma.array(band)
f_handler.close()
# Apply atmospheric correction
band = band[slice_lat0, slice_lat1]
if L_toa is not None:
band -= L_toa[band_index][:, :]
# Mask null and negative values: they are inferior or equal to
# atmospheric correction and should probably have be flagged as clouds.
mask_negative = (band <= 0.0)
logger.info('\tSet contrast')
valid_ratio_lower_threshold = 0.001 # 0.1%
valid_data_mask = (band.mask | contrast_mask | mask_negative)
valid_data = band.data[~valid_data_mask]
valid_ratio = float(valid_data.size) / float(band.data.size)
syntool_stats[bandname]['valid_ratio'] = valid_ratio
if valid_ratio_lower_threshold >= valid_ratio:
logger.warn('No valid values for {}'.format(bandname))
logger.warn('Using default min/max.')
# Use arbitrary extrema on land
if _vmin[band_index] is None:
_min = default_minmax[bandname][0]
_vmin[band_index] = _min
syntool_stats[bandname]['default_min'] = float(_min)
if _vmax[band_index] is None:
_max = default_minmax[bandname][1]
_vmax[band_index] = _max
syntool_stats[bandname]['default_max'] = float(_max)
else:
# TODO: add clipping for NIR
# _min = numpy.clip(_min, 1.5, 10.0)
# _max = numpy.clip(_max, 30.0, 60.0)
if _vmin[band_index] is None:
_min = numpy.percentile(valid_data, .5)
_vmin[band_index] = _min
syntool_stats[bandname]['p0050'] = float(_min)
syntool_stats[bandname]['min'] = float(numpy.min(valid_data))
if _vmax[band_index] is None:
_max = numpy.percentile(valid_data, 99.99)
_vmax[band_index] = _max
p99 = numpy.percentile(valid_data, 99.0)
syntool_stats[bandname]['p9900'] = float(p99)
syntool_stats[bandname]['p9999'] = float(_max)
syntool_stats[bandname]['max'] = float(numpy.max(valid_data))
logger.info('\tContrast : vmin={} / vmax={}'.format(
_vmin[band_index], _vmax[band_index]))
min_values = [_vmin[band_index] for band_index in range(len(bandnames))]
max_values = [_vmax[band_index] for band_index in range(len(bandnames))]
t_stop = datetime.utcnow()
syntool_stats['minmax_computation'] = (t_stop - t_start).total_seconds()
syntool_stats['final_min'] = float(numpy.min(min_values))
syntool_stats['final_max'] = float(numpy.max(max_values))
_min = numpy.log(numpy.min(min_values))
_max = numpy.log(numpy.max(max_values))
scale = (_max - _min) / 254.
offset = _min
for band_index in range(len(bandnames)):
bandname = bandnames[band_index]
logger.info('\tConstruct {} band'.format(bandname))
fieldname = '{}_radiance'.format(bandname)
file_path = os.path.join(infile, '{}.nc'.format(fieldname))
f_handler = netCDF4.Dataset(file_path, 'r')
band = f_handler.variables[fieldname][:]
band = numpy.ma.array(band)
f_handler.close()
# Apply atmospheric correction
band = band[slice_lat0, slice_lat1]
if L_toa is not None:
band -= L_toa[band_index][:, :]
# Mask null and negative values: they are inferior or equal to
# atmospheric correction and should probably have be flagged as clouds.
mask_negative = (band <= 0.0)
# Compute the logarithm only for radiance values that are higher than
# the atmospheric correction.
bnd = numpy.log(band.data, where=(~mask_negative))
logger.info('\tBytescaling')
byte = bytescale(bnd, cmin=_min, cmax=_max, low=0, high=254)
description = '{} radiance (log)'.format(bandname)
if band.mask is not numpy.ma.nomask:
byte[band.mask] = 255
# mask data for extreme latitudes
byte[data_mask] = 255
# Pixels with a radiance equal or inferior to atmospheric correction
# are clipped to the minimal value.
if 0 < mask_negative.size:
byte[numpy.where(mask_negative == True)] = 0 # noqa
band_range = [_vmin[band_index], _vmax[band_index]]
bands.append({
'array': byte,
'scale': scale,
'offset': offset,
'description': description,
'unittype': '',
'nodatavalue': 255,
'parameter_range': band_range
})
if write_netcdf:
bands[-1]['name'] = bandname
bands[-1]['long_name'] = bandname
bands[-1]['unittype'] = '1'
logger.info('Make sure nodata are at the same locations in all bands')
mask = numpy.any([_band['array'] == 255 for _band in bands], axis=0)
for band in bands:
band['array'][mask] = 255
t_stop = datetime.utcnow()
syntool_stats['bytescaling'] = (t_stop - t_start).total_seconds()
if write_netcdf:
metadata['spatial_resolution'] = 300
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
bands,
'swath',
ngcps=gcp_lon.shape)
else:
logger.info('Write geotiff')
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, bands)
logger.info(datetime.utcnow() - t0)
syntool_stats['total_time'] = (datetime.utcnow() - t0).total_seconds()
if log_path is not None:
import json
stats_path = os.path.join(log_path, '{}.json'.format(file_name))
with open(stats_path, 'w') as f:
json.dump(syntool_stats, f)
def sar_doppler(infile, outdir, pngkml=False,
vmin=-2.5, vmax=2.5, vmin_pal=-2.5, vmax_pal=2.5):
"""
"""
# Read/Process data
print 'Read/Process data'
sardop = SAFEOCNNCFile(infile, product='DOPPLER')
mission = sardop.read_global_attribute('missionName')
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
else:
raise Exception('S1A mission expected.')
start_time = sardop.get_start_time()
stop_time = sardop.get_end_time()
heading = sardop.read_values('rvlHeading')
if np.sin((90 - heading.mean()) * np.pi / 180) > 0:
sensor_pass = '******'
else:
sensor_pass = '******'
safe_name = os.path.basename(os.path.dirname(os.path.dirname(infile)))
sensor_mode = safe_name.split('_')[1]
if sensor_mode not in ['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'IW', 'EW']:
raise Exception('S[1-6]/IW/EW modes expected.')
sensor_swath = os.path.basename(infile).split('-')[1].upper()
sensor_polarisation = sardop.read_global_attribute('polarisation')
datagroup = safe_name.replace('.SAFE', '')
pid = datagroup.split('_')[-1]
dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
if 'rvlSwath' in sardop.get_dimensions():
nswath = sardop.get_dimsize('rvlSwath')
else:
nswath = 1
for iswath in range(nswath):
if nswath == 1:
radvel = sardop.read_values('rvlRadVel')
landflag = sardop.read_values('rvlLandFlag')
lon = sardop.read_values('lon')
lat = sardop.read_values('lat')
name = dataname
else:
radvel = sardop.read_values('rvlRadVel')[:, :, iswath]
landflag = sardop.read_values('rvlLandFlag')[:, :, iswath]
lon = sardop.read_values('lon')[:, :, iswath]
lat = sardop.read_values('lat')[:, :, iswath]
valid = np.where((ma.getmaskarray(lon) == False) & \
(ma.getmaskarray(lat) == False))
slices = [slice(valid[0].min(), valid[0].max() + 1),
slice(valid[1].min(), valid[1].max() + 1)]
radvel = radvel[slices]
landflag = landflag[slices]
lon = lon[slices]
lat = lat[slices]
name = dataname + '-' + str(iswath+1)
if sensor_pass == 'Ascending':
radvel *= -1
ngcps = np.ceil(np.array(lon.shape) / 10.) + 1
pix = np.linspace(0, lon.shape[1] - 1, num=ngcps[1]).round().astype('int32')
lin = np.linspace(0, lon.shape[0] - 1, num=ngcps[0]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
if gcplon.min() < -135 and gcplon.max() > 135:
gcplon[np.where(gcplon < 0)] += 360.
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time,
units='ms')
metadata['product_name'] = 'SAR_doppler'
metadata['name'] = name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = '' #'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'radial horizontal velocities'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei,
gcppix, gcplin)
band = []
#indndv = np.where(landflag != 0)
offset, scale = vmin, (vmax-vmin)/254.
np.clip(radvel, vmin, vmax, out=radvel)
array = np.round((radvel - offset) / scale).astype('uint8')
#array[indndv] = 255
colortable = stfmt.format_colortable('doppler', vmax=vmax, vmax_pal=vmax_pal,
vmin=vmin, vmin_pal=vmin_pal)
band.append({'array':array, 'scale':scale, 'offset':offset,
'description':'radial horizontal velocities', 'unittype':'m/s',
'nodatavalue':255, 'parameter_range':[vmin, vmax],
'colortable':colortable})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
def ascat_l2b(infile,
outdir,
vmin=0.,
vmax=25.4,
vmin_pal=0.,
vmax_pal=50 * 0.514,
write_netcdf=False):
"""
"""
# Read/Process data
dset = Dataset(infile)
nrow = len(dset.dimensions['NUMROWS'])
ncell = len(dset.dimensions['NUMCELLS'])
if ncell != 82:
raise Exception('Expects NUMCELLS=82 (KNMI ASCAT L2B 12.5km).')
source = dset.source
if 'metop-a' in source.lower():
platform = 'Metop-A'
elif 'metop-b' in source.lower():
platform = 'Metop-B'
else:
raise Exception('Platform ?')
start_time = datetime.strptime(dset.start_date + dset.start_time,
'%Y-%m-%d%H:%M:%S')
stop_time = datetime.strptime(dset.stop_date + dset.stop_time,
'%Y-%m-%d%H:%M:%S')
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
datagroup = os.path.splitext(os.path.basename(infile))[0]
if write_netcdf == False:
# Create GeoTIFF file
_to_geotiff(infile, outdir, vmin, vmax, vmin_pal, vmax_pal, nrow,
ncell, dtime, time_range, datagroup, platform, dset)
else:
for i in range(2):
if i == 0: # left swath
swath_slice = [slice(0, nrow), slice(0, ncell / 2)]
dataset_name = '{}_left'.format(datagroup)
else: # right swath
swath_slice = [slice(0, nrow), slice(ncell / 2, ncell)]
dataset_name = '{}_right'.format(datagroup)
lat = dset.variables['lat'][swath_slice]
lon = dset.variables['lon'][swath_slice]
irow = 0
while irow < nrow:
lon0 = lon[irow, 0] - 180
lon[irow:, :] = np.mod(lon[irow:, :] - lon0, 360) + lon0
notcont = (lon[irow + 1:, :] < lon0 + 90) & (lon[irow:-1, :] > lon0 + 270) | \
(lon[irow + 1:, :] > lon0 + 270) & (lon[irow:-1, :] < lon0 + 90)
indnotcont = np.where(notcont.any(axis=1))[0]
if indnotcont.size == 0:
irow = nrow
else:
indnotcont = indnotcont.min()
if indnotcont == 0:
raise Exception('Unexpected longitudes.')
irow = irow + indnotcont
ind = np.where(np.abs(lon[1:, :] - lon[:-1, :]) > 180.)
if ind[0].size != 0:
raise Exception('Failed to make longitudes continuous.')
if lon[nrow / 2, ncell / 4] > 180:
lon -= 360
elif lon[nrow / 2, ncell / 4] < -180:
lon += 360
print(lon)
wind_speed = dset.variables['wind_speed'][swath_slice]
wind_dir = dset.variables['wind_dir'][swath_slice]
wind_dir = np.mod(90. - wind_dir, 360.)
dgcp = 16.
ngcps = (np.ceil(np.array(lon.shape) / dgcp) + 1.).astype('int32')
pix = np.linspace(0, lon.shape[1] - 1,
num=ngcps[1]).round().astype('int32')
lin = np.linspace(0, lon.shape[0] - 1,
num=ngcps[0]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
# Construct metadata/geolocation/band(s)
print('Construct metadata/geolocation/band(s)')
metadata = {}
metadata['product_name'] = '{}_ASCAT_L2B'.format(platform)
metadata['datagroup'] = datagroup
metadata['name'] = dataset_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(
datetime.utcnow())
metadata['parameter'] = ['wind speed', 'wind direction']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(
gcplon, gcplat, gcphei, gcppix, gcplin)
print('Write netcdf')
# u/v -> bands
band = []
u = wind_speed * np.cos(np.deg2rad(wind_dir))
v = wind_speed * np.sin(np.deg2rad(wind_dir))
mask = np.ma.getmaskarray(u) | np.ma.getmaskarray(v)
vmin = -vmax
offset, scale = vmin, (vmax - vmin) / 254.
clipped = np.clip(np.ma.getdata(u), vmin, vmax)
array = np.round((clipped - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind u',
'unittype': 'm s-1',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'name': 'u',
'standard_name': 'eastward_wind'
})
clipped = np.clip(np.ma.getdata(v), vmin, vmax)
array = np.round((clipped - offset) / scale).astype('uint8')
array[mask] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind v',
'unittype': 'm s-1',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'name': 'v',
'standard_name': 'northward_wind'
})
# Write
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
metadata['spatial_resolution'] = 12500.
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def sar_wave(infile, outdir, pngkml=False):
"""
"""
# Read/Process data
print 'Read/Process data'
sarwave = SAFEOCNNCFile(infile, product='WAVE')
mission = sarwave.read_global_attribute('missionName')
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
else:
raise Exception('S1A mission expected.')
# start_time = sarwave.get_start_time() # WARNING : whole SAFE for imagettes !
# stop_time = sarwave.get_end_time() # WARNING : whole SAFE for imagettes !
start_t = sarwave.read_global_attribute('firstMeasurementTime')
if '.' in start_t:
start_time = datetime.strptime(start_t, '%Y-%m-%dT%H:%M:%S.%fZ')
else:
start_time = datetime.strptime(start_t, '%Y-%m-%dT%H:%M:%SZ')
stop_t = sarwave.read_global_attribute('lastMeasurementTime')
if '.' in stop_t:
stop_time = datetime.strptime(stop_t, '%Y-%m-%dT%H:%M:%S.%fZ')
else:
stop_time = datetime.strptime(stop_t, '%Y-%m-%dT%H:%M:%SZ')
heading = sarwave.read_values('oswHeading')
if np.sin((90 - heading[0, 0]) * np.pi / 180) > 0:
sensor_pass = '******'
else:
sensor_pass = '******'
safe_name = os.path.basename(os.path.dirname(os.path.dirname(infile)))
sensor_mode = safe_name.split('_')[1]
if sensor_mode not in ['WV', 'S1', 'S2', 'S3', 'S4', 'S5', 'S6']:
raise Exception('WV/S[1-6] modes expected.')
sensor_swath = os.path.basename(infile).split('-')[1].upper()
sensor_polarisation = sarwave.read_global_attribute('polarisation')
datagroup = safe_name.replace('.SAFE', '')
pid = datagroup.split('_')[-1]
dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
# Make wave spectrum figure
spec = sarwave.read_values('oswPolSpec')
k = sarwave.read_values('oswK')
phi = sarwave.read_values('oswPhi')
npartitions = sarwave.get_dimsize('oswPartitions')
partitions = sarwave.read_values('oswPartitions')
# TMP Bug : there are now 3 partitions, numbered 0, 1 and 3 ...
if npartitions == 3:
indp2 = np.where(partitions == 2)
indp3 = np.where(partitions == 3)
if indp2[0].size == 0 and indp3[0].size != 0:
partitions[indp3] = 2
# /TMP
hs = sarwave.read_values('oswHs')
flag = sarwave.read_values('oswLandFlag')
if sensor_mode == 'WV':
imnum = int(
os.path.splitext(os.path.basename(infile))[0].split('-')[-1])
else:
imnum = None
spec_size = (512, 512)
fontsize = 'small'
cmap = getColorMap(rgbFile='wind.pal')
fig = make_spec_fig(spec,
k,
phi,
heading,
npartitions,
partitions,
hs,
flag,
imnum=imnum,
spec_size=spec_size,
fontsize=fontsize,
cmap=cmap)
rgb = fig2rgb(fig)
plt.close(fig)
# Make geolocation
if sensor_mode == 'WV':
lon = sarwave.read_values('lon')[0, 0]
lat = sarwave.read_values('lat')[0, 0]
grdrasize = sarwave.read_values('oswGroundRngSize')[0, 0]
grdazsize = sarwave.read_values('oswAziSize')[0, 0]
geod = pyproj.Geod(ellps='WGS84')
lons = np.repeat(lon, 2)
lats = np.repeat(lat, 2)
az = heading[0, 0] + [0., 180.]
dist = np.repeat(grdazsize / 2., 2)
lonsmid, latsmid, dummy = geod.fwd(lons, lats, az, dist)
lons = np.repeat(lonsmid, 2)
lats = np.repeat(latsmid, 2)
az = heading[0, 0] + [-90, 90., 90., -90.]
dist = np.repeat(grdrasize / 2., 4)
gcplon, gcplat, dummy = geod.fwd(lons, lats, az, dist)
gcppix = np.array([0, spec_size[0], spec_size[0], 0])
gcplin = np.array([0, 0, spec_size[1], spec_size[1]])
if np.sin((90 - heading[0, 0]) * np.pi / 180) < 0:
# descending pass
gcppix = spec_size[0] - gcppix
gcplin = spec_size[1] - gcplin
gcphei = np.zeros(gcplin.size)
else:
gcplon = sarwave.read_values('lon')
gcplat = sarwave.read_values('lat')
gcphei = np.zeros(gcplon.shape)
nra = sarwave.get_dimsize('oswRaSize')
pix = np.arange(nra) * spec_size[0] + spec_size[0] / 2.
naz = sarwave.get_dimsize('oswAzSize')
lin = np.arange(naz - 1, -1, -1) * spec_size[1] + spec_size[1] / 2.
gcppix, gcplin = np.meshgrid(pix, lin)
if np.sin((90 - heading[0, 0]) * np.pi / 180) < 0:
# descending pass
gcppix = nra * spec_size[0] - gcppix
gcplin = naz * spec_size[1] - gcplin
if gcplon.min() < -135 and gcplon.max() > 135:
gcplon[np.where(gcplon < 0)] += 360.
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
metadata['product_name'] = 'SAR_wave'
metadata['name'] = dataname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = '' #'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = ''
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
band = []
band.append({'array': rgb[:, :, 0]})
band.append({'array': rgb[:, :, 1]})
band.append({'array': rgb[:, :, 2]})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
def _to_geotiff(infile, outdir, vmin, vmax, vmin_pal, vmax_pal, nrow, ncell,
dtime, time_range, datagroup, platform, dset):
""" """
# It is necessary to create a 1-row overlap between the two half-orbits in
# order for the merger to build a continuous shape
extra_row = 1
side_slices = {
'left': slice(0, ncell / 2),
'right': slice(ncell / 2, ncell)
}
for side in side_slices:
side_slice = side_slices[side]
side_lat = dset.variables['lat'][:, side_slice]
side_lon = dset.variables['lon'][:, side_slice]
# Make longitudes continuous but detect gaps bigger than 90 degrres:
# it will be necessary to split the dataset where these gaps appear
# because gdal may not be able to interpolate GCPs correctly later
# in the ingestor (it says the shape intersects itself).
side_lon, splits = make_continuous_lon(side_lon, max_gap=60.)
# Add begin and end to the splits list
splits.append(0)
splits.append(nrow)
# Add an artificial split in the middle of the swath
half_split = nrow / 2
splits.append(half_split)
# Make sure there are no duplicates and sort the indices in ascending
# order
splits = list(set(splits))
splits.sort()
l = len(splits)
parts = []
extra_row_part = -1
for i in range(l - 1):
extra_row = 0
if splits[i + 1] == half_split:
# Since the half split does not correspond to a void area in
# the original data, add an extra row so that there will be no
# visible cut when the ingestor merges the parts back together
extra_row_part = i
extra_row = 1
part = slice(splits[i], extra_row + splits[i + 1])
parts.append(part)
nparts = len(parts)
last_lon = None
last_lat = None
for part_no in range(0, nparts):
dataset_name = '{}_{}_{}'.format(datagroup, side, part_no)
part_slice = parts[part_no]
swath_slice = [part_slice, side_slice]
lat = side_lat[part_slice]
lon = side_lon[part_slice]
# Make sure there are no longitudes < -180 or the rastertiles
# plugin of syntool-ingestor will not produce anything
while 180. < np.min(lon):
lon = lon - 360.
while -180. > np.min(lon):
lon = lon + 360.
# Extract wind speed (module)
wind_speed = dset.variables['wind_speed'][swath_slice]
# Extract wind direction and make it counter-clockwise from east
# (as expected by the ingestor)
wind_dir = dset.variables['wind_dir'][swath_slice]
wind_dir = np.mod(90. - wind_dir, 360.)
# Build GCPs
dgcp = 16.
ngcps = (np.ceil(np.array(lon.shape) / dgcp) + 1.).astype('int32')
pix = np.linspace(0, lon.shape[1] - 1, num=ngcps[1]).round()
lin = np.linspace(0, lon.shape[0] - 1, num=ngcps[0]).round()
pix2d, lin2d = np.meshgrid(pix.astype('int32'),
lin.astype('int32'))
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(ngcps)
if part_no == extra_row_part + 1:
# Overwrite first GCP, but first you must make sure that
# longitudes are in the same -180/+180 range.
lon0 = gcplon[1, 0] - 180.
last_lon[:] = np.mod(last_lon[:] - lon0, 360.) + lon0
gcplon[0] = last_lon
gcplat[0] = last_lat
if part_no == extra_row_part:
# Mask last row since it also exists in the next part
wind_speed[-extra_row, :] = numpy.ma.masked
wind_dir[-extra_row, :] = numpy.ma.masked
# Store last GCPs lat/lon
extra_row = 1
last_lon = gcplon[-extra_row]
last_lat = gcplat[-extra_row]
# Construct metadata/geolocation/band(s)
print('Construct metadata/geolocation/band(s)')
utcnow = datetime.utcnow()
metadata = {}
metadata['product_name'] = '{}_ASCAT_L2B'.format(platform)
metadata['datagroup'] = datagroup
metadata['name'] = dataset_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(utcnow)
metadata['parameter'] = ['wind speed', 'wind direction']
metadata['original_x_size'] = ncell
metadata['original_y_size'] = nrow
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(
gcplon, gcplat, gcphei, gcppix, gcplin)
print('Write geotiff')
band = []
offset, scale = vmin, (vmax - vmin) / 254.
clipped = np.clip(np.ma.getdata(wind_speed), vmin, vmax)
array = np.round((clipped - offset) / scale).astype('uint8')
array[np.ma.getmaskarray(wind_speed)] = 255
colortable = stfmt.format_colortable('noaa_wind',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind speed',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
clipped = np.clip(np.ma.getdata(wind_dir), 0, 360)
array = np.round(clipped / 360. * 254.).astype('uint8')
array[np.ma.getmaskarray(wind_dir)] = 255
band.append({
'array': array,
'scale': 360. / 254.,
'offset': 0.,
'description': 'wind direction',
'unittype': 'deg',
'nodatavalue': 255,
'parameter_range': [0, 360.]
})
tifffile = stfmt.format_tifffilename(outdir,
metadata,
create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
def sar_roughness(infile,
outdir,
pngkml=False,
contrast=None,
vmin=None,
vmax=None,
landmaskpath=None,
write_netcdf=False,
gcp2height=0):
"""
"""
# Read/Process data
print 'Read/Process data'
sarmp = SAFEGeoTiffFile(infile)
sarim = SARImage(sarmp)
mission = sarim.get_info('mission')
if mission == 'S1A':
sensor_name = 'Sentinel-1A'
sensor_platform = 'Sentinel-1A'
source_provider = 'ESA'
elif mission == 'S1B':
sensor_name = 'Sentinel-1B'
sensor_platform = 'Sentinel-1B'
source_provider = 'ESA'
else:
raise Exception('Unknown mission')
timefmt = '%Y-%m-%dT%H:%M:%S.%f'
start_time = datetime.strptime(sarim.get_info('start_time'), timefmt)
stop_time = datetime.strptime(sarim.get_info('stop_time'), timefmt)
sensor_pass = sarim.get_info('pass')
sensor_mode = sarim.get_info('mode')
sensor_swath = sarim.get_info('swath')
sensor_polarisation = sarim.get_info('polarisation')
product = sarim.get_info('product')
if product == 'GRD':
spacing = [2, 2]
elif product == 'SLC':
if sensor_mode == 'WV':
mspacing = (15, 15)
elif re.match(r'^S[1-6]$', sensor_mode) != None:
mspacing = (15, 15)
elif sensor_mode == 'IW':
raise Exception('sar_roughness for IW SLC ?')
elif sensor_mode == 'EW':
raise Exception('sar_roughness for EW SLC ?')
else:
raise Exception('Unkown S1 mode : {}'.format(sensor_mode))
spacing = np.round(sarim.meters2pixels(mspacing))
else:
raise Exception('Unkown S1 product : {}'.format(product))
mspacing = sarim.pixels2meters(spacing)
datagroup = sarim.get_info('safe_name').replace('.SAFE', '')
pid = datagroup.split('_')[-1]
dataname = os.path.splitext(os.path.basename(infile))[0] + '-' + pid
ssr = np.sqrt(sarim.get_data('roughness', spacing=spacing))
########## TMP calibration constant ##########
# if sensor_mode == 'WV':
# caldir = '/home/cercache/project/mpc-sentinel1/analysis/s1_data_analysis/L1/WV/S1A_WV_SLC__1S/cal_cste'
# if sensor_polarisation == 'HH':
# if sensor_swath == 'WV1':
# caltmp = (55.80+56.91)/2.
# calname = 'cal_cste_hh_wv1.pkl'
# elif sensor_swath == 'WV2':
# caltmp = (40.65+40.32)/2.
# calname = 'cal_cste_hh_wv2.pkl'
# elif sensor_polarisation == 'VV':
# if sensor_swath == 'WV1':
# caltmp = 58.24
# calname = 'cal_cste_vv_wv1.pkl'
# elif sensor_swath == 'WV2':
# caltmp = 49.02
# calname = 'cal_cste_vv_wv2.pkl'
# calpath = os.path.join(caldir, calname)
# if os.path.exists(calpath) == True:
# caltmp = get_caltmp(calpath, start_time)
# elif re.match(r'^S[1-6]$', sensor_mode) != None:
# if start_time < datetime(2014, 7, 16, 0, 0, 0):
# if sensor_mode == 'S6':
# raise Exception('S6 calibration missing')
# sm2cal = {'S1':58., 'S2':56., 'S3':52., 'S4':52., 'S5':49.}
# else:
# # from commissioning phase report
# sm2cal = {'S1':3., 'S2':5., 'S3':-1.5, 'S4':4., 'S5':1., 'S6':4.75}
# caltmp = sm2cal[sensor_mode]
# elif sensor_mode == 'IW':
# if start_time < datetime(2014, 7, 16, 0, 0, 0):
# caltmp = 109.
# else:
# caltmp = 3. # from commissioning phase report
# elif sensor_mode == 'EW':
# if start_time < datetime(2014, 7, 16, 0, 0, 0):
# caltmp = 94.
# else:
# caltmp = -1. # <- -2. # from commissioning phase report
# else:
# raise Exception('Which tmp calibration constant for this mode ?')
# print '--> caltmp=%f' % caltmp
# ssr *= np.sqrt(10 ** (caltmp / 10.))
########## /TMP calibration constant ##########
dim = ssr.shape
# Set contrast
if vmin == None or vmax == None:
if contrast == None:
if sensor_mode == 'WV':
contrast = 'relative'
else:
contrast = 'sea'
if contrast == 'relative':
if sensor_mode == 'WV':
noborder = [
slice(int(dim[0] * .05), int(dim[0] * .95)),
slice(int(dim[1] * .05), int(dim[1] * .95))
]
else:
noborder = [
slice(int(dim[0] * .05), int(dim[0] * .95)),
slice(int(dim[1] * .1), int(dim[1] * .9))
]
values = ssr[noborder]
if landmaskpath != None and os.path.exists(landmaskpath):
lmspacing = np.round(sarim.meters2pixels(111.32 / 120 * 1000))
lmspacing -= np.mod(lmspacing, spacing)
lon = sarim.get_data('lon', spacing=lmspacing)
lat = sarim.get_data('lat', spacing=lmspacing)
lmdim = (lon.shape[0] + 1, lon.shape[1] + 1)
landmask = np.ones(lmdim, dtype=bool)
landmask[:-1, :-1] = get_landmask(lon, lat, landmaskpath)
lmfac = lmspacing / spacing
landmask = np.repeat(landmask, lmfac[0], axis=0)
landmask = np.repeat(landmask, lmfac[1], axis=1)
seaindex = np.where(landmask[noborder] == False)
if seaindex[0].size >= ssr.size * 0.01:
values = values[seaindex]
if vmin == None:
vmin = scoreatpercentile(values, 0.1)
if vmax == None:
vmax = scoreatpercentile(values, 99.9)
elif contrast == 'sea':
if sensor_polarisation in ['HH', 'VV']:
if vmin == None:
vmin = 0.
if vmax == None:
vmax = 2.
else:
if vmin == None:
vmin = 1.
if vmax == None:
vmax = 3.
elif contrast == 'ice':
if sensor_polarisation in ['HH', 'VV']:
if vmin == None:
vmin = 0.
if vmax == None:
vmax = 3.5
else:
if vmin == None:
vmin = 1.
if vmax == None:
vmax = 5.
else:
raise Exception('Unknown contrast name.')
print '--> vmin=%f vmax=%f' % (vmin, vmax)
ssr = ssr[::-1, :] # keep SAR orientation for geotiff
geoloc = sarim.get_info('geolocation_grid')
gcplin = (dim[0] * spacing[0] - 1 - geoloc['line'] + 0.5) / spacing[0]
gcppix = (geoloc['pixel'] + 0.5) / spacing[1]
gcplon = geoloc['longitude']
gcplat = geoloc['latitude']
gcphei = geoloc['height']
if gcp2height is not None:
geod = pyproj.Geod(ellps='WGS84')
gcpforw, gcpback, _ = geod.inv(gcplon[:, :-1], gcplat[:, :-1],
gcplon[:, 1:], gcplat[:, 1:])
gcpforw = np.hstack((gcpforw, gcpforw[:, [-1]]))
gcpback = np.hstack((gcpback[:, [0]], gcpback))
gcpinc = geoloc['incidence_angle']
mvdist = (gcp2height - gcphei) / np.tan(np.deg2rad(gcpinc))
mvforw = gcpforw
indneg = np.where(mvdist < 0)
mvdist[indneg] = -mvdist[indneg]
mvforw[indneg] = gcpback[indneg]
_gcplon, _gcplat, _ = geod.fwd(gcplon, gcplat, mvforw, mvdist)
gcplon = _gcplon
gcplat = _gcplat
gcphei.fill(gcp2height)
if gcplon.min() < -135 and gcplon.max() > 135:
gcplon[np.where(gcplon < 0)] += 360.
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
if sensor_polarisation in ['HH', 'VV']:
metadata['product_name'] = 'SAR_roughness'
else:
metadata['product_name'] = 'SAR_roughness_crosspol'
metadata['name'] = dataname
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = source_provider
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface roughness'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'SAR'
metadata['sensor_name'] = sensor_name
metadata['sensor_platform'] = sensor_platform
metadata['sensor_mode'] = sensor_mode
metadata['sensor_swath'] = sensor_swath
metadata['sensor_polarisation'] = sensor_polarisation
metadata['sensor_pass'] = sensor_pass
metadata['datagroup'] = datagroup
geolocation = {}
geolocation['projection'] = sarim._mapper._handler.GetGCPProjection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
band = []
scale = (vmax - vmin) / 254.
offset = vmin
indzero = np.where(ssr == 0)
array = np.clip(np.round((ssr - offset) / scale), 0, 254).astype('uint8')
array[indzero] = 255
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'sea surface roughness',
'unittype': '',
'nodatavalue': 255,
'parameter_range': [vmin, vmax]
})
# Write
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'sea_surface_roughness'
band[0]['long_name'] = 'sea surface roughness'
band[0]['unittype'] = '1'
metadata['spatial_resolution'] = mspacing.min()
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def modis_sst(infileid,
outdir,
download_dir='/tmp',
vmin=None,
vmax=None,
contrast='relative',
ngcps=(21, 25),
resample_radius=5000.,
resample_sigma=2500.,
denoise_kernel='boxcar',
denoise_width=20,
open_iterations=1,
nprocs=1,
pngkml=False,
write_netcdf=False,
file_range=None):
"""
"""
dic = None
# Check file containing ranges
if file_range is not None:
if not os.path.isfile(file_range):
raise Exception('file_range {} not found'.format(file_range))
# Read a txt file which contains three columns: yearday,vmin,vmax
with open(file_range, 'r') as f:
dic = {}
for line in f:
(fdoy, fmin, fmax) = line.split(',')
dic[int(fdoy)] = (float(fmin), float(fmax))
# modissstfname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/A2011338122500.L2_LAC_SST'
# modis02fname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/MYD021KM.A2011338.1225.005.2011339235825.hdf'
# modis03fname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/MYD03.A2011338.1225.005.2011339233301.hdf'
# modis35l2fname = '/mnt/data/sst/modis/MYD021KM.A2011338.1225/MYD35_L2.A2011338.1225.005.2011340001234.hdf'
if contrast == 'med':
listbox = [[-6., 35., 2.75, 42.48], [2.74, 30, 42.2, 47.00]]
elif contrast == 'cwe':
listbox = [[-23., 35.2, -5.5, 42.88], [-23., 42.8, 2.20, 51.]]
elif contrast == 'nwe':
listbox = [[-23., 50.8, 32.7, 68.]]
elif contrast == 'gom':
listbox = [[-98., 18.0, -80.5, 30.5]]
elif contrast == 'agulhas':
listbox = [[10.8437, -45.7404, 39.9799, -25.3019]]
elif contrast == 'gs':
listbox = [[-81.52, 20, -30, 45]]
else:
listbox = None
# Search/Download data
print 'Search/Download data'
if re.match(r'^[AT][0-9]{13}$', infileid) is None:
raise Exception('Input for modis_sst is an ID '
'(e.g. A2011338122500 or T2014143234500)')
platform = infileid[0]
date = datetime.strptime(infileid[1:], '%Y%j%H%M%S')
modissstid = {'A': 'MODISAL2SST', 'T': 'MODISTL2SST'}[platform]
modissstfname = modis.search_and_download(modissstid, date, download_dir)
modis02id = {'A': 'MYD021KM', 'T': 'MOD021KM'}[platform]
modis02fname = modis.search_and_download(modis02id, date, download_dir)
modis03id = {'A': 'MYD03', 'T': 'MOD03'}[platform]
modis03fname = modis.search_and_download(modis03id, date, download_dir)
modis35l2id = {'A': 'MYD35_L2', 'T': 'MOD35_L2'}[platform]
modis35l2fname = modis.search_and_download(modis35l2id, date, download_dir)
# Read/Process data
print 'Read/Process data'
# Read from SST file
modissstfile = modis.MODISL2File(modissstfname)
# lon = modissstfile.read_lon()
# lat = modissstfile.read_lat()
sst = modissstfile.read_sst() + 273.15
attrs = modissstfile.read_attributes()
modissstfile.close()
# Read from radiances file
modis02file = modis.MODIS02File(modis02fname)
rad11 = modis02file.read_radiance(31)
modis02file.close()
bt11 = modis.modis_bright(rad11, 31, 1)
# Read from geolocation file
modis03file = modis.MODIS03File(modis03fname)
lon = modis03file.read_lon()
lat = modis03file.read_lat()
modis03file.close()
# Read from cloud mask file
modis35l2file = modis.MODIS35L2File(modis35l2fname)
cloudmask = modis35l2file.read_cloudmask(byte=0)
modis35l2file.close()
cloudy = (np.bitwise_and(cloudmask, 2) == 0) & \
(np.bitwise_and(cloudmask, 4) == 0)
land = np.bitwise_and(cloudmask, 128) == 128 # Desert or Land
# land = (np.bitwise_and(cloudmask, 128) == 128) | \
# (np.bitwise_and(cloudmask, 64) == 64) # Desert or Land or Coastal
if listbox is not None:
mask_box = np.zeros(np.shape(sst))
for i in range(np.shape(listbox)[0]):
index_in = np.where((lon >= listbox[i][0]) & (lat >= listbox[i][1])
& (lon <= listbox[i][2])
& (lat <= listbox[i][3]))
mask_box[index_in] = 1
mask = cloudy | land | ma.getmaskarray(sst) | ma.getmaskarray(bt11) | (
mask_box == 0)
else:
mask = cloudy | land | ma.getmaskarray(sst) | ma.getmaskarray(bt11)
if mask.all():
print 'No data'
sys.exit(0)
# GCPs for resampling and geotiff georeference
scansize = 10
dtime0 = datetime.utcnow()
gcps = resample.get_gcps_from_bowtie(lon, lat, scansize, ngcps=ngcps)
#gcps = resample.get_gcps_from_bowtie_old(lon, lat, scansize, ngcps=ngcps)
dtime = datetime.utcnow() - dtime0
print 'Get GCPs from bowtie swath : {}'.format(dtime)
gcplon, gcplat, gcpnpixel, gcpnline = gcps
rspysize = lon.shape[0]
geod = pyproj.Geod(ellps='WGS84')
mid = abs(gcpnline[:, 0] - 0.5).argmin()
xdists = geod.inv(gcplon[mid, :-1], gcplat[mid, :-1], gcplon[mid, 1:],
gcplat[mid, 1:])[2]
xdist = np.sum(xdists) / abs(gcpnpixel[mid, -1] - gcpnpixel[mid, 0])
rspxsize = np.round(xdist / 1000.).astype('int') + 1
gcpline = gcpnline * rspysize
gcppixel = gcpnpixel * rspxsize
# Resample with LinearNDInterpolator in output space
dtime0 = datetime.utcnow()
pix, lin = resample.get_points_from_gcps(gcplon,
gcplat,
gcppixel,
gcpline,
rspxsize,
rspysize,
1,
lon,
lat,
nprocs=nprocs) - 0.5
dtime = datetime.utcnow() - dtime0
print 'Get input coordinates in new grid : {}'.format(dtime)
# Test input grid in output space
# import matplotlib.pyplot as plt
# for iscan in range(lon.shape[0] / scansize):
# pixscan = pix[iscan * scansize: (iscan+1) * scansize, :]
# linscan = lin[iscan * scansize: (iscan+1) * scansize, :]
# # maskscan = mask[iscan * scansize: (iscan+1) * scansize, :]
# # pixscan = pixscan[~maskscan]
# # linscan = linscan[~maskscan]
# plt.plot(pixscan.flatten(), linscan.flatten(), '+')
# plt.show()
# import pdb ; pdb.set_trace()
# \Test input grid in output space
dtime0 = datetime.utcnow()
sst.data[mask] = np.nan
bt11.data[mask] = np.nan
val = np.dstack((sst.data, bt11.data))
rspval = resample.resample_bowtie_linear(pix,
lin,
val,
scansize,
rspxsize,
rspysize,
show=False)
rspsst = rspval[:, :, 0]
rspbt11 = rspval[:, :, 1]
rspmask = ma.getmaskarray(rspsst) | ma.getmaskarray(rspbt11)
dtime = datetime.utcnow() - dtime0
print 'Interpolate in new grid : {}'.format(dtime)
# Resample with pyresample in lon/lat space
# rsplin, rsppix = np.mgrid[0:rspysize, 0:rspxsize] + 0.5
# rsplon, rsplat = resample.get_points_from_gcps(gcplon, gcplat, gcppixel,
# gcpline, rspxsize, rspysize,
# 0, rsppix, rsplin, nprocs=nprocs)
# # Test resample grid
# import matplotlib.pyplot as plt
# plt.plot(lon.flatten(), lat.flatten(), '+b')
# plt.plot(rsplon.flatten(), rsplat.flatten(), '+g')
# plt.plot(gcplon.flatten(), gcplat.flatten(), 'xr')
# plt.show()
# import pdb ; pdb.set_trace()
# # \Test resample grid
# # Test radius / sigma
# resample_radius = 5000.
# resample_sigma = 2500.
# sst.mask = False
# #sst.mask = sst.mask | (sst.data < 273.15+5) | (sst.data > 273.15+30)
# rspsst = resample.resample_gauss(lon, lat, sst, rsplon, rsplat,
# resample_radius, resample_sigma,
# nprocs=nprocs, show=True)
# import pdb ; pdb.set_trace()
# # \Test radius / sigma
# valid = np.where(mask == False)
# rspsst = resample.resample_gauss(lon[valid], lat[valid], sst[valid],
# rsplon, rsplat,
# resample_radius, resample_sigma,
# fill_value=None, nprocs=nprocs,
# show=False)
# rspbt11 = resample.resample_gauss(lon[valid], lat[valid], bt11[valid],
# rsplon, rsplat,
# resample_radius, resample_sigma,
# fill_value=None, nprocs=nprocs,
# show=False)
# rspmask = resample.resample_nearest(lon, lat, mask,
# rsplon, rsplat,
# resample_radius,
# fill_value=True, nprocs=nprocs,
# show=False)
# rspmask = rspmask | ma.getmaskarray(rspsst) | ma.getmaskarray(rspbt11)
# Denoise sst and open mask
rspsst.mask = rspmask
rspbt11.mask = rspmask
finalsst = denoise_sst(rspsst,
rspbt11,
kernel=denoise_kernel,
width=denoise_width,
show=False)
#finalsst = rspsst
finalmask = ~binary_opening(
~rspmask, structure=np.ones((3, 3)), iterations=open_iterations)
#finalmask = rspmask
finalsst.mask = finalmask
# Contrast
if vmin == None:
if contrast == 'relative':
vmin = np.percentile(finalsst.compressed(), 0.5)
#elif contrast == 'agulhas':
# dayofyear = float(attrs['start_time'].timetuple().tm_yday)
# vmin = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. - 9.
# #par = [277.94999694824219, 42, 2.5500030517578125, -219]
# par = [278.09999084472656, 0.62831853071795862,
# 2.4000091552734375, 0.1570796326794896]
# vmin = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific txt file is provided for the range
elif dic is not None:
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
# Read a txt file which contains three columns: yearday,vmin,vmax
extrema = dic.get(
dayofyear, dic[min(dic.keys(),
key=lambda k: abs(k - dayofyear))])
vmin = extrema[0]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
if vmax == None:
if contrast == 'relative':
vmax = np.percentile(finalsst.compressed(), 99.5)
#elif contrast == 'agulhas':
# dayofyear = float(attrs['start_time'].timetuple().tm_yday)
# vmax = 273.15 + 2. * np.cos((dayofyear - 45.) * 2. * np.pi / 365.) + 20. + 4.
# #par = [300.59999084472656, 21, 2.8499908447265625, -191]
# par = [300.59999084472656, 0.29919930034188508,
# 2.8499908447265625, 0.14959965017094254]
# vmax = par[0] + par[2] * np.cos(par[3] * dayofyear - par[1])
#if a specific text file is provided for the range
elif dic is not None:
dayofyear = float(attrs['start_time'].timetuple().tm_yday)
extrema = dic.get(
dayofyear, dic[min(dic.keys(),
key=lambda k: abs(k - dayofyear))])
vmax = extrema[1]
else:
raise Exception('Unknown contrast : {}'.format(contrast))
# Flip (geotiff in "swath sense")
finalsst = finalsst[::-1, ::-1]
gcppixel = rspxsize - gcppixel
gcpline = rspysize - gcpline
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(attrs['start_time'],
attrs['stop_time'],
units='ms')
metadata['product_name'] = 'SST_MODIS_denoised'
if contrast == 'relative':
metadata['name'] = os.path.splitext(os.path.basename(modissstfname))[0]
else:
metadata['name'] = '{}_{}'.format(
os.path.splitext(os.path.basename(modissstfname))[0], contrast)
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = [
modissstfname, modis02fname, modis03fname, modis35l2fname
]
metadata['source_provider'] = 'NASA'
metadata['processing_center'] = 'OceanDataLab'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'sea surface temperature'
metadata['type'] = 'remote sensing'
metadata['sensor_type'] = 'radiometer'
metadata['sensor_name'] = 'MODIS'
metadata['sensor_platform'] = attrs['platform']
metadata['sensor_pass'] = attrs['pass']
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
gcpheight = np.zeros(gcppixel.shape)
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcpheight,
gcppixel, gcpline)
band = []
indndv = np.where(ma.getmaskarray(finalsst) == True)
offset, scale = vmin, (vmax - vmin) / 254.
np.clip(finalsst.data, vmin, vmax, out=finalsst.data)
array = np.round((finalsst.data - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('cerbere_medspiration',
vmax=vmax,
vmax_pal=vmax,
vmin=vmin,
vmin_pal=vmin)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'sea surface temperature',
'unittype': 'K',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
# Write geotiff
if write_netcdf == False:
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)
# Write projected png/kml
if pngkml == True:
print 'Write projected png/kml'
stfmt.write_pngkml_proj(tifffile)
elif write_netcdf == True:
print 'Write netcdf'
ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True)
band[0]['name'] = 'denoised_sst'
band[0]['long_name'] = 'denoised sea surface temperature'
band[0]['standard_name'] = 'sea_surface_temperature'
# ymid = abs(gcpline[:, 0] - rspysize / 2.).argmin()
# xdists = geod.inv(gcplon[ymid, :-1], gcplat[ymid, :-1],
# gcplon[ymid, 1:], gcplat[ymid, 1:])[2] / \
# np.abs(gcppixel[ymid, 1:] - gcppixel[ymid, :-1])
# xmid = abs(gcppixel[0, :] - rspxsize / 2.).argmin()
# ydists = geod.inv(gcplon[:-1, xmid], gcplat[:-1, xmid],
# gcplon[1:, xmid], gcplat[1:, xmid])[2] / \
# np.abs(gcpline[1:, xmid] - gcpline[:-1, xmid])
# print xdists.min(), xdists.max(), xdists.mean()
# # e.g. 999.763079208 999.763084628 999.763082543
# print ydists.min(), ydists.max(), ydists.mean()
# # e.g. 1006.4149472 1008.60679776 1007.5888004
metadata['spatial_resolution'] = 1000.
stfmt.write_netcdf(ncfile,
metadata,
geolocation,
band,
'swath',
ngcps=gcplon.shape)
def smosstorm_amsr2_wind(infile,
outdir,
valid_max_lat=85.,
vmin=0.,
vmax=50.8,
vmin_pal=0.,
vmax_pal=50.):
"""
"""
# Read/Process data
## Get wind/lon/lat/time
dataset = Dataset(infile)
wind = dataset.variables['wind_speed_lf'][:, :]
lat = dataset.variables['latitude'][:]
if np.ma.is_masked(lat):
raise Exception('Unexpected masked lat')
### For lon: _FillValue = -9999 with scale_factor = 0.01 makes -99.99 to be masked !
### -> we assume -99.99 is actually a correct longitude
### -> auto masking is disabled and we just check that lon are in [-180, 180]
dataset.variables['longitude'].set_auto_mask(False)
lon = dataset.variables['longitude'][:]
if lon.min() < -180. or lon.max() > 180.:
raise Exception('Unexpected lon outside [-180., 180.]')
time = dataset.variables['time'][:]
time_units = dataset.variables['time'].units
## Transpose: (col,row) -> (row,col)
wind = wind.transpose()
lat = lat.transpose()
lon = lon.transpose()
## Keep only the minimum row slice with valid data
## (ie we don't keep first/last invalid data in along-track dimension)
valid = ~np.ma.getmaskarray(wind)
if valid_max_lat is not None:
# We don't want to take into account points where lat > valid_max_lat
valid[np.abs(lat) > valid_max_lat] = False
row_valid = np.where(np.any(valid, axis=1))
if row_valid[0].size == 0:
raise Exception('Data is fully masked.')
row_valid_min, row_valid_max = row_valid[0].min(), row_valid[0].max()
if row_valid_min == row_valid_max: # For having at least 2 GCPs in row dim
if row_valid_min == 0:
row_valid_max += 1
else:
row_valid_min -= 1
row_slice = slice(row_valid_min, row_valid_max + 1)
#row_slice = slice(None) # for test
rowcol_slice = [row_slice, slice(None)]
wind = wind[rowcol_slice]
lat = lat[rowcol_slice]
lon = lon[rowcol_slice]
time = time[row_slice]
## Make lon continuous
nrow, ncol = lon.shape
_nrow = np.ceil(nrow / 20.).astype('int')
_irow = np.linspace(0, nrow, num=_nrow,
endpoint=False).round().astype('int')
for irow in _irow:
lon0 = lon[irow, ncol / 2] - 180
lon[irow:, :] = np.mod(lon[irow:, :] - lon0, 360.) + lon0
lon = lon - np.floor((lon.min() + 180.) / 360.) * 360.
lon_range = lon.max() - lon.min()
#print lon.min(), lon.max(), lon_range
if lon_range > 310.:
raise Exception('Unexpected lon range : {}'.format(lon_range))
## Make GCPs
dgcp = 20. # spacing is about 10km
ngcps = np.ceil(np.array(lon.shape) / dgcp) + 1.
lin = np.linspace(0, lon.shape[0] - 1,
num=ngcps[0]).round().astype('int32')
pix = np.linspace(0, lon.shape[1] - 1,
num=ngcps[1]).round().astype('int32')
pix2d, lin2d = np.meshgrid(pix, lin)
pix2d = pix2d.flatten()
lin2d = lin2d.flatten()
### Commented code : we assume now that lon are correct (_FillValue is not)
# invalidgcp = np.where((ma.getmaskarray(lon[lin2d, pix2d]) == True) | \
# (ma.getmaskarray(lat[lin2d, pix2d]) == True))
# if invalidgcp[0].size != 0:
# validlonlat = np.where((ma.getmaskarray(lon) == False) & \
# (ma.getmaskarray(lat) == False))
# for x in invalidgcp[0]:
# dists = np.sqrt((lin2d[x] - validlonlat[0]) ** 2 + \
# (pix2d[x] - validlonlat[1]) ** 2)
# argmin = dists.argmin()
# if dists[argmin] <= dgcp / 2.:
# lin2d[x] = validlonlat[0][argmin]
# pix2d[x] = validlonlat[1][argmin]
# else:
# raise Exception('Invalid lon/lat for GCPs.')
gcplon = lon[lin2d, pix2d]
gcplat = lat[lin2d, pix2d]
gcppix = pix2d + 0.5
gcplin = lin2d + 0.5
gcphei = np.zeros(gcplon.shape)
## Dataset name and time range
### Commented code : used with old smosstorm files (storm colocated)
# head, tail = os.path.split(infile)
# head, tail = os.path.split(head)
# if tail != 'data':
# raise Exception('Input file parent directories are not '
# '<storm_name>/amsr2/data !')
# head, tail = os.path.split(head)
# if tail != 'amsr2':
# raise Exception('Input file parent directories are not '
# '<storm_name>/amsr2/data !')
# head, storm_name = os.path.split(head)
# dataset_name = os.path.splitext(os.path.basename(infile))[0] + '_' + storm_name
dataset_name = os.path.splitext(os.path.basename(infile))[0]
start_time = num2date(time.min(), time_units)
stop_time = num2date(time.max(), time_units)
# Show lon/lat
# import matplotlib.pyplot as plt
# #plt.plot(lon.flatten(), lat.flatten(), '+b')
# valid = np.ma.getmaskarray(wind) == False
# plt.plot(lon[valid], lat[valid], '+b')
# plt.plot(gcplon.flatten(), gcplat.flatten(), 'ok')
# gcplon = gcplon.reshape(ngcps)
# gcplat = gcplat.reshape(ngcps)
# plt.plot(gcplon[0, :], gcplat[0, :], 'om')
# plt.plot(gcplon[-1, :], gcplat[-1, :], 'om')
# plt.plot(gcplon[:, 0], gcplat[:, 0], 'or')
# plt.plot(gcplon[:, -1], gcplat[:, -1], 'or')
# xlim = plt.xlim()
# plt.plot(xlim, [85.05] * 2, '-r')
# plt.plot(xlim, [-85.05] * 2, '-r')
# plt.show()
# import pdb ; pdb.set_trace()
# Construct metadata/geolocation/band(s)
print 'Construct metadata/geolocation/band(s)'
metadata = {}
(dtime, time_range) = stfmt.format_time_and_range(start_time,
stop_time,
units='ms')
metadata['product_name'] = 'SMOSSTORM_AMSR2_wind'
metadata['name'] = dataset_name
metadata['datetime'] = dtime
metadata['time_range'] = time_range
metadata['source_URI'] = infile
metadata['source_provider'] = 'JAXA'
metadata['processing_center'] = 'SOLAB'
metadata['conversion_software'] = 'Syntool'
metadata['conversion_version'] = '0.0.0'
metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow())
metadata['parameter'] = 'wind speed'
geolocation = {}
geolocation['projection'] = stfmt.format_gdalprojection()
geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix,
gcplin)
band = []
indndv = np.where(np.ma.getmaskarray(wind))
offset, scale = vmin, (vmax - vmin) / 254.
_wind = np.clip(np.ma.getdata(wind), vmin, vmax)
array = np.round((_wind - offset) / scale).astype('uint8')
array[indndv] = 255
colortable = stfmt.format_colortable('matplotlib_jet',
vmax=vmax,
vmax_pal=vmax_pal,
vmin=vmin,
vmin_pal=vmin_pal)
band.append({
'array': array,
'scale': scale,
'offset': offset,
'description': 'wind speed',
'unittype': 'm/s',
'nodatavalue': 255,
'parameter_range': [vmin, vmax],
'colortable': colortable
})
# Write geotiff
print 'Write geotiff'
tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True)
stfmt.write_geotiff(tifffile, metadata, geolocation, band)