def load_cip_data(cip_15_file, cip_100_file, core_file, start_time, end_time,
reference):
"""
Load CIP data into xarray.Dataset.
This function loads and combines the data from the CIP15 and CIP100
imaging probes into an 'xarray.Datset'.
Args:
cip_15_file: Path pointing to the file containing the CIP15 data.
cip_100_file: Path pointing to the file containing the CIP100 data.
core_file: Path pointing to the file containing the core data.
start_time: Start time of the time period for which to extract the data.
end_time: End time of the time period for which to extract the data.
Return:
An xarray.Dataset containing the combined CIP data.
"""
cip_15 = xr.load_dataset(cip_15_file, decode_times=False)
time = pd.Timestamp(start_time)
year = time.year
month = time.month
day = time.day
time_0 = np.datetime64(f"{year}-{month:02}-{day}T00:00:00")
time = time_0 + cip_15["TIME"].data * np.timedelta64(1, "s")
j_start = np.where(time > start_time)[0][0]
j_end = np.where(time > end_time)[0][0]
time = time[j_start:j_end]
core = xr.load_dataset(core_file, decode_times=False)
cip_15 = cip_15[{"TIME": slice(j_start, j_end)}]
core = core.interp(Time=cip_15["TIME"])
altitude = core["ALT_GIN"]
latitude = core["LAT_GIN"]
longitude = core["LON_GIN"]
bins = cip_15["BIN_EDGES"][:] / 1e6
x_15 = 0.5 * (bins[1:] + bins[:-1])
n_15 = cip_15["SPEC"][:] * 1e6
dndd_15 = n_15 / (np.diff(bins).reshape(-1, 1))
cip_100 = xr.load_dataset(cip_100_file, decode_times=False)
cip_100 = cip_100[{"TIME": slice(j_start, j_end)}]
bins = cip_100["BIN_EDGES"][:] / 1e6
x_100 = 0.5 * (bins[1:] + bins[:-1])
n_100 = cip_100["SPEC"][:] * 1e6
dndd_100 = n_100 / (np.diff(bins).reshape(-1, 1))
start_15, end_15 = 0, 64
start_100, end_100 = 9, 64
x = np.concatenate([x_15[start_15:end_15], x_100[start_100:end_100]])
y = np.concatenate([dndd_15[start_15:end_15], dndd_100[start_100:end_100]])
n = np.concatenate([n_15[start_15:end_15], n_100[start_100:end_100]])
data = {
"time": (("time", ), time),
"diameter": (("diameter", ), x),
"dndd": (
(
"time",
"diameter",
),
y.T,
),
"n": (
(
"time",
"diameter",
),
n.T,
),
"altitude": (("time", ), altitude.data),
"latitude": (("time", ), latitude.data),
"longitude": (("time", ), longitude.data),
}
if reference:
lons = reference["longitude"].data
lats = reference["latitude"].data
d = reference["d"].data
reference_swath = geometry.SwathDefinition(lons=lons, lats=lats)
cip_swath = geometry.SwathDefinition(lons=longitude, lats=latitude)
d = kd_tree.resample_nearest(reference_swath,
d,
cip_swath,
10e3,
fill_value=np.nan)
data["d"] = (("time", ), d)
return xr.Dataset(data)
def srtm15plus_reproject(dct=None, gatts=None, sea_level=0):
import os
from pyproj import Proj
import pyproj
import acolite as ac
import numpy as np
from pyresample.bilinear import NumpyBilinearResampler
from pyresample import geometry
if dct is None:
if gatts is None:
print('No dct or gatts given.')
return ()
dct = {}
dct['xrange'] = gatts['xrange']
dct['yrange'] = gatts['yrange']
dct['pixel_size'] = gatts['pixel_size']
dct['xdim'] = (dct['xrange'][1] -
dct['xrange'][0]) / dct['pixel_size'][0]
dct['ydim'] = (dct['yrange'][1] -
dct['yrange'][0]) / dct['pixel_size'][1]
dct['p'] = pyproj.Proj(gatts['proj4_string'])
file = '/Volumes/SSD/Data/Bathymetry/SRTM15_V2.3.nc'
#dct = {'xrange': xrange, 'yrange': yrange, 'p': p,
# 'pixel_size': target_pixel_size, 'xdim': nx, 'ydim': ny}
#nc_projection = ac.shared.projection_netcdf(dct, add_half_pixel=True)
xrange = dct['xrange']
yrange = dct['yrange']
nx = dct['xdim']
ny = dct['ydim']
projection = dct['p'].srs
l_ = dct['p'](xrange, yrange, inverse=True)
limit = l_[1][1], l_[0][0], l_[1][0], l_[0][1]
## set up target definition
target_definition = geometry.AreaDefinition(
'area_id', 'description', 'proj_id', projection, nx, ny,
[xrange[0], yrange[1], xrange[1], yrange[0]])
## read lat/lon
lat = ac.shared.nc_data(file, 'lat')
lon = ac.shared.nc_data(file, 'lon')
lonoff = 0.25
latoff = 0.25
sublon = np.where((lon >= limit[1] - lonoff) & (lon <= limit[3] + lonoff))
sublat = np.where((lat >= limit[0] - latoff) & (lat <= limit[2] + latoff))
sub = [
sublon[0][0], sublat[0][0], sublon[0][-1] - sublon[0][0] + 1,
sublat[0][-1] - sublat[0][0] + 1
]
yi = lat[sublat].shape[0]
xi = lon[sublon].shape[0]
lons = np.tile(lon[sublon], yi).reshape(yi, xi)
lats = np.rot90(np.tile(lat[sublat], xi).reshape(xi, yi))
## set up source definition
source_definition = geometry.SwathDefinition(lons=lons, lats=lats)
## set up resampler
resampler = NumpyBilinearResampler(source_definition, target_definition,
30e3)
zin, zatt = ac.shared.nc_data(file, 'z', attributes=True, sub=sub)
zin = np.flipud(zin)
zout = resampler.resample(zin)
if sea_level is not None:
zout[zout <= sea_level] = 0
return (zout)
def load_drop_sonde_data(path, results=None):
data = []
files = Path(PATH).glob("faam-dropsonde*.nc")
for f in files:
ds_data = xr.load_dataset(f)
valid = (-90 <= ds_data["lat"].data) * (90 >= ds_data["lat"].data)
ds_data = ds_data.loc[{"time": valid}]
if results:
lons = results["longitude"].data
lats = results["latitude"].data
retrieval_swath = geometry.SwathDefinition(lons=lons, lats=lats)
lons = ds_data["lon"].data
lats = ds_data["lat"].data
ds_swath = geometry.SwathDefinition(lons=lons, lats=lats)
ni = kd_tree.get_neighbour_info(retrieval_swath,
ds_swath,
radius_of_influence=100e3,
neighbours=1)
(valid_input_index, valid_output_index, index_array,
distance_array) = ni
n = ds_data.time.size
n_levels = results.z.size
t_r = np.zeros(n)
t_a = np.zeros(n)
h2o_r = np.zeros(n)
h2o_a = np.zeros(n)
t_z = np.zeros((n, n_levels))
t_a_z = np.zeros((n, n_levels))
h2o_z = np.zeros((n, n_levels))
h2o_a_z = np.zeros((n, n_levels))
z = np.zeros((n, n_levels))
d = np.zeros((n))
lats_r = kd_tree.get_sample_from_neighbour_info(
"nn",
(n, ),
results["latitude"].data,
valid_input_index,
valid_output_index,
index_array,
fill_value=np.nan,
)
lons_r = kd_tree.get_sample_from_neighbour_info(
"nn",
(n, ),
results["longitude"].data,
valid_input_index,
valid_output_index,
index_array,
fill_value=np.nan,
)
d = kd_tree.get_sample_from_neighbour_info(
"nn",
(n, ),
results["d"].data,
valid_input_index,
valid_output_index,
index_array,
fill_value=np.nan,
)
for i in range(n_levels):
# t_z[:, i] = kd_tree.get_sample_from_neighbour_info(
# "nn",
# (n,),
# results["temperature"][:, i].data,
# valid_input_index,
# valid_output_index,
# index_array,
# fill_value=np.nan)
# t_a_z[:, i] = kd_tree.get_sample_from_neighbour_info(
# "nn",
# (n,),
# results["temperature_a_priori"][:, i].data,
# valid_input_index,
# valid_output_index,
# index_array,
# fill_value=np.nan)
h2o_z[:, i] = kd_tree.get_sample_from_neighbour_info(
"nn",
(n, ),
results["H2O"][:, i].data,
valid_input_index,
valid_output_index,
index_array,
fill_value=np.nan,
)
h2o_a_z[:, i] = kd_tree.get_sample_from_neighbour_info(
"nn",
(n, ),
results["H2O_a_priori"][:, i].data,
valid_input_index,
valid_output_index,
index_array,
fill_value=np.nan,
)
z[:, i] = kd_tree.get_sample_from_neighbour_info(
"nn",
(n, ),
results["altitude"][:, i].data,
valid_input_index,
valid_output_index,
index_array,
fill_value=np.nan,
)
for i in range(n):
if np.isnan(ds_data["alt"][i]):
t_r[i] = np.nan
t_a[i] = np.nan
h2o_r[i] = np.nan
h2o_a[i] = np.nan
continue
t_r[i] = np.interp(ds_data["alt"][i], z[i, :], t_z[i, :])
t_a[i] = np.interp(ds_data["alt"][i], z[i, :], t_a_z[i, :])
h2o_r[i] = np.interp(ds_data["alt"][i], z[i, :], h2o_z[i, :])
h2o_a[i] = np.interp(ds_data["alt"][i], z[i, :], h2o_a_z[i, :])
ds_data["t_retrieved"] = (("time", ), t_r)
ds_data["t_a_priori"] = (("time", ), t_a)
ds_data["h2o_retrieved"] = (("time", ), h2o_r)
ds_data["h2o_a_priori"] = (("time", ), h2o_a)
ds_data["lons_r"] = (("time"), lons_r)
ds_data["lats_r"] = (("time"), lats_r)
ds_data["d"] = (("time"), d)
data.append(ds_data)
return data
#: Create pyresample area_def object for resampling
area_def = prg.AreaDefinition(
'orthographic',
'orthographic',
'orthographic',
projection=crs.proj4_params,
width=lons.shape[1],
height=lons.shape[0],
area_extent=(crs.x_limits[0] * zoom_val,
crs.y_limits[0] * zoom_val,
crs.x_limits[1] * zoom_val,
crs.y_limits[1] * zoom_val))
# Remap to crs projection
swath_def = prg.SwathDefinition(lons=lons, lats=lats)
result3 = resample_nearest(swath_def,
data3,
area_def,
radius_of_influence=lons.shape[0] *
lons.shape[1] * 2.5,
fill_value=None)
result2 = resample_nearest(swath_def,
data2,
area_def,
radius_of_influence=lons.shape[0] *
lons.shape[1] * 2.5,
fill_value=None)
resultd = resample_nearest(swath_def,
datad,
area_def,
def load_dataset(self, satscene, filename=None, *args, **kwargs):
"""Read data from file and load it into *satscene*.
"""
del args
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
options = dict(conf.items(satscene.instrument_name + "-level2",
raw=True))
options["resolution"] = 1000
options["geofile"] = os.path.join(options["dir"], options["geofile"])
options.update(kwargs)
fparser = Parser(options.get("filename"))
gparser = Parser(options.get("geofile"))
if isinstance(filename, (list, set, tuple)):
# we got the entire dataset.
for fname in filename:
if fnmatch(os.path.basename(fname), fparser.globify()):
metadata = fparser.parse(os.path.basename(fname))
resolution = self.res[metadata["resolution"]]
self.datafiles[resolution] = fname
elif fnmatch(os.path.basename(fname), gparser.globify()):
self.geofile = fname
elif ((filename is not None) and
fnmatch(os.path.basename(options["filename"]), fparser.globify())):
# read just one file
logger.debug("Reading from file: " + str(options["filename"]))
filename = options["filename"]
resolution = self.res[os.path.basename(filename)[5]]
self.datafiles[resolution] = filename
if not self.datafiles:
# find files according to config
logger.debug(
"Didn't get any valid file as input, looking in defined places")
resolution = int(options["resolution"]) or 1000
for res in [250, 500, 1000]:
datafile = globify(os.path.join(options['dir'],
options["filename"]),
{'resolution': self.inv_res[res],
'start_time': satscene.time_slot})
try:
self.datafiles[res] = check_filename(datafile)
except IOError:
self.datafiles[res] = None
logger.warning("Can't find file for resolution %s with template: %s",
str(res), datafile)
try:
self.geofile = check_filename(globify(options["geofile"],
{'start_time': satscene.time_slot}))
except IOError:
self.geofile = None
logger.warning("Can't find geofile with template: %s",
options['geofile'])
resolution = options["resolution"]
cores = options.get("cores", max(multiprocessing.cpu_count() / 4, 1))
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
loaded_bands = []
# process by dataset, reflective and emissive datasets separately
resolutions = [250, 500, 1000]
for res in resolutions:
if res < resolution:
continue
logger.debug("Working on resolution %d", res)
self.filename = self.datafiles[res]
logger.debug("Using " + str(cores) + " cores for interpolation")
try:
self.data = SD(str(self.filename))
except HDF4Error as err:
logger.warning("Could not load data from " + str(self.filename)
+ ": " + str(err))
continue
datasets = datadict[res]
for dataset in datasets:
subdata = self.data.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(satscene.channels_to_load & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [i for i, band in enumerate(band_names)
if band in satscene.channels_to_load]
uncertainty = self.data.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(
subdata, uncertainty, indices, band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
if band_names[idx] in loaded_bands:
continue
satscene[band_names[idx]] = array[i]
# fix the resolution to match the loaded data.
satscene[band_names[idx]].resolution = res
loaded_bands.append(band_names[idx])
# Get the orbit number
if not satscene.orbit:
mda = self.data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = int(mda[orbit_idx + 111:orbit_idx + 116])
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
for band_name in loaded_bands:
lon, lat = self.get_lonlat(
satscene[band_name].resolution, satscene.time_slot, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
for band_name in loaded_bands:
band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest()
satscene[band_name].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+
str(satscene[
band_name].shape) + "_"
+ str(band_uid))
satscene[band_name].area_id = satscene[band_name].area.area_id
def modisl1b_resample(mxd02file, mxd03file, chan_list, fill_value=-99999.):
"""
given level1b 1km radiance file and geometry file
return projected images plus supporting data.
The resampling uses pyresample with a lambert azimuthal
equal area projection with lat_0 and lon_0 at
the scene center
Parameters
----------
mxd02file: str
file name NASA Laadsweb level1b Terra (MOD02) or Aqua (MYD02) hdf file converted to h5
format
mxd03file: str
file name NASA Laadsweb level geometry file for Terra (MOD03) or Aqua (MYD03) hdf file converted to h5 format
chan_list: vecto of strings
channels to resample onto lambert azimuthal projections, i.e. ['1','4','3']
fill_value: float
number to indicate missing grid data. Make sure this number is not in the range of actual
data values
Returns
-------
dictionary with keys/values:
channels: float32 3-dim array
projected for each channel radiances (W/m^2/sr/micron for EV channels) or reflectances (no units for RefSB channels)
dimensions: [height,width,numchans]
area_def_args: dict
dictionary with projection information for pyresample
basemap_args: dict
dictionary with projection information for basemap
geotiff_args: dict
dictionary with projection information for geotiff
fill_value: float32
fill value for missing data in channels (these resampled pixels are set to
np.nan after resample)
"""
radiance_list = []
for the_chan in chan_list:
#
# read channel channels
#
index = chan_dict[the_chan]['index']
field_name = chan_dict[the_chan]['field_name']
scale_name = chan_dict[the_chan]['scale']
offset_name = chan_dict[the_chan]['offset']
with h5py.File(mxd02file, 'r') as h5_file:
chan = h5_file['MODIS_SWATH_Type_L1B']['Data Fields'][field_name][
index, :, :]
hdf_fill_value = h5_file['MODIS_SWATH_Type_L1B']['Data Fields'][
field_name].attrs['_FillValue']
chan = chan.astype(np.float32)
#
# problem with modis fill_value for channel 29 -- says it should be 65535 but it is actually 65531
# accept anything larger than 60000
#
if hdf_fill_value == 65535:
hit = chan > 65530
else:
hit = chan == hdf_fill_value
chan[hit] = np.nan
scale = h5_file['MODIS_SWATH_Type_L1B']['Data Fields'][
field_name].attrs[scale_name][...]
offset = h5_file['MODIS_SWATH_Type_L1B']['Data Fields'][
field_name].attrs[offset_name][...]
chan_calibrated = (chan - offset[index]) * scale[index]
chan_calibrated = chan_calibrated.astype(
np.float32) #convert from 64 bit to 32bit to save space
print('in e582lib.modis_resample: found {} bad pixels'.format(
np.sum(np.isnan(chan_calibrated))))
chan_calibrated = np.ma.masked_invalid(chan_calibrated)
radiance_list.append(chan_calibrated)
for index, chan in enumerate(radiance_list):
print('index and mean {} {}'.format(index, np.mean(chan.ravel())))
with h5py.File(mxd03file) as geo_file:
lon_data = geo_file['MODIS_Swath_Type_GEO']['Geolocation Fields'][
'Longitude'][...]
lat_data = geo_file['MODIS_Swath_Type_GEO']['Geolocation Fields'][
'Latitude'][...]
#
# set up the lambert azimuthal equal area projection with corners large enough
# to fit image
#
corners = parseMeta(mxd02file)
proj_id = 'laea'
datum = 'WGS84'
lat_0_txt = '{lat_0:5.2f}'.format_map(corners)
lon_0_txt = '{lon_0:5.2f}'.format_map(corners)
area_dict = dict(datum=datum,
lat_0=lat_0_txt,
lon_0=lon_0_txt,
proj=proj_id,
units='m')
prj = pyproj.Proj(area_dict)
x, y = prj(corners['lon_list'], corners['lat_list'])
minx, maxx = np.min(x), np.max(x)
miny, maxy = np.min(y), np.max(y)
#
# back transform these to lon/lat
#
llcrnrlon, llcrnrlat = prj(minx, miny, inverse=True)
urcrnrlon, urcrnrlat = prj(maxx, maxy, inverse=True)
#
# 1300 m pixels are a reasonable compromise between the smallest 1km pixels and the biggest
# 4 km pixels at the sides of the swath
#
area_extent = [minx, miny, maxx, maxy]
x_pixel = 1.3e3
y_pixel = 1.3e3
#
# figure out how many pixels in the image
#
xsize = int((area_extent[2] - area_extent[0]) / x_pixel)
ysize = int((area_extent[3] - area_extent[1]) / y_pixel)
#
# here's the dictionary we need for basemap
#
basemap_args = dict()
basemap_args['ellps'] = 'WGS84'
basemap_args['llcrnrlon'] = llcrnrlon
basemap_args['llcrnrlat'] = llcrnrlat
basemap_args['urcrnrlon'] = urcrnrlon
basemap_args['urcrnrlat'] = urcrnrlat
basemap_args['projection'] = area_dict['proj']
basemap_args['lat_0'] = corners['lat_0']
basemap_args['lon_0'] = corners['lon_0']
#
#
# here is the dictionary for pyresample
#
area_id = 'granule'
area_name = 'modis swath 5min granule'
#
# here are all the arguments pyresample needs to regrid the swath
#
#
# first copy each of the reprojected channels into input_array,
# which is dimensioned [height,width,num_chans]
#
# pyresample will resample all num_chans channels onto the same grid
#
num_chans = len(chan_list)
dim_list = list(radiance_list[0].shape)
dim_list.append(num_chans)
input_array = np.empty(dim_list, dtype=np.float32)
for index, chan in enumerate(radiance_list):
input_array[:, :, index] = chan[:, :]
#
# now project all the images onto the lambert map
#
area_def_args = dict(area_id=area_id,
area_name=area_name,
proj_id=proj_id,
area_dict=area_dict,
xsize=xsize,
ysize=ysize,
area_extent=area_extent)
area_def = geometry.AreaDefinition(area_id, area_name, proj_id, area_dict,
xsize, ysize, area_extent)
swath_def = geometry.SwathDefinition(lons=lon_data, lats=lat_data)
#
# here is the resample step using 5 km region of influence (see pyresample docs)
#
channels = kd_tree.resample_nearest(swath_def,
input_array,
area_def,
radius_of_influence=25000,
nprocs=2,
fill_value=None)
# channels = kd_tree.resample_gauss(swath_def, input_array,
# area_def, radius_of_influence=50000,sigmas=[25000,25000],nprocs=2,fill_value=None)
#
# replace the number used for fill_value with np.nan
#
# nan_fill_value = np.array([np.nan],dtype=np.float32)[0]
# channels[channels==fill_value]=nan_fill_value
print('running modisl1b_resample: here are the channels to be resampled')
for index in range(num_chans):
print('channel and mean {} {}'.format(
chan_list[index], np.nanmean(channels[:, :, index].ravel())))
#
# replace negative fill_value with np.nan (32 bit)
#
print('pyresample area_def information:')
print('\ndump area definition:\n{}\n'.format(area_def))
print('\nx and y pixel dimensions in meters:\n{}\n{}\n'.format(
area_def.pixel_size_x, area_def.pixel_size_y))
#
# here is the dictionary for geotiff creation -- save for future use in rasterio
#
adfgeotransform = [
area_def.area_extent[0], area_def.pixel_size_x, 0,
area_def.area_extent[3], 0, -area_def.pixel_size_y
]
proj4_string = area_def.proj4_string
proj_id = area_def.proj_id
height, width, num_chans = channels.shape
geotiff_args = dict(width=width,
height=height,
adfgeotransform=adfgeotransform,
proj4_string=proj4_string,
proj_id=proj_id)
out_dict = dict(channels=channels,
area_def_args=area_def_args,
basemap_args=basemap_args,
geotiff_args=geotiff_args,
fill_value=fill_value)
print('completed modisl1b_resample')
return out_dict
class Test(unittest.TestCase):
pts_irregular = (np.array([[-1., 1.], ]),
np.array([[1., 2.], ]),
np.array([[-2., -1.], ]),
np.array([[2., -4.], ]))
pts_vert_parallel = (np.array([[-1., 1.], ]),
np.array([[1., 2.], ]),
np.array([[-1., -1.], ]),
np.array([[1., -2.], ]))
pts_both_parallel = (np.array([[-1., 1.], ]),
np.array([[1., 1.], ]),
np.array([[-1., -1.], ]),
np.array([[1., -1.], ]))
# Area definition with four pixels
target_def = geometry.AreaDefinition('areaD',
'Europe (3km, HRV, VTC)',
'areaD',
{'a': '6378144.0',
'b': '6356759.0',
'lat_0': '50.00',
'lat_ts': '50.00',
'lon_0': '8.00',
'proj': 'stere'},
4, 4,
[-1370912.72,
-909968.64000000001,
1029087.28,
1490031.3600000001])
# Input data around the target pixel at 0.63388324, 55.08234642,
in_shape = (100, 100)
data1 = np.ones((in_shape[0], in_shape[1]))
data2 = 2. * data1
lons, lats = np.meshgrid(np.linspace(-5., 5., num=in_shape[0]),
np.linspace(50., 60., num=in_shape[1]))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
radius = 50e3
neighbours = 32
input_idxs, output_idxs, idx_ref, dists = \
kd_tree.get_neighbour_info(swath_def, target_def,
radius, neighbours=neighbours,
nprocs=1)
input_size = input_idxs.sum()
index_mask = (idx_ref == input_size)
idx_ref = np.where(index_mask, 0, idx_ref)
def test_calc_abc(self):
# No np.nan inputs
pt_1, pt_2, pt_3, pt_4 = self.pts_irregular
res = bil._calc_abc(pt_1, pt_2, pt_3, pt_4, 0.0, 0.0)
self.assertFalse(np.isnan(res[0]))
self.assertFalse(np.isnan(res[1]))
self.assertFalse(np.isnan(res[2]))
# np.nan input -> np.nan output
res = bil._calc_abc(np.array([[np.nan, np.nan]]),
pt_2, pt_3, pt_4, 0.0, 0.0)
self.assertTrue(np.isnan(res[0]))
self.assertTrue(np.isnan(res[1]))
self.assertTrue(np.isnan(res[2]))
def test_get_ts_irregular(self):
res = bil._get_ts_irregular(self.pts_irregular[0],
self.pts_irregular[1],
self.pts_irregular[2],
self.pts_irregular[3],
0., 0.)
self.assertEqual(res[0], 0.375)
self.assertEqual(res[1], 0.5)
res = bil._get_ts_irregular(self.pts_vert_parallel[0],
self.pts_vert_parallel[1],
self.pts_vert_parallel[2],
self.pts_vert_parallel[3],
0., 0.)
self.assertTrue(np.isnan(res[0]))
self.assertTrue(np.isnan(res[1]))
def test_get_ts_uprights_parallel(self):
res = bil._get_ts_uprights_parallel(self.pts_vert_parallel[0],
self.pts_vert_parallel[1],
self.pts_vert_parallel[2],
self.pts_vert_parallel[3],
0., 0.)
self.assertEqual(res[0], 0.5)
self.assertEqual(res[1], 0.5)
def test_get_ts_parallellogram(self):
res = bil._get_ts_parallellogram(self.pts_both_parallel[0],
self.pts_both_parallel[1],
self.pts_both_parallel[2],
0., 0.)
self.assertEqual(res[0], 0.5)
self.assertEqual(res[1], 0.5)
def test_get_ts(self):
out_x = np.array([[0.]])
out_y = np.array([[0.]])
res = bil._get_ts(self.pts_irregular[0],
self.pts_irregular[1],
self.pts_irregular[2],
self.pts_irregular[3],
out_x, out_y)
self.assertEqual(res[0], 0.375)
self.assertEqual(res[1], 0.5)
res = bil._get_ts(self.pts_both_parallel[0],
self.pts_both_parallel[1],
self.pts_both_parallel[2],
self.pts_both_parallel[3],
out_x, out_y)
self.assertEqual(res[0], 0.5)
self.assertEqual(res[1], 0.5)
res = bil._get_ts(self.pts_vert_parallel[0],
self.pts_vert_parallel[1],
self.pts_vert_parallel[2],
self.pts_vert_parallel[3],
out_x, out_y)
self.assertEqual(res[0], 0.5)
self.assertEqual(res[1], 0.5)
def test_solve_quadratic(self):
res = bil._solve_quadratic(1, 0, 0)
self.assertEqual(res[0], 0.0)
res = bil._solve_quadratic(1, 2, 1)
self.assertTrue(np.isnan(res[0]))
res = bil._solve_quadratic(1, 2, 1, min_val=-2.)
self.assertEqual(res[0], -1.0)
# Test that small adjustments work
pt_1, pt_2, pt_3, pt_4 = self.pts_vert_parallel
pt_1 = self.pts_vert_parallel[0].copy()
pt_1[0][0] += 1e-7
res = bil._calc_abc(pt_1, pt_2, pt_3, pt_4, 0.0, 0.0)
res = bil._solve_quadratic(res[0], res[1], res[2])
self.assertAlmostEqual(res[0], 0.5, 5)
res = bil._calc_abc(pt_1, pt_3, pt_2, pt_4, 0.0, 0.0)
res = bil._solve_quadratic(res[0], res[1], res[2])
self.assertAlmostEqual(res[0], 0.5, 5)
def test_get_output_xy(self):
proj = Proj(self.target_def.proj4_string)
out_x, out_y = bil._get_output_xy(self.target_def, proj)
self.assertTrue(out_x.all())
self.assertTrue(out_y.all())
def test_get_input_xy(self):
proj = Proj(self.target_def.proj4_string)
in_x, in_y = bil._get_output_xy(self.swath_def, proj)
self.assertTrue(in_x.all())
self.assertTrue(in_y.all())
def test_get_bounding_corners(self):
proj = Proj(self.target_def.proj4_string)
out_x, out_y = bil._get_output_xy(self.target_def, proj)
in_x, in_y = bil._get_input_xy(self.swath_def, proj,
self.input_idxs, self.idx_ref)
res = bil._get_bounding_corners(in_x, in_y, out_x, out_y,
self.neighbours, self.idx_ref)
for i in range(len(res) - 1):
pt_ = res[i]
for j in range(2):
# Only the sixth output location has four valid corners
self.assertTrue(np.isfinite(pt_[5, j]))
def test_get_bil_info(self):
t__, s__, input_idxs, idx_arr = bil.get_bil_info(self.swath_def,
self.target_def)
# Only 6th index should have valid values
for i in range(len(t__)):
if i == 5:
self.assertAlmostEqual(t__[i], 0.684850870155, 5)
self.assertAlmostEqual(s__[i], 0.775433912393, 5)
else:
self.assertTrue(np.isnan(t__[i]))
self.assertTrue(np.isnan(s__[i]))
def test_get_sample_from_bil_info(self):
t__, s__, input_idxs, idx_arr = bil.get_bil_info(self.swath_def,
self.target_def)
# Sample from data1
res = bil.get_sample_from_bil_info(self.data1.ravel(), t__, s__,
input_idxs, idx_arr)
self.assertEqual(res[5], 1.)
# Sample from data2
res = bil.get_sample_from_bil_info(self.data2.ravel(), t__, s__,
input_idxs, idx_arr)
self.assertEqual(res[5], 2.)
# Reshaping
res = bil.get_sample_from_bil_info(self.data2.ravel(), t__, s__,
input_idxs, idx_arr,
output_shape=self.target_def.shape)
res = res.shape
self.assertEqual(res[0], self.target_def.shape[0])
self.assertEqual(res[1], self.target_def.shape[1])
def test_resample_bilinear(self):
# Single array
res = bil.resample_bilinear(self.data1,
self.swath_def,
self.target_def)
self.assertEqual(res.size, self.target_def.size)
# There should be only one pixel with value 1, all others are 0
self.assertEqual(res.sum(), 1)
# Single array with masked output
res = bil.resample_bilinear(self.data1,
self.swath_def,
self.target_def, fill_value=None)
self.assertTrue(hasattr(res, 'mask'))
# There should be only one valid pixel
self.assertEqual(self.target_def.size - res.mask.sum(), 1)
# Two stacked arrays
data = np.dstack((self.data1, self.data2))
res = bil.resample_bilinear(data,
self.swath_def,
self.target_def)
shp = res.shape
self.assertEqual(shp[0], self.target_def.size)
self.assertEqual(shp[1], 2)
def project_acolite_netcdf(ncf, output=None, settings={}, target_file=None):
import os
from pyproj import Proj
import acolite as ac
import numpy as np
from pyresample.bilinear import NumpyBilinearResampler
from pyresample import image, geometry
## read gatts
try:
gatts = ac.shared.nc_gatts(ncf)
except:
print('Error accessing {}, is this a NetCDF file?'.format(ncf))
return ()
if ('sensor' not in gatts):
print('No sensor attribute in file {}'.format(ncf))
return ()
## read datasets
datasets = ac.shared.nc_datasets(ncf)
if ('lat' not in datasets) or ('lon' not in datasets):
print('No lat/lon found in file {}'.format(ncf))
return ()
## parse settings
setu = ac.acolite.settings.parse(gatts['sensor'], settings=settings)
if (setu['output_projection_limit'] is None) & (setu['limit'] is not None):
setu['output_projection_limit'] = [l for l in setu['limit']]
if setu['output_projection_resolution'] is not None:
if len(setu['output_projection_resolution']) != 2:
print('Provide a two element target_pixel_size.')
return ()
else:
target_pixel_size = [
float(v) for v in setu['output_projection_resolution']
]
else:
if setu['default_projection_resolution'] is not None:
target_pixel_size = [
float(v) for v in setu['default_projection_resolution']
]
print('Using default grid size: {}x{}metres'.format(
target_pixel_size[0], target_pixel_size[1]))
if setu['output_projection_limit'] is not None:
if len(setu['output_projection_limit']) != 4:
print('Provide a four element output_projection_limit.')
return ()
else:
limit = [float(v) for v in setu['output_projection_limit']]
if (setu['output_projection_epsg'] is None) & \
(setu['output_projection_proj4'] is None):
lon = (limit[1] + limit[3]) / 2
lat = (limit[0] + limit[2]) / 2
utm_zone, epsg = ac.shared.utm_epsg(lon, lat)
print(utm_zone, epsg)
setu['output_projection_epsg'] = epsg
else:
if not setu['output_projection_metres']:
print('Provide a four element output_projection_limit.')
return ()
## projection
if setu['output_projection_epsg'] is not None:
#projection = '+init=EPSG:{}'.format(setu['output_projection_epsg'])
if 'EPSG' not in setu['output_projection_epsg']:
projection = 'EPSG:{}'.format(setu['output_projection_epsg'])
else:
projection = '{}'.format(setu['output_projection_epsg'])
elif setu['output_projection_proj4'] is not None:
projection = setu['output_projection_proj4']
else:
print('No EPSG or proj4 string provided.')
return ()
## user provided x and yrange
if setu['output_projection_metres']:
xrange_region = setu['output_projection_xrange']
yrange_region = setu['output_projection_yrange']
if (xrange_region is None) or (yrange_region is None):
print(
'Provide a output_projection_xrange and output_projection_yrange.'
)
return ()
if len(xrange_region) != 2:
print('Provide a two element output_projection_xrange.')
return ()
if len(yrange_region) != 2:
print('Provide a two element output_projection_yrange.')
return ()
## create output file name
bn = os.path.basename(ncf)
bd = os.path.dirname(ncf)
oname, nc = os.path.splitext(bn)
if output is not None:
bd = '{}'.format(output)
elif setu['output'] is not None:
bd = '{}'.format(setu['output'])
## add requested name or "reprojected"
oname = '{}_{}'.format(
oname, setu['output_projection_name']
if setu['output_projection_name'] is not None else "projected")
ncfo = '{}/{}{}'.format(bd, oname, nc)
print('Setting up target projection.')
p = Proj(projection)
## find region extent
if not setu['output_projection_metres']:
## project lat lon to metres
xrange_raw, yrange_raw = p((limit[1], limit[1], limit[3], limit[3]),
(limit[0], limit[2], limit[2], limit[0]))
xrange_raw = (min(xrange_raw), max(xrange_raw))
yrange_raw = (min(yrange_raw), max(yrange_raw))
xrange_region = [
xrange_raw[0] - (xrange_raw[0] % target_pixel_size[0] * 2),
xrange_raw[1] + target_pixel_size[0] * 2 -
(xrange_raw[1] % target_pixel_size[0] * 2)
]
yrange_region = [
yrange_raw[1] + target_pixel_size[1] * 2 -
(yrange_raw[1] % target_pixel_size[1] * 2),
yrange_raw[0] - (yrange_raw[0] % target_pixel_size[1] * 2)
]
## align grid to pixel size
if setu['output_projection_resolution_align']:
x_grid_off = xrange_region[0] % target_pixel_size[0], xrange_region[
1] % target_pixel_size[0]
y_grid_off = yrange_region[0] % target_pixel_size[1], yrange_region[
1] % target_pixel_size[1]
xrange = (xrange_region[0] -
x_grid_off[0]), (xrange_region[1] +
(target_pixel_size[0] - x_grid_off[1]))
yrange = (yrange_region[0] -
y_grid_off[0]), (yrange_region[1] +
(target_pixel_size[0] - y_grid_off[1]))
else:
xrange = [xrange_region[0], xrange_region[1]]
yrange = [yrange_region[0], yrange_region[1]]
## pixel sizes
ny = int((yrange[0] - yrange[1]) / target_pixel_size[1])
nx = int((xrange[1] - xrange[0]) / target_pixel_size[0])
print(xrange, yrange)
print(nx, ny)
## set up projection dict and nc_projection
dct = {
'xrange': xrange,
'yrange': yrange,
'p': p,
'pixel_size': target_pixel_size,
'xdim': nx,
'ydim': ny
}
nc_projection = ac.shared.projection_netcdf(dct, add_half_pixel=True)
## set up target definition
target_definition = geometry.AreaDefinition(
'area_id', 'description', 'proj_id', projection, nx, ny,
[xrange[0], yrange[1], xrange[1], yrange[0]])
## read lat/lon
lat = ac.shared.nc_data(ncf, 'lat')
lon = ac.shared.nc_data(ncf, 'lon')
## set up source definition
source_definition = geometry.SwathDefinition(lons=lon, lats=lat)
## set up resampler
if setu['output_projection_resampling_method'] == 'bilinear':
#resampler = NumpyBilinearResampler(source_definition, target_definition, 30e5, neighbours=81)
resampler = NumpyBilinearResampler(source_definition,
target_definition, 30e3)
## make new output attributes
gatts_out = {k: gatts[k] for k in gatts}
for k in dct:
gatts_out[k] = dct[k]
gatts_out['projection_key'] = [
k for k in nc_projection if k not in ['x', 'y']
][0]
## update oname in gatts
gatts_out['oname'] = oname
## run through datasets
new = True
for ds in datasets:
data_in, att = ac.shared.nc_data(ncf, ds, attributes=True)
if len(data_in.shape) != 2: continue
print('Reprojecting {} to {} {}x{}'.format(ds, projection, nx, ny))
data_out = resampler.resample(data_in)
data_in = None
if setu['output_projection_fillnans']:
data_out[data_out == 0] = np.nan
data_out = ac.shared.fillnan(data_out)
lsd = None
if ds not in ['lat', 'lon', 'vza', 'sza', 'vaa', 'saa', 'raa']:
lsd = setu['netcdf_compression_least_significant_digit']
ac.output.nc_write(
ncfo,
ds,
data_out,
attributes=gatts_out,
netcdf_compression=setu['netcdf_compression'],
netcdf_compression_level=setu['netcdf_compression_level'],
netcdf_compression_least_significant_digit=lsd,
nc_projection=nc_projection,
dataset_attributes=att,
new=new)
data_out = None
new = False
print('Wrote {}'.format(ncfo))
return (ncfo)
def load_avhrr(satscene, options):
"""Read avhrr data from file and load it into *satscene*.
"""
if "filename" not in options:
raise IOError("No filename given, cannot load.")
chns = satscene.channels_to_load & set(["1", "2", "3A", "3B", "4", "5"])
if len(chns) == 0:
return
values = {
"orbit": satscene.orbit,
"satname": satscene.satname,
"number": satscene.number,
"instrument": satscene.instrument_name,
"satellite": satscene.fullname
}
filename = os.path.join(
satscene.time_slot.strftime(options["dir"]) % values,
satscene.time_slot.strftime(options["filename"]) % values)
file_list = glob.glob(filename)
if len(file_list) > 1:
raise IOError("More than one l1b file matching!")
elif len(file_list) == 0:
raise IOError("No l1b file matching!: " + filename)
filename = file_list[0]
LOG.debug("Loading from " + filename)
import avhrr # AHAMAP module
avh = avhrr.avhrr(filename)
avh.get_unprojected()
instrument_data = avh.build_raw()
available_channels = set([])
data_channels = {}
for chn in instrument_data.data:
channel_name = chn.info.info["channel_id"][3:].upper()
available_channels |= set([channel_name])
data_channels[channel_name] = chn.data
for chn in satscene.channels_to_load:
if chn in available_channels:
if chn in ["1", "2", "3A"]:
gain = instrument_data.info["vis_gain"]
intercept = instrument_data.info["vis_intercept"]
units = "%"
else:
gain = instrument_data.info["ir_gain"]
intercept = instrument_data.info["ir_intercept"]
units = "K"
chn_array = np.ma.array(data_channels[chn])
missing_data = instrument_data.info["missing_data"]
chn_array = np.ma.masked_inside(chn_array, missing_data - EPSILON,
missing_data + EPSILON)
no_data = instrument_data.info["nodata"]
chn_array = np.ma.masked_inside(chn_array, no_data - EPSILON,
no_data + EPSILON)
satscene[chn] = chn_array
satscene[chn].data = np.ma.masked_less(
satscene[chn].data * gain + intercept, 0)
satscene[chn].info['units'] = units
else:
LOG.warning("Channel " + str(chn) + " not available, not loaded.")
# Compulsory global attribudes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
lons = instrument_data.londata / math.pi * 180
lats = instrument_data.latdata / math.pi * 180
try:
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lons, lats=lats)
except ImportError:
satscene.area = None
satscene.lat = lats
satscene.lon = lons
def setUpClass(cls):
import xarray as xr
import dask.array as da
cls.area_def = geometry.AreaDefinition('areaD',
'Europe (3km, HRV, VTC)',
'areaD',
{'a': '6378144.0',
'b': '6356759.0',
'lat_0': '50.00',
'lat_ts': '50.00',
'lon_0': '8.00',
'proj': 'stere'},
800,
800,
[-1370912.72,
-909968.64000000001,
1029087.28,
1490031.3600000001])
dfa = da.from_array # shortcut
cls.chunks = chunks = 5
cls.tgrid = geometry.CoordinateDefinition(
lons=dfa(np.array([
[11.5, 12.562036, 12.9],
[11.5, 12.562036, 12.9],
[11.5, 12.562036, 12.9],
[11.5, 12.562036, 12.9],
]), chunks=chunks),
lats=dfa(np.array([
[55.715613, 55.715613, 55.715613],
[55.715613, 55.715613, 55.715613],
[55.715613, np.nan, 55.715613],
[55.715613, 55.715613, 55.715613],
]), chunks=chunks))
cls.tdata_1d = xr.DataArray(
dfa(np.array([1., 2., 3.]), chunks=chunks), dims=('my_dim1',))
cls.tlons_1d = xr.DataArray(
dfa(np.array([11.280789, 12.649354, 12.080402]), chunks=chunks),
dims=('my_dim1',))
cls.tlats_1d = xr.DataArray(
dfa(np.array([56.011037, 55.629675, 55.641535]), chunks=chunks),
dims=('my_dim1',))
cls.tswath_1d = geometry.SwathDefinition(lons=cls.tlons_1d,
lats=cls.tlats_1d)
cls.data_2d = xr.DataArray(
da.from_array(np.fromfunction(lambda y, x: y * x, (50, 10)),
chunks=5),
dims=('my_dim_y', 'my_dim_x'))
cls.data_3d = xr.DataArray(
da.from_array(np.fromfunction(lambda y, x, b: y * x * b, (50, 10, 3)),
chunks=5),
dims=('my_dim_y', 'my_dim_x', 'bands'),
coords={'bands': ['r', 'g', 'b']})
cls.lons_2d = xr.DataArray(
da.from_array(np.fromfunction(lambda y, x: 3 + x, (50, 10)),
chunks=5),
dims=('my_dim_y', 'my_dim_x'))
cls.lats_2d = xr.DataArray(
da.from_array(np.fromfunction(lambda y, x: 75 - y, (50, 10)),
chunks=5),
dims=('my_dim_y', 'my_dim_x'))
cls.swath_def_2d = geometry.SwathDefinition(lons=cls.lons_2d,
lats=cls.lats_2d)
cls.src_area_2d = geometry.AreaDefinition(
'areaD_src', 'Europe (3km, HRV, VTC)', 'areaD',
{'a': '6378144.0', 'b': '6356759.0', 'lat_0': '52.00',
'lat_ts': '52.00', 'lon_0': '5.00', 'proj': 'stere'}, 50, 10,
[-1370912.72, -909968.64000000001, 1029087.28,
1490031.3600000001])
) # was readvar.getRasterHeight() in the following stanza
readvar = p.getBand('latitude')
lats = np.zeros(w * h, np.float32)
readvar.readPixels(0, 0, w, h, lats)
latmin = lats.min()
latmax = lats.max()
lats.shape = h, w
readvar = p.getBand('longitude')
lons = np.zeros(w * h, np.float32)
readvar.readPixels(0, 0, w, h, lons)
lonmin = lons.min()
lonmax = lons.max()
lons.shape = h, w
#definition of the swath object
swath = geometry.SwathDefinition(lons=lons, lats=lats)
start_time_str = str(p.getStartTime())
start_time = datetime.datetime.strptime(start_time_str, '%d-%b-%Y %H:%M:%S.%f')
stop_time_str = str(p.getEndTime())
stop_time = datetime.datetime.strptime(stop_time_str, '%d-%b-%Y %H:%M:%S.%f')
#test if swath and area overlap. If not, exit
if not (area.overlaps(swath)):
if args.verbose:
print "product outside of area"
p.dispose()
if (istar or iszip) and args.delete:
rmtree(product)
sys.exit()
def load(scene, *args, **kwargs):
"""Loads the *channels* into the satellite *scene*.
A possible *calibrate* keyword argument is passed to the AAPP reader
Should be 0 for off, 1 for default, and 2 for radiances only.
However, as the AAPP-lvl1b file contains radiances this reader cannot
return counts, so calibrate=0 is not allowed/supported. The radiance to
counts conversion is not possible.
"""
del args
calibrate = kwargs.get("calibrate", True)
if calibrate == 0:
raise ValueError('calibrate=0 is not supported! ' +
'This reader cannot return counts')
if kwargs.get("filename") is not None:
filename = kwargs["filename"]
else:
filename = (kwargs.get("filename", None)
or get_filename(scene, "level2"))
if isinstance(filename, (list, tuple, set)):
filenames = filename
else:
filenames = [filename]
LOG.debug("Using file(s) %s", str(filename))
readers = [EpsAvhrrL1bReader(filename) for filename in filenames]
arrs = {}
llons = []
llats = []
loaded_channels = set()
for reader in readers:
for chname, arr in reader.get_channels(scene.channels_to_load,
calibrate).items():
arrs.setdefault(chname, []).append(arr)
loaded_channels.add(chname)
if scene.orbit is None:
scene.orbit = int(reader["ORBIT_START"][0])
scene.info["orbit_number"] = scene.orbit
lons, lats = reader.get_full_lonlats()
llons.append(lons)
llats.append(lats)
for chname in loaded_channels:
scene[chname] = np.vstack(arrs[chname])
if chname in ["1", "2", "3A"]:
scene[chname].info["units"] = "%"
elif chname in ["4", "5", "3B"]:
scene[chname].info["units"] = "K"
lons = np.vstack(llons)
lats = np.vstack(llats)
try:
scene.area = geometry.SwathDefinition(lons, lats)
except NameError:
scene.lons, scene.lats = lons, lats
def __init__(self,
in_area,
out_area,
in_latlons=None,
mode=None,
radius=10000,
nprocs=1):
if (mode is not None and mode not in ["quick", "nearest"]):
raise ValueError("Projector mode must be 'nearest' or 'quick'")
self.area_file = get_area_file()
self.in_area = None
self.out_area = None
self._cache = None
self._filename = None
self.mode = "quick"
self.radius = radius
self.conf = ConfigParser.ConfigParser()
self.conf.read(os.path.join(CONFIG_PATH, "mpop.cfg"))
# TODO:
# - Rework so that in_area and out_area can be lonlats.
# - Add a recompute flag ?
# Setting up the input area
try:
self.in_area = get_area_def(in_area)
in_id = in_area
except (utils.AreaNotFound, AttributeError):
try:
in_id = in_area.area_id
self.in_area = in_area
except AttributeError:
try:
self.in_area = geometry.SwathDefinition(lons=in_latlons[0],
lats=in_latlons[1])
in_id = in_area
except TypeError:
raise utils.AreaNotFound(
"Input area " + str(in_area) + " must be defined in " +
self.area_file + ", be an area object"
" or longitudes/latitudes must be "
"provided.")
# Setting up the output area
try:
self.out_area = get_area_def(out_area)
out_id = out_area
except (utils.AreaNotFound, AttributeError):
try:
out_id = out_area.area_id
self.out_area = out_area
except AttributeError:
raise utils.AreaNotFound("Output area " + str(out_area) +
" must be defined in " +
self.area_file + " or "
"be an area object.")
#if self.in_area == self.out_area:
# return
# choosing the right mode if necessary
if mode is None:
try:
dicts = in_area.proj_dict, out_area.proj_dict
del dicts
self.mode = "quick"
except AttributeError:
self.mode = "nearest"
else:
self.mode = mode
filename = (in_id + "2" + out_id + "_" +
str(_get_area_hash(self.in_area)) + "to" +
str(_get_area_hash(self.out_area)) + "_" + self.mode +
".npz")
projections_directory = "/var/tmp"
try:
projections_directory = self.conf.get("projector",
"projections_directory")
except ConfigParser.NoSectionError:
pass
self._filename = os.path.join(projections_directory, filename)
try:
self._cache = {}
self._file_cache = np.load(self._filename)
except:
logger.info("Computing projection from %s to %s...", in_id, out_id)
if self.mode == "nearest":
valid_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(self.in_area,
self.out_area,
self.radius,
neighbours=1,
nprocs=nprocs)
del distance_array
self._cache = {}
self._cache['valid_index'] = valid_index
self._cache['valid_output_index'] = valid_output_index
self._cache['index_array'] = index_array
elif self.mode == "quick":
ridx, cidx = \
utils.generate_quick_linesample_arrays(self.in_area,
self.out_area)
self._cache = {}
self._cache['row_idx'] = ridx
self._cache['col_idx'] = cidx
def setUp(self):
"""Do some setup for common things."""
import dask.array as da
from xarray import DataArray
from pyresample import geometry, kd_tree
self.pts_irregular = (np.array([
[-1., 1.],
]), np.array([
[1., 2.],
]), np.array([
[-2., -1.],
]), np.array([
[2., -4.],
]))
self.pts_vert_parallel = (np.array([
[-1., 1.],
]), np.array([
[1., 2.],
]), np.array([
[-1., -1.],
]), np.array([
[1., -2.],
]))
self.pts_both_parallel = (np.array([
[-1., 1.],
]), np.array([
[1., 1.],
]), np.array([
[-1., -1.],
]), np.array([
[1., -1.],
]))
# Area definition with four pixels
self.target_def = geometry.AreaDefinition(
'areaD', 'Europe (3km, HRV, VTC)', 'areaD', {
'a': '6378144.0',
'b': '6356759.0',
'lat_0': '50.00',
'lat_ts': '50.00',
'lon_0': '8.00',
'proj': 'stere'
}, 4, 4,
[-1370912.72, -909968.64000000001, 1029087.28, 1490031.3600000001])
# Input data around the target pixel at 0.63388324, 55.08234642,
in_shape = (100, 100)
self.data1 = DataArray(da.ones((in_shape[0], in_shape[1])),
dims=('y', 'x'))
self.data2 = 2. * self.data1
self.data3 = self.data1 + 9.5
lons, lats = np.meshgrid(np.linspace(-25., 40., num=in_shape[0]),
np.linspace(45., 75., num=in_shape[1]))
self.source_def = geometry.SwathDefinition(lons=lons, lats=lats)
self.radius = 50e3
self.neighbours = 32
valid_input_index, output_idxs, index_array, dists = \
kd_tree.get_neighbour_info(self.source_def, self.target_def,
self.radius, neighbours=self.neighbours,
nprocs=1)
input_size = valid_input_index.sum()
index_mask = (index_array == input_size)
index_array = np.where(index_mask, 0, index_array)
self.valid_input_index = valid_input_index
self.index_array = index_array
shp = self.source_def.shape
self.cols, self.lines = np.meshgrid(np.arange(shp[1]),
np.arange(shp[0]))
def test_init(self, gah, gni, gqla):
"""Creation of coverage.
"""
# in case of wrong number of arguments
self.assertRaises(TypeError, Projector)
self.assertRaises(TypeError, Projector, random_string(20))
# in case of string arguments
in_area_id = random_string(20)
out_area_id = random_string(20)
area_type = utils.parse_area_file.return_value.__getitem__.return_value
gni.side_effect = [("a", "b", "c", "d")] * 10
self.proj = Projector(in_area_id, out_area_id)
self.assertEquals(utils.parse_area_file.call_count, 2)
area_file = mpop.projector.get_area_file()
utils.parse_area_file.assert_any_call(area_file, in_area_id)
utils.parse_area_file.assert_any_call(area_file, out_area_id)
self.assertEquals(self.proj.in_area, area_type)
self.assertEquals(self.proj.out_area, area_type)
# in case of undefined areas
mock = MagicMock(side_effect=Exception("raise"))
with patch.object(utils, 'parse_area_file', mock):
self.assertRaises(Exception, Projector, "raise", random_string(20))
self.assertRaises(Exception, Projector, random_string(20), "raise")
# in case of geometry objects as input
with patch.object(utils, 'AreaNotFound', Exception):
mock = MagicMock(
side_effect=[utils.AreaNotFound("raise"),
MagicMock()])
with patch.object(utils, 'parse_area_file', mock):
in_area = geometry.AreaDefinition()
self.proj = Projector(in_area, out_area_id)
print self.proj.in_area
self.assertEquals(self.proj.in_area, in_area)
in_area = geometry.SwathDefinition()
utils.parse_area_file.return_value.__getitem__.side_effect = [
AttributeError, out_area_id
]
self.proj = Projector(in_area, out_area_id)
self.assertEquals(self.proj.in_area, in_area)
out_area = geometry.AreaDefinition()
utils.parse_area_file.return_value.__getitem__.side_effect = [
in_area_id, AttributeError
]
self.proj = Projector(in_area_id, out_area)
self.assertEquals(self.proj.out_area, out_area)
# in case of lon/lat is input
utils.parse_area_file.return_value.__getitem__.side_effect = [
AttributeError, out_area_id
]
lonlats = ("great_lons", "even_greater_lats")
self.proj = Projector("raise", out_area_id, lonlats)
geometry.SwathDefinition.assert_called_with(lons=lonlats[0],
lats=lonlats[1])
utils.parse_area_file.return_value.__getitem__.side_effect = None
# in case of wrong mode
self.assertRaises(ValueError,
Projector,
random_string(20),
random_string(20),
mode=random_string(20))
utils.parse_area_file.return_value.__getitem__.side_effect = [
"a", "b", "c", "d"
]
gqla.side_effect = [("ridx", "cidx")]
# quick mode cache
self.proj = Projector(in_area_id, out_area_id, mode="quick")
cache = getattr(self.proj, "_cache")
self.assertTrue(cache['row_idx'] is not None)
self.assertTrue(cache['col_idx'] is not None)
# nearest mode cache
self.proj = Projector(in_area_id, out_area_id, mode="nearest")
cache = getattr(self.proj, "_cache")
self.assertTrue(cache['valid_index'] is not None)
self.assertTrue(cache['valid_output_index'] is not None)
self.assertTrue(cache['index_array'] is not None)
def load_virr(satscene, options):
"""Read the VIRR hdf5 file"""
if "filename" not in options:
raise IOError("No 1km virr filename given, cannot load")
values = {
"orbit": satscene.orbit,
"satname": satscene.satname,
"instrument": satscene.instrument_name,
"satellite": satscene.fullname
}
filename = \
os.path.join(satscene.time_slot.strftime(options["dir"]) % values,
satscene.time_slot.strftime(
options["filename"])
% values)
LOGGER.debug("Filename= %s", filename)
datasets = ['EV_Emissive', 'EV_RefSB']
calibrate = options['calibrate']
h5f = h5py.File(filename, 'r')
# Get geolocation information
lons = h5f['Longitude'][:]
lats = h5f['Latitude'][:]
sunz = h5f['SolarZenith'][:]
slope = h5f['SolarZenith'].attrs['Slope'][0]
intercept = h5f['SolarZenith'].attrs['Intercept'][0]
sunz = sunz * slope + intercept
sunz = np.where(np.greater(sunz, 85.0), 85.0, sunz)
# Get the calibration information
# Emissive radiance coefficients:
emis_offs = h5f['Emissive_Radiance_Offsets'][:]
emis_scales = h5f['Emissive_Radiance_Scales'][:]
# Central wave number (unit = cm-1) for the three IR bands
# It is ordered according to decreasing wave number (increasing wavelength):
# 3.7 micron, 10.8 micron, 12 micron
emiss_centroid_wn = h5f.attrs['Emmisive_Centroid_Wave_Number']
# VIS/NIR calibration stuff:
refsb_cal_coeff = h5f.attrs['RefSB_Cal_Coefficients']
visnir_scales = refsb_cal_coeff[0::2]
visnir_offs = refsb_cal_coeff[1::2]
refsb_effective_wl = h5f.attrs['RefSB_Effective_Wavelength']
# Read the band data:
for dset in datasets:
band_data = h5f[dset]
valid_range = band_data.attrs['valid_range']
LOGGER.debug("valid-range = " + str(valid_range))
fillvalue = band_data.attrs['_FillValue']
band_names = band_data.attrs['band_name'].split(',')
slope = band_data.attrs['Slope']
intercept = band_data.attrs['Intercept']
units = band_data.attrs['units']
long_name = band_data.attrs['long_name']
LOGGER.debug('band names = ' + str(band_names))
for (i, band) in enumerate(band_names):
if band not in satscene.channels_to_load:
continue
LOGGER.debug("Reading channel %s, i=%d", band, i)
data = band_data[i]
bandmask = np.logical_or(np.less(data, valid_range[0]),
np.greater(data, valid_range[1]))
if calibrate:
if dset in ['EV_Emissive']:
data = (np.array([emis_offs[:, i]]).transpose() +
data * np.array([emis_scales[:, i]]).transpose())
# Radiance to Tb conversion.
# Pyspectral wants SI units,
# but radiance data are in mW/m^2/str/cm^-1 and wavenumbers are in cm^-1
# Therefore multply wavenumber by 100 and radiances by
# 10^-5
data = rad2temp(emiss_centroid_wn[i] * 100., data * 1e-5)
if dset in ['EV_RefSB']:
data = (visnir_offs[i] + data * visnir_scales[i]) / np.cos(
np.deg2rad(sunz))
satscene[band] = np.ma.masked_array(data,
mask=bandmask,
copy=False)
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lons, lats=lats)
h5f.close()
def read_swath_and_create_geometry(dataFolder, fileIndex, year, printStatus):
if printStatus:
print(
' Step 2: Reading in the binary grid and creating the original geometry'
)
print(' Reading in binary data from file')
fileID = ref.indexAndYearToFileID(fileIndex, year)
metadata_dictionary = read_metadata_dictionary(dataFolder, fileID)
swathID = ref.fileNameToSwathID(fileID)
dataPath = dataFolder.joinpath('Raw', str(year), 'Data',
swathID + '.hgt.grd')
g = np.fromfile(str(dataPath), dtype='<f4')
# cast to float 2
g = g.astype(np.dtype('<f2'))
grid = np.reshape(g, (metadata_dictionary['GRD Latitude Lines'],
metadata_dictionary['GRD Longitude Samples']))
if printStatus:
print(' Preparing original geometry of the swath from metadata')
grid = np.where(grid > grid.min(), grid, np.nan)
min_swath_lon = metadata_dictionary['GRD Starting Longitude']
max_swath_lon = metadata_dictionary['GRD Starting Longitude'] + metadata_dictionary['GRD Longitude Samples'] * \
metadata_dictionary['GRD Longitude Spacing']
min_swath_lat = metadata_dictionary['GRD Starting Latitude'] + metadata_dictionary['GRD Latitude Lines'] * \
metadata_dictionary['GRD Latitude Spacing']
max_swath_lat = metadata_dictionary['GRD Starting Latitude']
lats = np.linspace(min_swath_lat, max_swath_lat,
metadata_dictionary['GRD Latitude Lines'])
lons = np.linspace(min_swath_lon, max_swath_lon,
metadata_dictionary['GRD Longitude Samples'])
grid = np.flipud(grid)
if printStatus:
# print(" The grid shape is (" + str(len(lats)) + "," + str(len(lons)) + ")")
print(" The grid shape is (" + str(np.shape(grid)[0]) +
"," + str(np.shape(grid)[1]) + ")")
# Original Area definition in swath geometry:
Lons, Lats = np.meshgrid(lons, lats)
Lons = np.reshape(Lons, (np.size(Lons), ))
Lats = np.reshape(Lats, (np.size(Lats), ))
grid = np.reshape(grid, (np.size(grid), ))
# Remove nans so averaging is ubiquitous
non_nans = np.invert(np.isnan(grid))
if printStatus:
print(' Removed ' + str(np.sum(np.isnan(grid))) +
' nan points out of ' + str(np.size(grid)) + ' grid points')
Lons = Lons[non_nans]
Lats = Lats[non_nans]
grid = grid[non_nans]
area_original = geometry.SwathDefinition(lons=Lons, lats=Lats)
return (area_original, grid)
def setUpClass(cls):
cls.pts_irregular = (np.array([
[-1., 1.],
]), np.array([
[1., 2.],
]), np.array([
[-2., -1.],
]), np.array([
[2., -4.],
]))
cls.pts_vert_parallel = (np.array([
[-1., 1.],
]), np.array([
[1., 2.],
]), np.array([
[-1., -1.],
]), np.array([
[1., -2.],
]))
cls.pts_both_parallel = (np.array([
[-1., 1.],
]), np.array([
[1., 1.],
]), np.array([
[-1., -1.],
]), np.array([
[1., -1.],
]))
# Area definition with four pixels
target_def = geometry.AreaDefinition(
'areaD', 'Europe (3km, HRV, VTC)', 'areaD', {
'a': '6378144.0',
'b': '6356759.0',
'lat_0': '50.00',
'lat_ts': '50.00',
'lon_0': '8.00',
'proj': 'stere'
}, 4, 4,
[-1370912.72, -909968.64000000001, 1029087.28, 1490031.3600000001])
# Input data around the target pixel at 0.63388324, 55.08234642,
in_shape = (100, 100)
cls.data1 = np.ones((in_shape[0], in_shape[1]))
cls.data2 = 2. * cls.data1
cls.data3 = cls.data1 + 9.5
lons, lats = np.meshgrid(np.linspace(-25., 40., num=in_shape[0]),
np.linspace(45., 75., num=in_shape[1]))
cls.swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
radius = 50e3
cls.neighbours = 32
input_idxs, output_idxs, idx_ref, dists = \
kd_tree.get_neighbour_info(cls.swath_def, target_def,
radius, neighbours=cls.neighbours,
nprocs=1)
input_size = input_idxs.sum()
index_mask = (idx_ref == input_size)
idx_ref = np.where(index_mask, 0, idx_ref)
cls.input_idxs = input_idxs
cls.target_def = target_def
cls.idx_ref = idx_ref
def load(self, satscene, *args, **kwargs):
"""Read data from file and load it into *satscene*.
"""
lonlat_is_loaded = False
prodfilename = kwargs.get('filename')
if "SST" not in satscene.channels_to_load:
LOG.warning("No SST product requested. Nothing to be done...")
return
try:
area_name = satscene.area_id or satscene.area.area_id
except AttributeError:
area_name = "satproj_?????_?????"
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
# Reading the products
product = "sst"
classes = {product: OsisafNarSstProduct}
LOG.debug("Loading " + product)
if isinstance(prodfilename, (list, tuple, set)):
for fname in prodfilename:
kwargs['filename'] = fname
self.load(satscene, *args, **kwargs)
return
elif (prodfilename and 'OSISAF-L3C' in os.path.basename(prodfilename)):
if os.path.basename(prodfilename).split("_")[2] == 'SST':
filename = prodfilename
else:
LOG.warning("Product file name is not as expected: " +
str(prodfilename))
return
else:
filename = get_filename(satscene, area_name)
LOG.info("Filename = " + str(filename))
chn = classes[product]()
chn.read(filename, lonlat_is_loaded == False)
satscene.channels.append(chn)
lons, lats = chn.lon.data, chn.lat.data
lonlat_is_loaded = True
nodata_mask = False
try:
from pyresample import geometry
lons = np.ma.masked_array(lons, nodata_mask)
lats = np.ma.masked_array(lats, nodata_mask)
area = geometry.SwathDefinition(lons=lons, lats=lats)
except ImportError:
area = None
if area:
chn.area = area
else:
chn.lon = lons
chn.lat = lats
LOG.info("Loading OSISAF SST parameters done")
return
class Test(unittest.TestCase):
area_def = geometry.AreaDefinition(
'areaD', 'Europe (3km, HRV, VTC)', 'areaD', {
'a': '6378144.0',
'b': '6356759.0',
'lat_0': '50.00',
'lat_ts': '50.00',
'lon_0': '8.00',
'proj': 'stere'
}, 800, 800,
[-1370912.72, -909968.64000000001, 1029087.28, 1490031.3600000001])
tdata = numpy.array([1, 2, 3])
tlons = numpy.array([11.280789, 12.649354, 12.080402])
tlats = numpy.array([56.011037, 55.629675, 55.641535])
tswath = geometry.SwathDefinition(lons=tlons, lats=tlats)
tgrid = geometry.CoordinateDefinition(lons=numpy.array([12.562036]),
lats=numpy.array([55.715613]))
def test_nearest_base(self):
res = kd_tree.resample_nearest(self.tswath,
self.tdata.ravel(),
self.tgrid,
100000,
reduce_data=False,
segments=1)
self.assertTrue(res[0] == 2, 'Failed to calculate nearest neighbour')
def test_gauss_base(self):
with catch_warnings() as w:
res = kd_tree.resample_gauss(self.tswath,
self.tdata.ravel(),
self.tgrid,
50000,
25000,
reduce_data=False,
segments=1)
self.assertFalse(len(w) != 1, 'Failed to create neighbour warning')
self.assertFalse(('Searching' not in str(w[0].message)),
'Failed to create correct neighbour warning')
self.assertAlmostEqual(res[0], 2.2020729, 5,
'Failed to calculate gaussian weighting')
def test_custom_base(self):
def wf(dist):
return 1 - dist / 100000.0
with catch_warnings() as w:
res = kd_tree.resample_custom(self.tswath,
self.tdata.ravel(),
self.tgrid,
50000,
wf,
reduce_data=False,
segments=1)
self.assertFalse(len(w) != 1, 'Failed to create neighbour warning')
self.assertFalse(('Searching' not in str(w[0].message)),
'Failed to create correct neighbour warning')
self.assertAlmostEqual(res[0], 2.4356757, 5,
'Failed to calculate custom weighting')
def test_gauss_uncert(self):
sigma = utils.fwhm2sigma(41627.730557884883)
with catch_warnings() as w:
res, stddev, count = kd_tree.resample_gauss(self.tswath,
self.tdata,
self.tgrid,
100000,
sigma,
with_uncert=True)
self.assertTrue(len(w) > 0, 'Failed to create neighbour warning')
self.assertTrue((any('Searching' in str(_w.message) for _w in w)),
'Failed to create correct neighbour warning')
expected_res = 2.20206560694
expected_stddev = 0.707115076173
expected_count = 3
self.assertAlmostEqual(
res[0], expected_res, 5,
'Failed to calculate gaussian weighting with uncertainty')
self.assertAlmostEqual(
stddev[0], expected_stddev, 5,
'Failed to calculate uncertainty for gaussian weighting')
self.assertEqual(
count[0], expected_count,
'Wrong data point count for gaussian weighting with uncertainty')
def test_custom_uncert(self):
def wf(dist):
return 1 - dist / 100000.0
with catch_warnings() as w:
res, stddev, counts = kd_tree.resample_custom(self.tswath,
self.tdata,
self.tgrid,
100000,
wf,
with_uncert=True)
self.assertTrue(len(w) > 0, 'Failed to create neighbour warning')
self.assertTrue((any('Searching' in str(_w.message) for _w in w)),
'Failed to create correct neighbour warning')
self.assertAlmostEqual(
res[0], 2.32193149, 5,
'Failed to calculate custom weighting with uncertainty')
self.assertAlmostEqual(
stddev[0], 0.81817972, 5,
'Failed to calculate custom for gaussian weighting')
self.assertEqual(
counts[0], 3,
'Wrong data point count for custom weighting with uncertainty')
def test_nearest(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
segments=1)
cross_sum = res.sum()
expected = 15874591.0
self.assertEqual(cross_sum,
expected,
msg='Swath resampling nearest failed')
def test_nearest_1d(self):
data = numpy.fromfunction(lambda x, y: x * y, (800, 800))
lons = numpy.fromfunction(lambda x: 3 + x / 100., (500, ))
lats = numpy.fromfunction(lambda x: 75 - x / 10., (500, ))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(self.area_def,
data.ravel(),
swath_def,
50000,
segments=1)
cross_sum = res.sum()
expected = 35821299.0
self.assertEqual(res.shape, (500, ),
msg='Swath resampling nearest 1d failed')
self.assertEqual(cross_sum,
expected,
msg='Swath resampling nearest 1d failed')
def test_nearest_empty(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
segments=1)
cross_sum = res.sum()
expected = 0
self.assertEqual(cross_sum,
expected,
msg='Swath resampling nearest empty failed')
def test_nearest_empty_multi(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data_multi,
self.area_def,
50000,
segments=1)
self.assertEqual(res.shape, (800, 800, 3),
msg='Swath resampling nearest empty multi failed')
def test_nearest_empty_multi_masked(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data_multi,
self.area_def,
50000,
segments=1,
fill_value=None)
self.assertEqual(
res.shape, (800, 800, 3),
msg='Swath resampling nearest empty multi masked failed')
def test_nearest_empty_masked(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
segments=1,
fill_value=None)
cross_sum = res.mask.sum()
expected = res.size
self.assertTrue(cross_sum == expected,
msg='Swath resampling nearest empty masked failed')
def test_nearest_segments(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
segments=2)
cross_sum = res.sum()
expected = 15874591.0
self.assertEqual(cross_sum,
expected,
msg='Swath resampling nearest segments failed')
def test_nearest_remap(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
segments=1)
remap = kd_tree.resample_nearest(self.area_def,
res.ravel(),
swath_def,
5000,
segments=1)
cross_sum = remap.sum()
expected = 22275.0
self.assertEqual(cross_sum,
expected,
msg='Grid remapping nearest failed')
def test_nearest_mp(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
nprocs=2,
segments=1)
cross_sum = res.sum()
expected = 15874591.0
self.assertEqual(cross_sum,
expected,
msg='Swath resampling mp nearest failed')
def test_nearest_multi(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
res = kd_tree.resample_nearest(swath_def,
data_multi,
self.area_def,
50000,
segments=1)
cross_sum = res.sum()
expected = 3 * 15874591.0
self.assertEqual(cross_sum,
expected,
msg='Swath multi channel resampling nearest failed')
def test_nearest_multi_unraveled(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.dstack((data, data, data))
res = kd_tree.resample_nearest(swath_def,
data_multi,
self.area_def,
50000,
segments=1)
cross_sum = res.sum()
expected = 3 * 15874591.0
self.assertEqual(cross_sum,
expected,
msg='Swath multi channel resampling nearest failed')
def test_gauss_sparse(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_gauss(swath_def,
data.ravel(),
self.area_def,
50000,
25000,
fill_value=-1,
segments=1)
cross_sum = res.sum()
expected = 15387753.9852
self.assertAlmostEqual(cross_sum,
expected,
places=3,
msg='Swath gauss sparse nearest failed')
def test_gauss(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-5, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
with catch_warnings() as w:
res = kd_tree.resample_gauss(swath_def,
data.ravel(),
self.area_def,
50000,
25000,
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 4872.81050892
self.assertAlmostEqual(cross_sum,
expected,
msg='Swath resampling gauss failed')
def test_gauss_fwhm(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-5, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
with catch_warnings() as w:
res = kd_tree.resample_gauss(swath_def,
data.ravel(),
self.area_def,
50000,
utils.fwhm2sigma(41627.730557884883),
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 4872.81050892
self.assertAlmostEqual(cross_sum,
expected,
msg='Swath resampling gauss failed')
def test_gauss_multi(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
res = kd_tree.resample_gauss(swath_def,
data_multi,
self.area_def,
50000, [25000, 15000, 10000],
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 1461.84313918
self.assertAlmostEqual(
cross_sum,
expected,
msg='Swath multi channel resampling gauss failed')
def test_gauss_multi_uncert(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
# The assertion below checks if there is only one warning raised
# and whether it contains a specific message from pyresample
# On python 2.7.9+ the resample_gauss method raises multiple deprecation warnings
# that cause to fail, so we ignore the unrelated warnings.
res, stddev, counts = kd_tree.resample_gauss(swath_def,
data_multi,
self.area_def,
50000,
[25000, 15000, 10000],
segments=1,
with_uncert=True)
self.assertTrue(
len(w) >= 1, 'Failed to create neighbour radius warning')
self.assertTrue(
any(['Possible more' in str(x.message) for x in w]),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
cross_sum_counts = counts.sum()
expected = 1461.84313918
expected_stddev = [0.446193170875, 0.443606880035, 0.438586349519]
expected_counts = 4934802.0
self.assertTrue(res.shape == stddev.shape
and stddev.shape == counts.shape
and counts.shape == (800, 800, 3))
self.assertAlmostEqual(
cross_sum,
expected,
msg='Swath multi channel resampling gauss failed on data')
for i, e_stddev in enumerate(expected_stddev):
cross_sum_stddev = stddev[:, :, i].sum()
print(cross_sum_stddev, e_stddev)
self.assertAlmostEqual(
cross_sum_stddev,
e_stddev,
msg=
'Swath multi channel resampling gauss failed on stddev (channel {})'
.format(i))
self.assertAlmostEqual(
cross_sum_counts,
expected_counts,
msg='Swath multi channel resampling gauss failed on counts')
def test_gauss_multi_mp(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
res = kd_tree.resample_gauss(swath_def,
data_multi,
self.area_def,
50000, [25000, 15000, 10000],
nprocs=2,
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 1461.84313918
self.assertAlmostEqual(
cross_sum,
expected,
msg='Swath multi channel resampling gauss failed')
def test_gauss_multi_mp_segments(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
res = kd_tree.resample_gauss(swath_def,
data_multi,
self.area_def,
50000, [25000, 15000, 10000],
nprocs=2,
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 1461.84313918
self.assertAlmostEqual(
cross_sum,
expected,
msg='Swath multi channel segments resampling gauss failed')
def test_gauss_multi_mp_segments_empty(self):
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 165 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
res = kd_tree.resample_gauss(swath_def,
data_multi,
self.area_def,
50000, [25000, 15000, 10000],
nprocs=2,
segments=1)
cross_sum = res.sum()
self.assertTrue(cross_sum == 0,
msg=('Swath multi channel segments empty '
'resampling gauss failed'))
def test_custom(self):
def wf(dist):
return 1 - dist / 100000.0
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-5, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
with catch_warnings() as w:
res = kd_tree.resample_custom(swath_def,
data.ravel(),
self.area_def,
50000,
wf,
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 4872.81050729
self.assertAlmostEqual(cross_sum,
expected,
msg='Swath custom resampling failed')
def test_custom_multi(self):
def wf1(dist):
return 1 - dist / 100000.0
def wf2(dist):
return 1
def wf3(dist):
return numpy.cos(dist)**2
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
res = kd_tree.resample_custom(swath_def,
data_multi,
self.area_def,
50000, [wf1, wf2, wf3],
segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
cross_sum = res.sum()
expected = 1461.842980746
self.assertAlmostEqual(
cross_sum,
expected,
msg='Swath multi channel custom resampling failed')
def test_reduce(self):
data = numpy.fromfunction(lambda y, x: (y + x), (1000, 1000))
lons = numpy.fromfunction(lambda y, x: -180 + (360.0 / 1000) * x,
(1000, 1000))
lats = numpy.fromfunction(lambda y, x: -90 + (180.0 / 1000) * y,
(1000, 1000))
grid_lons, grid_lats = self.area_def.get_lonlats()
lons, lats, data = data_reduce.swath_from_lonlat_grid(
grid_lons, grid_lats, lons, lats, data, 7000)
cross_sum = data.sum()
expected = 20514375.0
self.assertAlmostEqual(cross_sum, expected, msg='Reduce data failed')
def test_reduce_boundary(self):
data = numpy.fromfunction(lambda y, x: (y + x), (1000, 1000))
lons = numpy.fromfunction(lambda y, x: -180 + (360.0 / 1000) * x,
(1000, 1000))
lats = numpy.fromfunction(lambda y, x: -90 + (180.0 / 1000) * y,
(1000, 1000))
boundary_lonlats = self.area_def.get_boundary_lonlats()
lons, lats, data = data_reduce.swath_from_lonlat_boundaries(
boundary_lonlats[0], boundary_lonlats[1], lons, lats, data, 7000)
cross_sum = data.sum()
expected = 20514375.0
self.assertAlmostEqual(cross_sum, expected, msg='Reduce data failed')
def test_cartesian_reduce(self):
data = numpy.fromfunction(lambda y, x: (y + x), (1000, 1000))
lons = numpy.fromfunction(lambda y, x: -180 + (360.0 / 1000) * x,
(1000, 1000))
lats = numpy.fromfunction(lambda y, x: -90 + (180.0 / 1000) * y,
(1000, 1000))
#grid = utils.generate_cartesian_grid(self.area_def)
grid = self.area_def.get_cartesian_coords()
lons, lats, data = data_reduce.swath_from_cartesian_grid(
grid, lons, lats, data, 7000)
cross_sum = data.sum()
expected = 20514375.0
self.assertAlmostEqual(cross_sum,
expected,
msg='Cartesian reduce data failed')
def test_area_con_reduce(self):
data = numpy.fromfunction(lambda y, x: (y + x), (1000, 1000))
lons = numpy.fromfunction(lambda y, x: -180 + (360.0 / 1000) * x,
(1000, 1000))
lats = numpy.fromfunction(lambda y, x: -90 + (180.0 / 1000) * y,
(1000, 1000))
grid_lons, grid_lats = self.area_def.get_lonlats()
valid_index = data_reduce.get_valid_index_from_lonlat_grid(
grid_lons, grid_lats, lons, lats, 7000)
data = data[valid_index]
cross_sum = data.sum()
expected = 20514375.0
self.assertAlmostEqual(cross_sum, expected, msg='Reduce data failed')
def test_area_con_cartesian_reduce(self):
data = numpy.fromfunction(lambda y, x: (y + x), (1000, 1000))
lons = numpy.fromfunction(lambda y, x: -180 + (360.0 / 1000) * x,
(1000, 1000))
lats = numpy.fromfunction(lambda y, x: -90 + (180.0 / 1000) * y,
(1000, 1000))
cart_grid = self.area_def.get_cartesian_coords()
valid_index = data_reduce.get_valid_index_from_cartesian_grid(
cart_grid, lons, lats, 7000)
data = data[valid_index]
cross_sum = data.sum()
expected = 20514375.0
self.assertAlmostEqual(cross_sum,
expected,
msg='Cartesian reduce data failed')
def test_masked_nearest(self):
data = numpy.ones((50, 10))
data[:, 5:] = 2
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
mask = numpy.ones((50, 10))
mask[:, :5] = 0
masked_data = numpy.ma.array(data, mask=mask)
res = kd_tree.resample_nearest(swath_def,
masked_data.ravel(),
self.area_def,
50000,
segments=1)
expected_mask = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_nearest_mask.dat'),
sep=' ').reshape((800, 800))
expected_data = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_nearest_data.dat'),
sep=' ').reshape((800, 800))
self.assertTrue(numpy.array_equal(expected_mask, res.mask),
msg='Resampling of swath mask failed')
self.assertTrue(numpy.array_equal(expected_data, res.data),
msg='Resampling of swath masked data failed')
def test_masked_nearest_1d(self):
data = numpy.ones((800, 800))
data[:400, :] = 2
lons = numpy.fromfunction(lambda x: 3 + x / 100., (500, ))
lats = numpy.fromfunction(lambda x: 75 - x / 10., (500, ))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
mask = numpy.ones((800, 800))
mask[400:, :] = 0
masked_data = numpy.ma.array(data, mask=mask)
res = kd_tree.resample_nearest(self.area_def,
masked_data.ravel(),
swath_def,
50000,
segments=1)
self.assertEqual(res.mask.sum(),
108,
msg='Swath resampling masked nearest 1d failed')
def test_masked_gauss(self):
data = numpy.ones((50, 10))
data[:, 5:] = 2
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
mask = numpy.ones((50, 10))
mask[:, :5] = 0
masked_data = numpy.ma.array(data, mask=mask)
res = kd_tree.resample_gauss(swath_def,
masked_data.ravel(),
self.area_def,
50000,
25000,
segments=1)
expected_mask = numpy.fromfile(os.path.join(os.path.dirname(__file__),
'test_files',
'mask_test_mask.dat'),
sep=' ').reshape((800, 800))
expected_data = numpy.fromfile(os.path.join(os.path.dirname(__file__),
'test_files',
'mask_test_data.dat'),
sep=' ').reshape((800, 800))
expected = expected_data.sum()
cross_sum = res.data.sum()
self.assertTrue(numpy.array_equal(expected_mask, res.mask),
msg='Gauss resampling of swath mask failed')
self.assertAlmostEqual(
cross_sum,
expected,
places=3,
msg='Gauss resampling of swath masked data failed')
def test_masked_fill_float(self):
data = numpy.ones((50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
fill_value=None,
segments=1)
expected_fill_mask = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_fill_value.dat'),
sep=' ').reshape((800, 800))
fill_mask = res.mask
self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask),
msg='Failed to create fill mask on float data')
def test_masked_fill_int(self):
data = numpy.ones((50, 10)).astype('int')
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
data.ravel(),
self.area_def,
50000,
fill_value=None,
segments=1)
expected_fill_mask = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_fill_value.dat'),
sep=' ').reshape((800, 800))
fill_mask = res.mask
self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask),
msg='Failed to create fill mask on integer data')
def test_masked_full(self):
data = numpy.ones((50, 10))
data[:, 5:] = 2
mask = numpy.ones((50, 10))
mask[:, :5] = 0
masked_data = numpy.ma.array(data, mask=mask)
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
masked_data.ravel(),
self.area_def,
50000,
fill_value=None,
segments=1)
expected_fill_mask = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_full_fill.dat'),
sep=' ').reshape((800, 800))
fill_mask = res.mask
self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask),
msg='Failed to create fill mask on masked data')
def test_masked_full_multi(self):
data = numpy.ones((50, 10))
data[:, 5:] = 2
mask1 = numpy.ones((50, 10))
mask1[:, :5] = 0
mask2 = numpy.ones((50, 10))
mask2[:, 5:] = 0
mask3 = numpy.ones((50, 10))
mask3[:25, :] = 0
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
mask_multi = numpy.column_stack(
(mask1.ravel(), mask2.ravel(), mask3.ravel()))
masked_data = numpy.ma.array(data_multi, mask=mask_multi)
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
res = kd_tree.resample_nearest(swath_def,
masked_data,
self.area_def,
50000,
fill_value=None,
segments=1)
expected_fill_mask = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_full_fill_multi.dat'),
sep=' ').reshape((800, 800, 3))
fill_mask = res.mask
cross_sum = res.sum()
expected = 357140.0
self.assertAlmostEqual(cross_sum,
expected,
msg='Failed to resample masked data')
self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask),
msg='Failed to create fill mask on masked data')
def test_dtype(self):
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
grid_def = geometry.GridDefinition(lons, lats)
lons = numpy.asarray(lons, dtype='f4')
lats = numpy.asarray(lats, dtype='f4')
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
valid_input_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(swath_def,
grid_def,
50000, neighbours=1, segments=1)
def test_nearest_from_sample(self):
data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
valid_input_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(swath_def,
self.area_def,
50000, neighbours=1, segments=1)
res = kd_tree.get_sample_from_neighbour_info('nn', (800, 800),
data.ravel(),
valid_input_index,
valid_output_index,
index_array)
cross_sum = res.sum()
expected = 15874591.0
self.assertEqual(
cross_sum,
expected,
msg='Swath resampling from neighbour info nearest failed')
def test_custom_multi_from_sample(self):
def wf1(dist):
return 1 - dist / 100000.0
def wf2(dist):
return 1
def wf3(dist):
return numpy.cos(dist)**2
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
valid_input_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(swath_def,
self.area_def,
50000, segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
res = kd_tree.get_sample_from_neighbour_info(
'custom', (800, 800),
data_multi,
valid_input_index,
valid_output_index,
index_array,
distance_array,
weight_funcs=[wf1, wf2, wf3])
cross_sum = res.sum()
expected = 1461.842980746
self.assertAlmostEqual(
cross_sum,
expected,
msg=
'Swath multi channel custom resampling from neighbour info failed 1'
)
res = kd_tree.get_sample_from_neighbour_info(
'custom', (800, 800),
data_multi,
valid_input_index,
valid_output_index,
index_array,
distance_array,
weight_funcs=[wf1, wf2, wf3])
# Look for error where input data has been manipulated
cross_sum = res.sum()
expected = 1461.842980746
self.assertAlmostEqual(
cross_sum,
expected,
msg=
'Swath multi channel custom resampling from neighbour info failed 2'
)
def test_masked_multi_from_sample(self):
data = numpy.ones((50, 10))
data[:, 5:] = 2
mask1 = numpy.ones((50, 10))
mask1[:, :5] = 0
mask2 = numpy.ones((50, 10))
mask2[:, 5:] = 0
mask3 = numpy.ones((50, 10))
mask3[:25, :] = 0
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
mask_multi = numpy.column_stack(
(mask1.ravel(), mask2.ravel(), mask3.ravel()))
masked_data = numpy.ma.array(data_multi, mask=mask_multi)
lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
valid_input_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(swath_def,
self.area_def,
50000, neighbours=1, segments=1)
res = kd_tree.get_sample_from_neighbour_info('nn', (800, 800),
masked_data,
valid_input_index,
valid_output_index,
index_array,
fill_value=None)
expected_fill_mask = numpy.fromfile(os.path.join(
os.path.dirname(__file__), 'test_files',
'mask_test_full_fill_multi.dat'),
sep=' ').reshape((800, 800, 3))
fill_mask = res.mask
self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask),
msg='Failed to create fill mask on masked data')
def SDR2Geotiff(HDF, output_dir, areaid, radius):
radiance, qf1_viirsdnbsdr, qf2_scan_sdr, lon_data, lat_data, LunarZenithAngle, qf2_viirssdrgeo, qf2_viirssdrgeo_tc = readhdfDatasets(
HDF)
HDF = ntpath.basename(HDF) #get filename without path
#QF Flags from https://ncc.nesdis.noaa.gov/documents/documentation/viirs-sdr-dataformat.pdf
#1.VIIRS Fill Values (p.70)
radiance_fillvalues = np.array([-999.3, -999.5, -999.8, -999.9])
radiance_mask = np.isin(radiance, radiance_fillvalues)
#2.Edge-of-swath pixels
edge_of_swath_mask = np.zeros_like(radiance, dtype='bool')
edge_of_swath_mask[:, 0:254] = 1
edge_of_swath_mask[:, 3810:4064] = 1
#3.QF1_VIIRSDNBSDR flags (p.73)
#SDR_Quality_mask = (qf1_viirsdnbsdr & 3)>0
Saturated_Pixel_mask = ((qf1_viirsdnbsdr & 12) >> 2) > 0
Missing_Data_mask = ((qf1_viirsdnbsdr & 48) >> 4) > 0
Out_of_Range_mask = ((qf1_viirsdnbsdr & 64) >> 6) > 0
#4.QF2_VIIRSSDRGEO flags (p.90)
qf2_viirssdrgeo_do0_mask = (qf2_viirssdrgeo & 1) > 0
qf2_viirssdrgeo_do1_mask = ((qf2_viirssdrgeo & 2) >> 1) > 0
qf2_viirssdrgeo_do2_mask = ((qf2_viirssdrgeo & 4) >> 2) > 0
qf2_viirssdrgeo_do3_mask = ((qf2_viirssdrgeo & 8) >> 3) > 0
#Combine pixel level flags
viirs_mask = np.logical_or.reduce((
radiance_mask,
edge_of_swath_mask,
#SDR_Quality_mask,
Saturated_Pixel_mask,
Missing_Data_mask,
Out_of_Range_mask,
qf2_viirssdrgeo_do0_mask,
qf2_viirssdrgeo_do1_mask,
qf2_viirssdrgeo_do2_mask,
qf2_viirssdrgeo_do3_mask,
))
fill_value = 255.0
radiance[viirs_mask] = fill_value #set fill value for masked pixels in DNB
LunarZenithAngle[
viirs_mask] = fill_value #set fill value for masked pixels in DNB
swath_def = geometry.SwathDefinition(
xr.DataArray(da.from_array(lon_data, chunks=4096), dims=('y', 'x')),
xr.DataArray(da.from_array(lat_data, chunks=4096), dims=('y', 'x')))
metadata_dict = {'name': 'dnb', 'area': swath_def}
scn = Scene()
scn['Radiance'] = xr.DataArray(
da.from_array(radiance, chunks=4096),
attrs=metadata_dict,
dims=('y', 'x')
) #https://satpy.readthedocs.io/en/latest/dev_guide/xarray_migration.html#id1
scn['LunarZenithAngle'] = xr.DataArray(da.from_array(LunarZenithAngle,
chunks=4096),
attrs=metadata_dict,
dims=('y', 'x'))
scn.load(["Radiance"])
scn.load(["LunarZenithAngle"])
#print(scn)
proj_scn = scn.resample(areaSettings.getarea(areaid),
radius_of_influence=radius)
proj_scn.save_datasets(writer='geotiff',
base_dir=output_dir,
file_pattern="{}.{}.{}".format(
HDF, "{name}", "tif"),
enhancement_config=False,
dtype=np.float32,
fill_value=fill_value)
def test_custom_multi_from_sample(self):
def wf1(dist):
return 1 - dist / 100000.0
def wf2(dist):
return 1
def wf3(dist):
return numpy.cos(dist)**2
data = numpy.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100))
lons = numpy.fromfunction(lambda y, x: 3 + (10.0 / 100) * x,
(5000, 100))
lats = numpy.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y,
(5000, 100))
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
data_multi = numpy.column_stack(
(data.ravel(), data.ravel(), data.ravel()))
with catch_warnings() as w:
valid_input_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(swath_def,
self.area_def,
50000, segments=1)
self.assertFalse(
len(w) != 1, 'Failed to create neighbour radius warning')
self.assertFalse(
('Possible more' not in str(w[0].message)),
'Failed to create correct neighbour radius warning')
res = kd_tree.get_sample_from_neighbour_info(
'custom', (800, 800),
data_multi,
valid_input_index,
valid_output_index,
index_array,
distance_array,
weight_funcs=[wf1, wf2, wf3])
cross_sum = res.sum()
expected = 1461.842980746
self.assertAlmostEqual(
cross_sum,
expected,
msg=
'Swath multi channel custom resampling from neighbour info failed 1'
)
res = kd_tree.get_sample_from_neighbour_info(
'custom', (800, 800),
data_multi,
valid_input_index,
valid_output_index,
index_array,
distance_array,
weight_funcs=[wf1, wf2, wf3])
# Look for error where input data has been manipulated
cross_sum = res.sum()
expected = 1461.842980746
self.assertAlmostEqual(
cross_sum,
expected,
msg=
'Swath multi channel custom resampling from neighbour info failed 2'
)
def resample_to_grid_only_valid_return(input_data,
src_lon,
src_lat,
target_lon,
target_lat,
methods='nn',
weight_funcs=None,
min_neighbours=1,
search_rad=18000,
neighbours=8,
fill_values=None):
"""
resamples data from dictionary of numpy arrays using pyresample
to given grid.
Searches for the neighbours and then resamples the data
to the grid given in togrid if at least
min_neighbours neighbours are found
Parameters
----------
input_data : dict of numpy.arrays
src_lon : numpy.array
longitudes of the input data
src_lat : numpy.array
src_latitudes of the input data
target_lon : numpy.array
longitudes of the output data
target_src_lat : numpy.array
src_latitudes of the output data
methods : string or dict, optional
method of spatial averaging. this is given to pyresample
and can be
'nn' : nearest neighbour
'custom' : custom weight function has to be supplied in weight_funcs
see pyresample documentation for more details
can also be a dictionary with a method for each array in input data dict
weight_funcs : function or dict of functions, optional
if method is 'custom' a function like func(distance) has to be given
can also be a dictionary with a function for each array in input data dict
min_neighbours: int, optional
if given then only points with at least this number of neighbours will be
resampled
Default : 1
search_rad : float, optional
search radius in meters of neighbour search
Default : 18000
neighbours : int, optional
maximum number of neighbours to look for for each input grid point
Default : 8
fill_values : number or dict, optional
if given the output array will be filled with this value if no valid
resampled value could be computed, if not a masked array will be returned
can also be a dict with a fill value for each variable
Returns
-------
data : dict of numpy.arrays
resampled data on part of the target grid over which data was found
mask: numpy.ndarray
boolean mask into target grid that specifies where data was resampled
Raises
------
ValueError :
if empty dataset is resampled
"""
output_data = {}
if target_lon.ndim == 2:
target_lat = target_lat.ravel()
target_lon = target_lon.ravel()
input_swath = geometry.SwathDefinition(src_lon, src_lat)
output_swath = geometry.SwathDefinition(target_lon, target_lat)
(valid_input_index, valid_output_index, index_array,
distance_array) = kd_tree.get_neighbour_info(input_swath,
output_swath,
search_rad,
neighbours=neighbours)
# throw away points with less than min_neighbours neighbours
# find points with valid neighbours
# get number of found neighbours for each grid point/row
if neighbours > 1:
nr_neighbours = np.isfinite(distance_array).sum(1)
neigh_condition = nr_neighbours >= min_neighbours
mask = np.invert(neigh_condition)
enough_neighbours = np.nonzero(neigh_condition)[0]
if neighbours == 1:
nr_neighbours = np.isfinite(distance_array)
neigh_condition = nr_neighbours >= min_neighbours
mask = np.invert(neigh_condition)
enough_neighbours = np.nonzero(neigh_condition)[0]
distance_array = np.reshape(distance_array,
(distance_array.shape[0], 1))
index_array = np.reshape(index_array, (index_array.shape[0], 1))
if enough_neighbours.size == 0:
raise ValueError("No points with at least %d neighbours found" %
min_neighbours)
# remove neighbourhood info of input grid points that have no neighbours to not have to
# resample to whole output grid for small input grid file
distance_array = distance_array[enough_neighbours, :]
index_array = index_array[enough_neighbours, :]
valid_output_index = valid_output_index[enough_neighbours]
for param in input_data:
data = input_data[param]
if type(methods) == dict:
method = methods[param]
else:
method = methods
if method is not 'nn':
if type(weight_funcs) == dict:
weight_func = weight_funcs[param]
else:
weight_func = weight_funcs
else:
weight_func = None
neigh_slice = slice(None, None, None)
# check if method is nn, if so only use first row of index_array and
# distance_array
if method == 'nn':
neigh_slice = (slice(None, None, None), 0)
if type(fill_values) == dict:
fill_value = fill_values[param]
else:
fill_value = fill_values
output_array = kd_tree.get_sample_from_neighbour_info(
method,
enough_neighbours.shape,
data,
valid_input_index,
valid_output_index,
index_array[neigh_slice],
distance_array[neigh_slice],
weight_funcs=weight_func,
fill_value=fill_value)
output_data[param] = output_array
return output_data, mask
def load(self, keys):
"""Read data from file and return the corresponding projectables.
"""
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
projectables = []
platform_name = self.mda['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE']
keys = [key for key in keys if key.resolution == self.resolution]
if len(keys) == 0:
logger.debug("Nothing to read in %s.", self.filename)
return projectables
datasets = datadict[self.resolution]
key_names = [key.name for key in keys]
for dataset in datasets:
subdata = self.sd.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(set(key_names) & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [i for i, band in enumerate(band_names)
if band in key_names]
uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(
subdata, uncertainty, indices, band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
dsid = [key for key in keys if key.name ==
band_names[idx]][0]
area = self.navigation_reader.get_lonlats(self.resolution)
projectable = Projectable(array[i], id=dsid, area=area)
if ((platform_name == 'Aqua' and dsid.name in ["6", "27", "36"]) or
(platform_name == 'Terra' and dsid.name in ["29"])):
height, width = projectable.shape
row_indices = projectable.mask.sum(1) == width
if row_indices.sum() == height:
continue
projectable.mask[indices, :] = True
projectables.append(projectable)
return projectables
# Get the orbit number
if not satscene.orbit:
mda = self.data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
for band_name in loaded_bands:
lon, lat = self.get_lonlat(satscene[band_name].resolution, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
for band_name in loaded_bands:
band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest()
satscene[band_name].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+
str(satscene[
band_name].shape) + "_"
+ str(band_uid))
satscene[band_name].area_id = satscene[band_name].area.area_id
def load_generic(satscene, filename, resolution, cores):
"""Read modis data, generic part.
"""
try:
data = SD(str(filename))
except HDF4Error as err:
logger.warning("Could not load data from " + str(filename)
+ ": " + str(err))
return
datadict = {
1000: ['EV_250_Aggr1km_RefSB',
'EV_500_Aggr1km_RefSB',
'EV_1KM_RefSB',
'EV_1KM_Emissive'],
500: ['EV_250_Aggr500_RefSB',
'EV_500_RefSB'],
250: ['EV_250_RefSB']}
datasets = datadict[resolution]
loaded_bands = []
# process by dataset, reflective and emissive datasets separately
for dataset in datasets:
subdata = data.select(dataset)
band_names = subdata.attributes()["band_names"].split(",")
if len(satscene.channels_to_load & set(band_names)) > 0:
# get the relative indices of the desired channels
indices = [i for i, band in enumerate(band_names)
if band in satscene.channels_to_load]
uncertainty = data.select(dataset + "_Uncert_Indexes")
if dataset.endswith('Emissive'):
array = calibrate_tb(subdata, uncertainty, indices, band_names)
else:
array = calibrate_refl(subdata, uncertainty, indices)
for (i, idx) in enumerate(indices):
satscene[band_names[idx]] = array[i]
# fix the resolution to match the loaded data.
satscene[band_names[idx]].resolution = resolution
loaded_bands.append(band_names[idx])
# Get the orbit number
if not satscene.orbit:
mda = data.attributes()["CoreMetadata.0"]
orbit_idx = mda.index("ORBITNUMBER")
satscene.orbit = int(mda[orbit_idx + 111:orbit_idx + 116])
# Get the geolocation
# if resolution != 1000:
# logger.warning("Cannot load geolocation at this resolution (yet).")
# return
lat, lon = get_lat_lon(satscene, resolution, filename, cores)
area = geometry.SwathDefinition(lons=lon, lats=lat)
for band_name in loaded_bands:
satscene[band_name].area = area
# Trimming out dead sensor lines (detectors) on aqua:
# (in addition channel 21 is noisy)
if satscene.satname == "aqua":
for band in ["6", "27", "36"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+ str(satscene[band].shape) + "_"
+ str(band))
# Trimming out dead sensor lines (detectors) on terra:
# (in addition channel 27, 30, 34, 35, and 36 are nosiy)
if satscene.satname == "terra":
for band in ["29"]:
if not satscene[band].is_loaded() or satscene[band].data.mask.all():
continue
width = satscene[band].data.shape[1]
height = satscene[band].data.shape[0]
indices = satscene[band].data.mask.sum(1) < width
if indices.sum() == height:
continue
satscene[band] = satscene[band].data[indices, :]
satscene[band].area = geometry.SwathDefinition(
lons=satscene[band].area.lons[indices, :],
lats=satscene[band].area.lats[indices, :])
satscene[band].area.area_id = ("swath_" + satscene.fullname + "_"
+ str(satscene.time_slot) + "_"
+ str(satscene[band].shape) + "_"
+ str(band))
def load(self, satscene, *args, **kwargs):
"""Read data from file and load it into *satscene*.
"""
lonlat_is_loaded = False
geofilename = kwargs.get('geofilename')
prodfilename = kwargs.get('filename')
products = []
if "CTTH" in satscene.channels_to_load:
products.append("ctth")
if "CT" in satscene.channels_to_load:
products.append("cloudtype")
if "CMA" in satscene.channels_to_load:
products.append("cloudmask")
if "PC" in satscene.channels_to_load:
products.append("precipclouds")
if "CPP" in satscene.channels_to_load:
products.append("cpp")
if len(products) == 0:
return
try:
area_name = satscene.area_id or satscene.area.area_id
except AttributeError:
area_name = "satproj_?????_?????"
# Looking for geolocation file
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
try:
geodir = conf.get(satscene.instrument_name + "-level3",
"cloud_product_geodir",
vars=os.environ)
except NoOptionError:
LOG.warning("No option 'geodir' in level3 section")
geodir = None
if not geofilename and geodir:
# Load geo file from config file:
try:
if not satscene.orbit:
orbit = ""
else:
orbit = satscene.orbit
geoname_tmpl = conf.get(satscene.instrument_name + "-level3",
"cloud_product_geofilename",
raw=True,
vars=os.environ)
filename_tmpl = (satscene.time_slot.strftime(geoname_tmpl)
% {"orbit": str(orbit).zfill(5) or "*",
"area": area_name,
"satellite": satscene.satname + satscene.number})
file_list = glob.glob(os.path.join(geodir, filename_tmpl))
if len(file_list) > 1:
LOG.warning("More than 1 file matching for geoloaction: "
+ str(file_list))
elif len(file_list) == 0:
LOG.warning(
"No geolocation file matching!: "
+ os.path.join(geodir, filename_tmpl))
else:
geofilename = file_list[0]
except NoOptionError:
geofilename = None
# Reading the products
classes = {"ctth": CloudTopTemperatureHeight,
"cloudtype": CloudType,
"cloudmask": CloudMask,
"precipclouds": PrecipitationClouds,
"cpp": CloudPhysicalProperties
}
nodata_mask = False
area = None
lons = None
lats = None
chn = None
shape = None
read_external_geo = {}
for product in products:
LOG.debug("Loading " + product)
if isinstance(prodfilename, (list, tuple, set)):
for fname in prodfilename:
kwargs['filename'] = fname
self.load(satscene, *args, **kwargs)
return
elif (prodfilename and
os.path.basename(prodfilename).startswith('S_NWC')):
if os.path.basename(prodfilename).split("_")[2] == NEW_PRODNAMES[product]:
filename = prodfilename
else:
continue
else:
filename = conf.get(satscene.instrument_name + "-level3",
"cloud_product_filename",
raw=True,
vars=os.environ)
directory = conf.get(satscene.instrument_name + "-level3",
"cloud_product_dir",
vars=os.environ)
pathname_tmpl = os.path.join(directory, filename)
LOG.debug("Path = " + str(pathname_tmpl))
if not satscene.orbit:
orbit = ""
else:
orbit = satscene.orbit
filename_tmpl = (satscene.time_slot.strftime(pathname_tmpl)
% {"orbit": str(orbit).zfill(5) or "*",
"area": area_name,
"satellite": satscene.satname + satscene.number,
"product": product})
file_list = glob.glob(filename_tmpl)
if len(file_list) == 0:
product_name = NEW_PRODNAMES.get(product, product)
LOG.info("No " + str(product) +
" product in old format matching")
filename_tmpl = (satscene.time_slot.strftime(pathname_tmpl)
% {"orbit": str(orbit).zfill(5) or "*",
"area": area_name,
"satellite": satscene.satname + satscene.number,
"product": product_name})
file_list = glob.glob(filename_tmpl)
if len(file_list) > 1:
LOG.warning("More than 1 file matching for " + product + "! "
+ str(file_list))
continue
elif len(file_list) == 0:
LOG.warning(
"No " + product + " matching!: " + filename_tmpl)
continue
else:
filename = file_list[0]
chn = classes[product]()
chn.read(filename, lonlat_is_loaded == False)
satscene.channels.append(chn)
# Check if geolocation is loaded:
if not chn.area:
read_external_geo[product] = chn
shape = chn.shape
# Check if some 'channel'/product needs geolocation. If some product does
# not have geolocation, get it from the geofilename:
if not read_external_geo:
LOG.info("Loading PPS parameters done.")
return
# Load geolocation
interpolate = False
if geofilename:
geodict = get_lonlat(geofilename)
lons, lats = geodict['lon'], geodict['lat']
if lons.shape != shape or lats.shape != shape:
interpolate = True
row_indices = geodict['row_indices']
column_indices = geodict['col_indices']
lonlat_is_loaded = True
else:
LOG.warning("No Geo file specified: " +
"Geolocation will be loaded from product")
if lonlat_is_loaded:
if interpolate:
from geotiepoints import SatelliteInterpolator
cols_full = np.arange(shape[1])
rows_full = np.arange(shape[0])
satint = SatelliteInterpolator((lons, lats),
(row_indices,
column_indices),
(rows_full, cols_full))
# satint.fill_borders("y", "x")
lons, lats = satint.interpolate()
try:
from pyresample import geometry
lons = np.ma.masked_array(lons, nodata_mask)
lats = np.ma.masked_array(lats, nodata_mask)
area = geometry.SwathDefinition(lons=lons,
lats=lats)
except ImportError:
area = None
for chn in read_external_geo.values():
if area:
chn.area = area
else:
chn.lat = lats
chn.lon = lons
LOG.info("Loading PPS parameters done.")
return
