def atest_interpolation_coast(self):
"""Test interpolation."""
reader = reader_ROMS_native.Reader('/disk2/data/SVIM/ocean_avg_20081201.nc')
num_points = 50
np.random.seed(0) # To get the same random numbers each time
lons = np.random.uniform(12, 16, num_points)
lats = np.random.uniform(68.3, 68.3, num_points)
z = np.random.uniform(-100, 0, num_points)
x, y = reader.lonlat2xy(lons, lats)
variables = ['x_sea_water_velocity', 'y_sea_water_velocity',
'sea_water_temperature']
# Read a block of data covering the points
data = reader.get_variables(variables, time=reader.start_time,
x=x, y=y, z=z, block=True)
import matplotlib.pyplot as plt
plt.imshow(data['x_sea_water_velocity'][0,:,:])
plt.colorbar()
plt.show()
b = ReaderBlock(data, interpolation_horizontal='nearest')
env, prof = b.interpolate(x, y, z, variables,
profiles=['x_sea_water_velocity'],
profiles_depth=[-30, 0])
def test_interpolation_missing(self):
"""Test interpolation."""
reader = reader_ROMS_native.Reader(script_folder +
'/../test_data/2Feb2016_Nordic_sigma_3d/Nordic-4km_SLEVELS_avg_00_subset2Feb2016.nc')
num_points = 50
np.random.seed(0) # To get the same random numbers each time
lons = np.random.uniform(10, 11, num_points)
lats = np.random.uniform(66, 67.0, num_points)
z = np.random.uniform(-200, 0, num_points)
x, y = reader.lonlat2xy(lons, lats)
variables = ['x_sea_water_velocity', 'y_sea_water_velocity',
'sea_water_temperature']
# Read a block of data covering the points
data = reader.get_variables(variables, time=reader.start_time,
x=x, y=y, z=z, block=True)
# Introduce missing values
data['x_sea_water_velocity'].mask[[data['x_sea_water_velocity']>.08]] = True
b = ReaderBlock(data, interpolation_horizontal='linearND')
env, prof = b.interpolate(x, y, z, variables,
profiles=['x_sea_water_velocity'],
profiles_depth=[-30, 0])
self.assertAlmostEqual(env['x_sea_water_velocity'][10],
0.0567555511984)
self.assertAlmostEqual(prof['x_sea_water_velocity'][5,48],
-0.0949447782936)
def test_repeated(self):
"""Check that block can be used for interpolation to several sets of positions"""
reader = reader_netCDF_CF_generic.Reader(script_folder +
'/../test_data/14Jan2016_NorKyst_z_3d/NorKyst-800m_ZDEPTHS_his_00_3Dsubset.nc')
# 100 points within 50x50 pixels over sea (corner of domain)
num_points = 100
np.random.seed(0) # To get the same random numbers each time
x = np.random.uniform(reader.xmin, reader.xmin+800*50, num_points)
y = np.random.uniform(reader.ymax-800*50, reader.ymax, num_points)
z = np.random.uniform(-200, 0, num_points)
variables = ['x_sea_water_velocity', 'y_sea_water_velocity',
'sea_water_temperature']
# Read a block of data covering the points
data = reader.get_variables(variables, time=reader.start_time,
x=x, y=y, z=z, block=True)
b = ReaderBlock(data, interpolation_horizontal='nearest')
env, prof = b.interpolate(x, y, z, 'sea_water_temperature')
x2 = x[20:30]
y2 = y[20:30]
z2 = z[20:30]
env2, prof2 = b.interpolate(x2, y2, z2, 'sea_water_temperature')
env3, prof3 = b.interpolate(x, y, z, 'sea_water_temperature')
self.assertEqual(env['sea_water_temperature'][0],
env3['sea_water_temperature'][0])
self.assertEqual(env['sea_water_temperature'][20],
env2['sea_water_temperature'][0])
def test_interpolation_3dArrays(self):
"""Test interpolation."""
reader = reader_netCDF_CF_generic.Reader(script_folder +
'/../test_data/14Jan2016_NorKyst_z_3d/NorKyst-800m_ZDEPTHS_his_00_3Dsubset.nc')
# 100000 points within 50x50 pixels over sea (corner of domain)
num_points = 1000
np.random.seed(0) # To get the same random numbers each time
x = np.random.uniform(reader.xmin, reader.xmin+800*50, num_points)
y = np.random.uniform(reader.ymax-800*50, reader.ymax, num_points)
z = np.random.uniform(-200, 0, num_points)
variables = ['x_sea_water_velocity', 'y_sea_water_velocity',
'sea_water_temperature']
# Read a block of data covering the points
data = reader.get_variables(variables, time=reader.start_time,
x=x, y=y, z=z, block=True)
b = ReaderBlock(data, interpolation_horizontal='nearest')
env, prof = b.interpolate(x, y, z, variables,
profiles=['sea_water_temperature'],
profiles_depth=[-30, 0])
self.assertAlmostEqual(env['x_sea_water_velocity'][100],
0.0750191849228)
self.assertAlmostEqual(prof['sea_water_temperature'][0,11],
7.5499997138977051)
self.assertAlmostEqual(prof['sea_water_temperature'][-1,11],
8.38000011444)
self.assertEqual(prof['z'][-1], b.z[-1])
def test_zNone(self):
d = {}
d['x'] = np.arange(5)
d['y'] = np.arange(7)
d['z'] = 0
d['time'] = None
d['var'] = np.random.rand(5,6)
rb = ReaderBlock(d)
z = None
i,p = rb.interpolate(np.array([1, 2]), np.array([2, 3]), z, 'var')
self.assertTrue(i['var'][0] > 0)
def test_interpolation_horizontal(self):
data_dict, x, y, z = self.get_synthetic_data_dict()
# Make block from dictionary, and apply tests
b = ReaderBlock(data_dict)
self.assertEqual(b.data_dict['var2d'].shape,
(len(b.y), len(b.x)))
self.assertEqual(b.data_dict['var3d'].shape,
(len(b.z), len(b.y), len(b.x)))
# Make some element positions
interpolator2d = b.Interpolator2DClass(b.x, b.y, x, y)
values = interpolator2d(data_dict['var2d'])
# Checking output is as expected
self.assertEqual(values[10], 1.6487979858538129)
self.assertEqual(sum(values.mask), 15)
def test_covers_positions(self):
data_dict, x, y, z = self.get_synthetic_data_dict()
# Make block from dictionary, and apply tests
b = ReaderBlock(data_dict)
xn = np.linspace(-70, 470, 100)
yn = np.linspace(10, 340, 100)
self.assertTrue(b.covers_positions(xn, yn))
xn = np.linspace(500, 600, 100)
yn = np.linspace(400, 500, 100)
self.assertFalse(b.covers_positions(xn, yn))
xn = np.linspace(400, 500, 100)
yn = np.linspace(0, 30, 100)
self.assertFalse(b.covers_positions(xn, yn))
def get_variables_interpolated(self,
variables,
profiles=None,
profiles_depth=None,
time=None,
lon=None,
lat=None,
z=None,
block=False,
rotate_to_proj=None):
# Raise error if time not not within coverage of reader
if not self.covers_time(time):
raise ValueError('%s is outside time coverage (%s - %s) of %s' %
(time, self.start_time, self.end_time, self.name))
# Check which particles are covered (indep of time)
ind_covered = self.covers_positions(lon, lat, z)
if len(ind_covered) == 0:
raise ValueError(('All %s particles (%.2f-%.2fE, %.2f-%.2fN) ' +
'are outside domain of %s (%s)') %
(len(lon), lon.min(), lon.max(), lat.min(),
lat.max(), self.name, self.coverage_string()))
# Find reader time_before/time_after
time_nearest, time_before, time_after, i1, i2, i3 = \
self.nearest_time(time)
logging.debug('Reader time:\n\t\t%s (before)\n\t\t%s (after)' %
(time_before, time_after))
if time == time_before:
time_after = None
reader_x, reader_y = self.lonlat2xy(lon[ind_covered], lat[ind_covered])
z = z.copy()[ind_covered] # Send values and not reference
# to avoid modifications
if block is False or self.return_block is False:
# Analytical reader, continous in space and time
env_before = self._get_variables(
variables,
profiles,
profiles_depth,
time,
#time_before,
reader_x,
reader_y,
z,
block=block)
logging.debug('Fetched env-before')
else:
# Swap before- and after-blocks if matching times
if str(variables) in self.var_block_before:
block_before_time = self.var_block_before[str(variables)].time
if str(variables) in self.var_block_after:
block_after_time = self.var_block_after[str(
variables)].time
if block_before_time != time_before:
if block_after_time == time_before:
self.var_block_before[str(variables)] = \
self.var_block_after[str(variables)]
if block_after_time != time_after:
if block_before_time == time_before:
self.var_block_after[str(variables)] = \
self.var_block_before[str(variables)]
# Fetch data, if no buffer is available
if (not str(variables) in self.var_block_before) or \
(self.var_block_before[str(variables)].time !=
time_before):
reader_data_dict = \
self._get_variables(variables, profiles,
profiles_depth, time_before,
reader_x, reader_y, z,
block=block)
self.var_block_before[str(variables)] = \
ReaderBlock(reader_data_dict,
interpolation_horizontal=self.interpolation)
try:
len_z = len(self.var_block_before[str(variables)].z)
except:
len_z = 1
logging.debug(('Fetched env-block (size %ix%ix%i) ' +
'for time before (%s)') %
(len(self.var_block_before[str(variables)].x),
len(self.var_block_before[str(variables)].y),
len_z, time_before))
if not str(variables) in self.var_block_after or \
self.var_block_after[str(variables)].time != time_after:
if time_after is None:
self.var_block_after[str(variables)] = \
self.var_block_before[str(variables)]
else:
reader_data_dict = \
self._get_variables(variables, profiles,
profiles_depth, time_after,
reader_x, reader_y, z,
block=block)
self.var_block_after[str(variables)] = \
ReaderBlock(
reader_data_dict,
interpolation_horizontal=self.interpolation)
try:
len_z = len(self.var_block_after[str(variables)].z)
except:
len_z = 1
logging.debug(('Fetched env-block (size %ix%ix%i) ' +
'for time after (%s)') %
(len(self.var_block_after[str(variables)].x),
len(self.var_block_after[str(variables)].y),
len_z, time_after))
if self.var_block_before[str(variables)].covers_positions(
reader_x, reader_y) is False or \
self.var_block_after[str(variables)].covers_positions(
reader_x, reader_y) is False:
logging.warning('Data block from %s not large enough to '
'cover element positions within timestep. '
'Buffer size (%s) must be increased.' %
(self.name, str(self.buffer)))
############################################################
# Interpolate before/after blocks onto particles in space
############################################################
logging.debug('Interpolating before (%s) in space (%s)' %
(self.var_block_before[str(variables)].time,
self.interpolation))
env_before, env_profiles_before = self.var_block_before[str(
variables)].interpolate(reader_x, reader_y, z, variables,
profiles, profiles_depth)
if (time_after is not None) and (time_before != time):
logging.debug('Interpolating after (%s) in space (%s)' %
(self.var_block_after[str(variables)].time,
self.interpolation))
env_after, env_profiles_after = self.var_block_after[str(
variables)].interpolate(reader_x, reader_y, z, variables,
profiles, profiles_depth)
#######################
# Time interpolation
#######################
env_profiles = None
if (time_after is not None) and (time_before != time):
weight_after = ((time - time_before).total_seconds() /
(time_after - time_before).total_seconds())
logging.debug(
('Interpolating before (%s, weight %.2f) and'
'\n\t\t after (%s, weight %.2f) in time') %
(self.var_block_before[str(variables)].time, 1 - weight_after,
self.var_block_after[str(variables)].time, weight_after))
env = {}
for var in variables:
# Weighting together, and masking invalid entries
env[var] = np.ma.masked_invalid(
(env_before[var] * (1 - weight_after) +
env_after[var] * weight_after))
if var in standard_names.keys():
if (env[var].min() < standard_names[var]['valid_min']) \
or (env[var].max() >
standard_names[var]['valid_max']):
logging.info('Invalid values found for ' + var)
logging.info(env[var])
sys.exit('quitting')
# Interpolating vertical profiles in time
if profiles is not None:
env_profiles = {}
logging.info('Interpolating profiles in time')
# Truncating layers not present both before and after
numlayers = np.minimum(len(env_profiles_before['z']),
len(env_profiles_after['z']))
env_profiles['z'] = env_profiles_before['z'][0:numlayers + 1]
for var in env_profiles_before.keys():
if var == 'z':
continue
env_profiles[var] = (
env_profiles_before[var][0:numlayers, :] *
(1 - weight_after) +
env_profiles_after[var][0:numlayers, :] * weight_after)
else:
env_profiles = None
else:
logging.debug('No time interpolation needed - right on time.')
env = env_before
if profiles is not None:
env_profiles = env_profiles_before
####################
# Rotate vectors
####################
if rotate_to_proj is not None:
if (rotate_to_proj.srs
== self.proj.srs) or (rotate_to_proj.is_latlong() is True
and self.proj.is_latlong() is True):
logging.debug('Reader SRS is the same as calculation SRS - '
'rotation of vectors is not needed.')
else:
vector_pairs = []
for var in variables:
for vector_pair in vector_pairs_xy:
if var in vector_pair:
counterpart = list(set(vector_pair) -
set([var]))[0]
if counterpart in variables:
vector_pairs.append(vector_pair)
else:
sys.exit('Missing component of vector pair:' +
counterpart)
# Extract unique vector pairs
vector_pairs = [
list(x) for x in set(tuple(x) for x in vector_pairs)
]
if len(vector_pairs) > 0:
for vector_pair in vector_pairs:
env[vector_pair[0]], env[vector_pair[1]] = \
self.rotate_vectors(reader_x, reader_y,
env[vector_pair[0]],
env[vector_pair[1]],
self.proj, rotate_to_proj)
if profiles is not None and vector_pair[0] in profiles:
sys.exit('Rotating profiles of vectors '
'is not yet implemented')
# Masking non-covered pixels
if len(ind_covered) != len(lon):
logging.debug('Masking %i elements outside coverage' %
(len(lon) - len(ind_covered)))
for var in variables:
tmp = np.nan * np.ones(lon.shape)
tmp[ind_covered] = env[var].copy()
env[var] = np.ma.masked_invalid(tmp)
# Filling also fin missing columns
# for env_profiles outside coverage
if env_profiles is not None and var in env_profiles.keys():
tmp = np.nan * np.ones(
(env_profiles[var].shape[0], len(lon)))
tmp[:, ind_covered] = env_profiles[var].copy()
env_profiles[var] = np.ma.masked_invalid(tmp)
return env, env_profiles