def test_get_time_with_tolerance(self):
time_dims = [datetime.strptime('2001-02-12 09:00:00.00000', '%Y-%m-%d %H:%M:%S.%f'),
datetime.strptime('2001-02-12 09:14:02.00000', '%Y-%m-%d %H:%M:%S.%f')]
returned_indices = [None, 26]
F = FileReader(self.stub.ncfile.name)
file_indices = [F.time_to_index(i, tolerance=10) for i in time_dims]
test.assert_equal(file_indices, returned_indices)
def test_get_negative_time_series(self):
F1 = FileReader(self.stub.ncfile.name,
dims={'node': [10]},
variables=['zeta'])
F2 = FileReader(self.stub.ncfile.name,
dims={
'node': [10],
'time': -np.arange(10, 40)
},
variables=['zeta'])
test.assert_almost_equal(F2.data.zeta,
F1.data.zeta[-np.arange(10, 40)],
decimal=5)
def test_get_layer(self):
vertical_velocity = np.array([
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815,
0.01509815, 0.01509815, 0.01509815, 0.01509815
])
F = FileReader(self.stub.ncfile.name,
dims={
'siglay': [5],
'time': [100, 101]
},
variables=['ww'])
test.assert_almost_equal(np.squeeze(F.data.ww),
vertical_velocity,
decimal=5)
def test_get_time_with_datetime(self):
time_dims = [datetime.strptime('2001-02-12 09:00:00.00000', '%Y-%m-%d %H:%M:%S.%f'),
datetime.strptime('2001-02-14 12:00:00.00000', '%Y-%m-%d %H:%M:%S.%f')]
returned_indices = [26, 76]
F = FileReader(self.stub.ncfile.name, dims={'time': time_dims})
test.assert_equal(F._dims['time'][0], returned_indices[0])
test.assert_equal(F._dims['time'][-1], returned_indices[-1])
def test_add_files(self):
# Make another stub file which follows in time from the existing one. Then only load a section of that in
# time and make sure the results are the same as if we'd loaded them manually and added them together.
next_stub = StubFile(self.endtime, self.endtime + relativedelta(months=1), self.interval,
lon=self.lon, lat=self.lat, triangles=self.triangles, zone='30N')
# Append the new stub file to the old one.
F1 = FileReader(self.stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww'])
F2 = FileReader(next_stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww'])
all_times = np.concatenate((F1.time.datetime[:], F2.time.datetime[:]), axis=0)
all_data = np.concatenate((F1.data.ww[:], F2.data.ww[:]), axis=0)
# Repeat the process, but use the __add__ method in FileReader.
F1 = FileReader(self.stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww'])
F2 = FileReader(next_stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww'])
F = F2 >> F1
test.assert_equal(F.time.datetime, all_times)
test.assert_equal(F.data.ww, all_data)
def test_get_water_column(self):
F = FileReader(self.stub.ncfile.name,
dims={
'node': [5],
'time': 10
},
variables=['temp'])
test.assert_almost_equal(np.squeeze(F.data.temp),
self.reference.data.temp[10, :, 5],
decimal=5)
def test_get_layer(self):
F = FileReader(self.stub.ncfile.name,
dims={
'siglay': [5],
'time': [100]
},
variables=['ww'])
test.assert_almost_equal(np.squeeze(F.data.ww),
self.reference.data.ww[100, 5, :],
decimal=5)
def test_get_single_time_negative_index(self):
F = FileReader(self.stub.ncfile.name,
dims={
'node': [10],
'time': [-10]
},
variables=['zeta'])
test.assert_almost_equal(np.squeeze(F.data.zeta),
self.reference.data.zeta[-10, 10],
decimal=5)
def test_get_time_series(self):
F = FileReader(self.stub.ncfile.name,
dims={
'node': [10],
'time': np.arange(10, 40)
},
variables=['zeta'])
test.assert_almost_equal(np.squeeze(F.data.zeta),
self.reference.data.zeta[10:40, 10],
decimal=5)
def test_get_single_time_negative_index(self):
surface_elevation = np.array([[0.22058453]])
F = FileReader(self.stub.ncfile.name,
dims={
'node': [10],
'time': [-10, -9]
},
variables=['zeta'])
test.assert_almost_equal(np.squeeze(F.data.zeta),
surface_elevation,
decimal=5)
def test_get_single_time(self):
surface_elevation = np.array([-1.41013891])
F = FileReader(self.stub.ncfile.name,
dims={
'node': [10],
'time': [10, 11]
},
variables=['zeta'])
test.assert_almost_equal(np.squeeze(F.data.zeta),
surface_elevation,
decimal=5)
def test_get_bounding_box(self):
wesn = [-5, -3, 50, 55]
extents = [
-4.9847326278686523, -3.0939722061157227, 50.19110107421875,
54.946651458740234
]
F = FileReader(self.stub.ncfile.name, dims={'wesn': wesn})
test.assert_equal(F.grid.lon.min(), extents[0])
test.assert_equal(F.grid.lon.max(), extents[1])
test.assert_equal(F.grid.lat.min(), extents[2])
test.assert_equal(F.grid.lat.max(), extents[3])
def setUp(self):
self.starttime, self.endtime, self.interval, self.lon, self.lat, self.triangles = _prep(
)
self.stub = StubFile(self.starttime,
self.endtime,
self.interval,
lon=self.lon,
lat=self.lat,
triangles=self.triangles,
zone='30N')
self.reference = FileReader(self.stub.ncfile.name,
variables=['ww', 'zeta', 'temp', 'h'])
def retrieve_file_data(self):
where_str = 'buoy_name in ('
for this_buoy in self.buoy_list:
where_str += this_buoy + ','
where_str = where_str[0:-1] + ')'
buoy_lat_lons = self.wco_database.select_query('sites', where_str,
'buoy_name, lon, lat')
first_file = True
model_all_dicts = {}
for this_file in self.file_list:
if first_file:
fvcom_data_reader = FileReader(this_file, ['temp', 'salinity'])
close_node = []
for this_buoy_ll in buoy_lat_lons:
close_node.append()
model_depths = fvcom_data_reader.grid.siglay * fvcom_data_reader.grid.h * -1
for this_buoy in self.buoy_list:
model_all_dicts[this_buoy] = {
'dt_time': mod_times,
'temp': mod_t_vals,
'salinity': mod_s_vals
}
first_file = False
else:
fvcom_data_reader = FileReader(this_file, ['temp', 'salinity'],
dims={'node': [close_node]})
for this_buoy in self.buoy_list:
model_all_dicts
model_depths = fvcom_data_reader.grid.siglay * fvcom_data_reader.grid.h * -1
for this_buoy in self.buoy_list:
model_all_dicts[this_buoy]['depth'] = model_dp
for this_buoy in self.buoy_list:
self.model_data[this_buoy] = model_all_dicts[this_buoy]
def test_get_water_column(self):
water_column = np.array([
9.99821472, 10.90714264, 11.81607151, 12.72500038, 13.6339283,
14.54285717, 15.45178604, 16.36071396, 17.26964378, 18.1785717
])
F = FileReader(self.stub.ncfile.name,
dims={
'node': [5],
'time': [10, 11]
},
variables=['temp'])
test.assert_almost_equal(np.squeeze(F.data.temp),
water_column,
decimal=5)
def setUp(self):
self.starttime, self.endtime, self.interval, self.lon, self.lat, self.triangles = _prep(
)
self.model = StubFile(self.starttime,
self.endtime,
self.interval,
lon=self.lon,
lat=self.lat,
triangles=self.triangles,
zone='30N')
self.fvcom = FileReader(self.model.ncfile.name,
variables=['u', 'v', 'ua', 'va'])
self.fvcom.grid.x, self.fvcom.grid.y, _ = utm_from_lonlat(
self.fvcom.grid.lon, self.fvcom.grid.lat, zone='30N')
self.fvcom.grid.xc, self.fvcom.grid.yc, _ = utm_from_lonlat(
self.fvcom.grid.lonc, self.fvcom.grid.latc, zone='30N')
def test_get_time_series(self):
surface_elevation = np.array([
-1.41013891, -0.98554639, -0.3139296, 0.4363727, 1.07729949,
1.44820437, 1.45612114, 1.09906549, 0.46653234, -0.28293574,
-0.96148683, -1.3990441, -1.48593514, -1.20038097, -0.61395488,
0.12635717, 0.83499817, 1.33434937, 1.49924987, 1.28836787,
0.75456029, 0.03162443, -0.699238, -1.25483851, -1.49591749,
-1.3620492, -0.88678734, -0.18925488, 0.55571376, 1.1613944
])
F = FileReader(self.stub.ncfile.name,
dims={
'node': [10],
'time': [10, 40]
},
variables=['zeta'])
test.assert_almost_equal(np.squeeze(F.data.zeta),
surface_elevation,
decimal=5)
def test_non_temporal_variable(self):
h = np.asarray([
1.64808428, 12.75706577, 18.34670639, 24.29236031, 29.7772541,
25.00211716, 22.69193077, 18.70510674, 21.96312141, 27.35856438,
35.32657623, 32.48567581, 38.93023682, 43.63704681, 51.21723175,
53.23581314, 59.78393555, 55.53053284, 52.84440994, 57.36302185,
62.2620163, 66.50558472, 61.24137878, 60.91600418, 67.42472839,
73.38938904, 70.63117981, 70.62969208, 75.18034363, 79.09741974,
84.4043808, 81.0752182, 88.22835541, 90.34424591, 97.57055664,
98.27231598, 100.0000000, 96.82516479, 91.1933136, 88.29994202,
89.59196472, 91.40013885, 85.90748596, 79.28456879, 74.37998199,
70.46596527, 70.78884888, 70.06604004, 63.42258453, 63.06575394,
59.99647141, 57.27880096, 55.11286545, 61.5132103, 62.31158066,
59.2288208, 53.60129929, 50.73873138, 56.42451477, 52.42653656,
44.78648376, 39.55376434, 32.51250839, 28.38024521, 20.91413689,
18.19268227, 11.62014961, 7.51470757, 38.44644928, 45.77177048,
34.9041214, 51.38194275, 77.87741852, 81.04411316
])
F = FileReader(self.stub.ncfile.name, variables=['h'])
test.assert_almost_equal(F.data.h, h)
def InitialiseGrid(self,
lower_left_ll,
upper_right_ll,
grid_res,
depth_layers,
time_varying_depth=True):
"""
Sets up the grids to interpolate to and from. Part of the work is reducing the domain, then comes the trick step of interpolating onto vertical layers
which vary in time and may create discontiguous areas of the model domain.
Parameters
----------
lower_left_ll : 1d array
A two entry array of the lon/lat of the lower left corner of the regular grid to be interpolated onto
upper_right_ll : 1d array
The corresponding lon/lat of the upper right corner
grid_res : float
The resolution in degrees of the regular grid
depth layers : array
Array of the depths to interpolate onto. These are positive down starting from the top of the free surface or the zero mean
depending on the setting of time_varying_depth
"""
if self._noisy:
print('Rank {}: Loading grid'.format(self.rank))
# Check ther regular grid limits fit with the grid resolution specified
if not round((upper_right_ll[0] - lower_left_ll[0]) % grid_res,
10) % grid_res == 0:
print(
'Longitudes not divisible by grid resolution, extending grid')
upper_right_ll[0] = round(
np.ceil((upper_right_ll[0] - lower_left_ll[0]) / grid_res) *
grid_res, 10) + lower_left_ll[0]
if not round((upper_right_ll[1] - lower_left_ll[1]) % grid_res,
10) % grid_res == 0:
print('Latitudes not divisible by grid resolution, extending grid')
upper_right_ll[1] = round(
np.ceil((upper_right_ll[1] - lower_left_ll[1]) / grid_res) *
grid_res, 10) + lower_left_ll[1]
self.regular_grid.lons = np.linspace(
lower_left_ll[0], upper_right_ll[0],
(upper_right_ll[0] - lower_left_ll[0]) / grid_res + 1)
self.regular_grid.lats = np.linspace(
lower_left_ll[1], upper_right_ll[1],
(upper_right_ll[1] - lower_left_ll[1]) / grid_res + 1)
self.regular_grid.mesh_lons, self.regular_grid.mesh_lats = np.meshgrid(
self.regular_grid.lons, self.regular_grid.lats)
self.regular_grid.dep_lays = np.asarray(depth_layers)
# And load the fvcom grid, reducing to the interpolation area only
fvcom_grid_fr = FileReader(self.fvcom_file, ['zeta'],
dims={'time': self.time_indices})
fvcom_nodes_reduce = np.logical_and(
np.logical_and(fvcom_grid_fr.grid.lon >= lower_left_ll[0],
fvcom_grid_fr.grid.lon <= upper_right_ll[0]),
np.logical_and(fvcom_grid_fr.grid.lat >= lower_left_ll[1],
fvcom_grid_fr.grid.lat <= upper_right_ll[1]))
fvcom_nodes_reduce = np.squeeze(np.argwhere(fvcom_nodes_reduce))
## extend the node list to include attached nodes outside the area, this should make the interpolation better
fvcom_nodes = np.isin(fvcom_grid_fr.grid.triangles, fvcom_nodes_reduce)
self.fvcom_grid.nodes = np.unique(fvcom_grid_fr.grid.triangles[
np.asarray(np.sum(fvcom_nodes, axis=1), dtype=bool), :])
self.fvcom_grid.triangles, self.fvcom_grid.elements = reduce_triangulation(
fvcom_grid_fr.grid.triangles,
self.fvcom_grid.nodes,
return_elements=True)
self.fvcom_grid.ll = np.asarray([
fvcom_grid_fr.grid.lon[self.fvcom_grid.nodes],
fvcom_grid_fr.grid.lat[self.fvcom_grid.nodes]
]).T
self.fvcom_grid.elements_ll = np.asarray([
fvcom_grid_fr.grid.lonc[self.fvcom_grid.elements],
fvcom_grid_fr.grid.latc[self.fvcom_grid.elements]
]).T
self.regular_grid.initial_mask = mask_to_fvcom_meshgrid(
self.fvcom_grid.ll, self.fvcom_grid.triangles,
self.regular_grid.mesh_lons, self.regular_grid.mesh_lats)
self.fvcom_grid.h = fvcom_grid_fr.grid.h[self.fvcom_grid.nodes]
self.fvcom_grid.zeta = fvcom_grid_fr.data.zeta[:,
self.fvcom_grid.nodes]
self.fvcom_grid.zeta_center = node_to_centre(
fvcom_grid_fr.data.zeta, fvcom_grid_fr)[:,
self.fvcom_grid.elements]
self.fvcom_grid.sigma = fvcom_grid_fr.grid.siglay[:, self.fvcom_grid.
nodes]
self.fvcom_grid.h_center = fvcom_grid_fr.grid.h_center[
self.fvcom_grid.elements]
self.fvcom_grid.sigma_center = fvcom_grid_fr.grid.siglay_center[:,
self.
fvcom_grid
.
elements]
if time_varying_depth:
if self._noisy:
print('Rank {}: Calculating time varying mask'.format(
self.rank))
fvcom_dep_lays = -unstructured_grid_depths(
self.fvcom_grid.h, self.fvcom_grid.zeta, self.fvcom_grid.sigma)
## change to depth from free surface since I *think* this is what cmems does?
self.fvcom_grid.dep_lays = fvcom_dep_lays - np.tile(
np.min(fvcom_dep_lays, axis=1)[:, np.newaxis, :],
[1, fvcom_dep_lays.shape[1], 1])
self.fvcom_grid.total_depth = np.max(self.fvcom_grid.dep_lays,
axis=1)
self.regular_grid.mask = np.ones([
len(self.time_indices),
len(self.regular_grid.lons),
len(self.regular_grid.lats),
len(self.regular_grid.dep_lays)
],
dtype=bool)
for this_t in np.arange(0, len(self.time_indices)):
# retriangulate for each depth layer, can be multiple if there are split regions and interpolate
for this_depth_lay_ind, this_depth_lay in enumerate(
self.regular_grid.dep_lays):
if self._noisy:
print('Rank {}: Time step {}, Depth {}'.format(
self.rank, this_t, this_depth_lay_ind))
this_depth_layer_nodes = np.where(
self.fvcom_grid.total_depth[
this_t, :] >= this_depth_lay)[0]
if this_depth_layer_nodes.size:
this_depth_tri = reduce_triangulation(
self.fvcom_grid.triangles, this_depth_layer_nodes)
this_td_mask = np.ones(
self.regular_grid.mesh_lons.shape, dtype=bool)
this_td_mask[
~self.regular_grid.initial_mask] = mask_to_fvcom(
self.fvcom_grid.ll[this_depth_layer_nodes],
this_depth_tri,
self.regular_grid.
mesh_lons[~self.regular_grid.initial_mask],
self.regular_grid.
mesh_lats[~self.regular_grid.initial_mask],
split_domain_check=True).T
self.regular_grid.mask[
this_t, :, :, this_depth_lay_ind] = this_td_mask.T
else:
print('Not implemented yet')
def test_get_multipe_latc(self):
latc = np.array((52.8649059, 52.90310308))
F = FileReader(self.stub.ncfile.name, dims={'nele': [29, 34]})
test.assert_almost_equal(F.grid.latc, latc, decimal=5)
def test_get_time_with_string(self):
time_dims = ['2001-02-12 09:00:00.00000', '2001-02-14 12:00:00.00000']
returned_indices = [26, 78]
F = FileReader(self.stub.ncfile.name, dims={'time': time_dims})
test.assert_equal(F._dims['time'], returned_indices)
def test_get_multipe_lat(self):
lat = np.array((56.89470906, 58.49899088))
F = FileReader(self.stub.ncfile.name, dims={'node': [29, 34]})
test.assert_almost_equal(F.grid.lat, lat, decimal=5)
def test_get_multipe_lonc(self):
lonc = np.array((-4.67915533, 0.61115498))
F = FileReader(self.stub.ncfile.name, dims={'nele': [0, 5]})
test.assert_almost_equal(F.grid.lonc, lonc, decimal=5)
def test_get_multipe_lon(self):
lon = np.array((-5.78687373, -3.26585943))
F = FileReader(self.stub.ncfile.name, dims={'node': [0, 5]})
test.assert_almost_equal(F.grid.lon, lon, decimal=5)
def test_get_single_latc(self):
latc = 52.864905897403958
F = FileReader(self.stub.ncfile.name, dims={'nele': [29]})
test.assert_almost_equal(F.grid.latc, latc, decimal=5)
def test_get_single_lonc(self):
lonc = -4.67915533
F = FileReader(self.stub.ncfile.name, dims={'nele': [0]})
test.assert_almost_equal(F.grid.lonc, lonc, decimal=5)
def test_get_single_lat(self):
lat = 56.89470906
F = FileReader(self.stub.ncfile.name, dims={'node': [29]})
test.assert_almost_equal(F.grid.lat, lat, decimal=5)
def test_get_single_lon(self):
lon = -5.78687373
F = FileReader(self.stub.ncfile.name, dims={'node': [0]})
test.assert_almost_equal(F.grid.lon, lon, decimal=5)
def InterpolateRegular(self, variable, mode='nodes'):
"""
Actually do the interpolation
"""
if mode in ['nodes', 'surface']:
self.fvcom_grid.select_points = self.fvcom_grid.nodes
self.fvcom_grid.select_ll = self.fvcom_grid.ll
else:
self.fvcom_grid.select_points = self.fvcom_grid.elements
self.fvcom_grid.select_ll = self.fvcom_grid.elements_ll
if mode in ['nodes', 'surface']:
self.fvcom_grid.select_dep_lays = self.fvcom_grid.dep_lays
else:
fvcom_dep_lays = -unstructured_grid_depths(
self.fvcom_grid.h_center, self.fvcom_grid.zeta_center,
self.fvcom_grid.sigma_center)
## change to depth from free surface since I *think* this is what cmems does?
self.fvcom_grid.select_dep_lays = fvcom_dep_lays - np.tile(
np.min(fvcom_dep_lays, axis=1)[:, np.newaxis, :],
[1, fvcom_dep_lays.shape[1], 1])
if mode == 'surface':
reg_grid_data = np.zeros([
len(self.time_indices),
len(self.regular_grid.lons),
len(self.regular_grid.lats)
])
else:
reg_grid_data = np.zeros([
len(self.time_indices),
len(self.regular_grid.lons),
len(self.regular_grid.lats),
len(self.regular_grid.dep_lays)
])
reg_grid_data[:] = np.nan
fvcom_data = getattr(
FileReader(self.fvcom_file, [variable],
dims={
'time': self.time_indices
}).data, variable)[..., self.fvcom_grid.select_points]
for this_t in np.arange(0, len(self.time_indices)):
if self._noisy:
print('Rank {}: Interpolating time step {} for {}'.format(
self.rank, this_t, variable))
if mode == 'surface':
depth_lay_data = fvcom_data[this_t, :]
interp_data = self._Interpolater(depth_lay_data).T
interp_data[self.regular_grid.mask[this_t, :, :, 0]] = np.nan
reg_grid_data[this_t, :, :] = interp_data
else:
depth_lay_data = np.zeros([
len(self.regular_grid.dep_lays),
len(self.fvcom_grid.select_points)
])
for i in np.arange(0, len(self.fvcom_grid.select_points)):
depth_lay_data[:, i] = np.interp(
self.regular_grid.dep_lays,
self.fvcom_grid.select_dep_lays[this_t, :, i],
fvcom_data[this_t, :, i],
left=np.nan,
right=np.nan)
# Replace surface data as it can't interpolate properly there
if self.regular_grid.dep_lays[0] == 0:
depth_lay_data[0, :] = fvcom_data[this_t, 0, :]
for this_dep_lay_ind, this_dep_lay in enumerate(
self.regular_grid.dep_lays):
interp_data = self._Interpolater(
depth_lay_data[this_dep_lay_ind, :]).T
interp_data[self.regular_grid.mask[
this_t, :, :, this_dep_lay_ind]] = np.nan
reg_grid_data[this_t, :, :, this_dep_lay_ind] = interp_data
return reg_grid_data
def test_non_temporal_variable_with_dimension(self):
h = np.asarray([1.64808428, 12.75706577, 18.34670639, 24.29236031])
F = FileReader(self.stub.ncfile.name,
variables=['h'],
dims={'node': np.arange(4)})
test.assert_almost_equal(F.data.h, h)
