def test_get_time_with_tolerance(self): time_dims = [datetime.strptime('2001-02-12 09:00:00.00000', '%Y-%m-%d %H:%M:%S.%f'), datetime.strptime('2001-02-12 09:14:02.00000', '%Y-%m-%d %H:%M:%S.%f')] returned_indices = [None, 26] F = FileReader(self.stub.ncfile.name) file_indices = [F.time_to_index(i, tolerance=10) for i in time_dims] test.assert_equal(file_indices, returned_indices)
def test_get_negative_time_series(self): F1 = FileReader(self.stub.ncfile.name, dims={'node': [10]}, variables=['zeta']) F2 = FileReader(self.stub.ncfile.name, dims={ 'node': [10], 'time': -np.arange(10, 40) }, variables=['zeta']) test.assert_almost_equal(F2.data.zeta, F1.data.zeta[-np.arange(10, 40)], decimal=5)
def test_get_layer(self): vertical_velocity = np.array([ 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815, 0.01509815 ]) F = FileReader(self.stub.ncfile.name, dims={ 'siglay': [5], 'time': [100, 101] }, variables=['ww']) test.assert_almost_equal(np.squeeze(F.data.ww), vertical_velocity, decimal=5)
def test_get_time_with_datetime(self): time_dims = [datetime.strptime('2001-02-12 09:00:00.00000', '%Y-%m-%d %H:%M:%S.%f'), datetime.strptime('2001-02-14 12:00:00.00000', '%Y-%m-%d %H:%M:%S.%f')] returned_indices = [26, 76] F = FileReader(self.stub.ncfile.name, dims={'time': time_dims}) test.assert_equal(F._dims['time'][0], returned_indices[0]) test.assert_equal(F._dims['time'][-1], returned_indices[-1])
def test_add_files(self): # Make another stub file which follows in time from the existing one. Then only load a section of that in # time and make sure the results are the same as if we'd loaded them manually and added them together. next_stub = StubFile(self.endtime, self.endtime + relativedelta(months=1), self.interval, lon=self.lon, lat=self.lat, triangles=self.triangles, zone='30N') # Append the new stub file to the old one. F1 = FileReader(self.stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww']) F2 = FileReader(next_stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww']) all_times = np.concatenate((F1.time.datetime[:], F2.time.datetime[:]), axis=0) all_data = np.concatenate((F1.data.ww[:], F2.data.ww[:]), axis=0) # Repeat the process, but use the __add__ method in FileReader. F1 = FileReader(self.stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww']) F2 = FileReader(next_stub.ncfile.name, dims={'siglay': [5], 'time': [0, -10]}, variables=['ww']) F = F2 >> F1 test.assert_equal(F.time.datetime, all_times) test.assert_equal(F.data.ww, all_data)
def test_get_water_column(self): F = FileReader(self.stub.ncfile.name, dims={ 'node': [5], 'time': 10 }, variables=['temp']) test.assert_almost_equal(np.squeeze(F.data.temp), self.reference.data.temp[10, :, 5], decimal=5)
def test_get_layer(self): F = FileReader(self.stub.ncfile.name, dims={ 'siglay': [5], 'time': [100] }, variables=['ww']) test.assert_almost_equal(np.squeeze(F.data.ww), self.reference.data.ww[100, 5, :], decimal=5)
def test_get_single_time_negative_index(self): F = FileReader(self.stub.ncfile.name, dims={ 'node': [10], 'time': [-10] }, variables=['zeta']) test.assert_almost_equal(np.squeeze(F.data.zeta), self.reference.data.zeta[-10, 10], decimal=5)
def test_get_time_series(self): F = FileReader(self.stub.ncfile.name, dims={ 'node': [10], 'time': np.arange(10, 40) }, variables=['zeta']) test.assert_almost_equal(np.squeeze(F.data.zeta), self.reference.data.zeta[10:40, 10], decimal=5)
def test_get_single_time_negative_index(self): surface_elevation = np.array([[0.22058453]]) F = FileReader(self.stub.ncfile.name, dims={ 'node': [10], 'time': [-10, -9] }, variables=['zeta']) test.assert_almost_equal(np.squeeze(F.data.zeta), surface_elevation, decimal=5)
def test_get_single_time(self): surface_elevation = np.array([-1.41013891]) F = FileReader(self.stub.ncfile.name, dims={ 'node': [10], 'time': [10, 11] }, variables=['zeta']) test.assert_almost_equal(np.squeeze(F.data.zeta), surface_elevation, decimal=5)
def test_get_bounding_box(self): wesn = [-5, -3, 50, 55] extents = [ -4.9847326278686523, -3.0939722061157227, 50.19110107421875, 54.946651458740234 ] F = FileReader(self.stub.ncfile.name, dims={'wesn': wesn}) test.assert_equal(F.grid.lon.min(), extents[0]) test.assert_equal(F.grid.lon.max(), extents[1]) test.assert_equal(F.grid.lat.min(), extents[2]) test.assert_equal(F.grid.lat.max(), extents[3])
def setUp(self): self.starttime, self.endtime, self.interval, self.lon, self.lat, self.triangles = _prep( ) self.stub = StubFile(self.starttime, self.endtime, self.interval, lon=self.lon, lat=self.lat, triangles=self.triangles, zone='30N') self.reference = FileReader(self.stub.ncfile.name, variables=['ww', 'zeta', 'temp', 'h'])
def retrieve_file_data(self): where_str = 'buoy_name in (' for this_buoy in self.buoy_list: where_str += this_buoy + ',' where_str = where_str[0:-1] + ')' buoy_lat_lons = self.wco_database.select_query('sites', where_str, 'buoy_name, lon, lat') first_file = True model_all_dicts = {} for this_file in self.file_list: if first_file: fvcom_data_reader = FileReader(this_file, ['temp', 'salinity']) close_node = [] for this_buoy_ll in buoy_lat_lons: close_node.append() model_depths = fvcom_data_reader.grid.siglay * fvcom_data_reader.grid.h * -1 for this_buoy in self.buoy_list: model_all_dicts[this_buoy] = { 'dt_time': mod_times, 'temp': mod_t_vals, 'salinity': mod_s_vals } first_file = False else: fvcom_data_reader = FileReader(this_file, ['temp', 'salinity'], dims={'node': [close_node]}) for this_buoy in self.buoy_list: model_all_dicts model_depths = fvcom_data_reader.grid.siglay * fvcom_data_reader.grid.h * -1 for this_buoy in self.buoy_list: model_all_dicts[this_buoy]['depth'] = model_dp for this_buoy in self.buoy_list: self.model_data[this_buoy] = model_all_dicts[this_buoy]
def test_get_water_column(self): water_column = np.array([ 9.99821472, 10.90714264, 11.81607151, 12.72500038, 13.6339283, 14.54285717, 15.45178604, 16.36071396, 17.26964378, 18.1785717 ]) F = FileReader(self.stub.ncfile.name, dims={ 'node': [5], 'time': [10, 11] }, variables=['temp']) test.assert_almost_equal(np.squeeze(F.data.temp), water_column, decimal=5)
def setUp(self): self.starttime, self.endtime, self.interval, self.lon, self.lat, self.triangles = _prep( ) self.model = StubFile(self.starttime, self.endtime, self.interval, lon=self.lon, lat=self.lat, triangles=self.triangles, zone='30N') self.fvcom = FileReader(self.model.ncfile.name, variables=['u', 'v', 'ua', 'va']) self.fvcom.grid.x, self.fvcom.grid.y, _ = utm_from_lonlat( self.fvcom.grid.lon, self.fvcom.grid.lat, zone='30N') self.fvcom.grid.xc, self.fvcom.grid.yc, _ = utm_from_lonlat( self.fvcom.grid.lonc, self.fvcom.grid.latc, zone='30N')
def test_get_time_series(self): surface_elevation = np.array([ -1.41013891, -0.98554639, -0.3139296, 0.4363727, 1.07729949, 1.44820437, 1.45612114, 1.09906549, 0.46653234, -0.28293574, -0.96148683, -1.3990441, -1.48593514, -1.20038097, -0.61395488, 0.12635717, 0.83499817, 1.33434937, 1.49924987, 1.28836787, 0.75456029, 0.03162443, -0.699238, -1.25483851, -1.49591749, -1.3620492, -0.88678734, -0.18925488, 0.55571376, 1.1613944 ]) F = FileReader(self.stub.ncfile.name, dims={ 'node': [10], 'time': [10, 40] }, variables=['zeta']) test.assert_almost_equal(np.squeeze(F.data.zeta), surface_elevation, decimal=5)
def test_non_temporal_variable(self): h = np.asarray([ 1.64808428, 12.75706577, 18.34670639, 24.29236031, 29.7772541, 25.00211716, 22.69193077, 18.70510674, 21.96312141, 27.35856438, 35.32657623, 32.48567581, 38.93023682, 43.63704681, 51.21723175, 53.23581314, 59.78393555, 55.53053284, 52.84440994, 57.36302185, 62.2620163, 66.50558472, 61.24137878, 60.91600418, 67.42472839, 73.38938904, 70.63117981, 70.62969208, 75.18034363, 79.09741974, 84.4043808, 81.0752182, 88.22835541, 90.34424591, 97.57055664, 98.27231598, 100.0000000, 96.82516479, 91.1933136, 88.29994202, 89.59196472, 91.40013885, 85.90748596, 79.28456879, 74.37998199, 70.46596527, 70.78884888, 70.06604004, 63.42258453, 63.06575394, 59.99647141, 57.27880096, 55.11286545, 61.5132103, 62.31158066, 59.2288208, 53.60129929, 50.73873138, 56.42451477, 52.42653656, 44.78648376, 39.55376434, 32.51250839, 28.38024521, 20.91413689, 18.19268227, 11.62014961, 7.51470757, 38.44644928, 45.77177048, 34.9041214, 51.38194275, 77.87741852, 81.04411316 ]) F = FileReader(self.stub.ncfile.name, variables=['h']) test.assert_almost_equal(F.data.h, h)
def InitialiseGrid(self, lower_left_ll, upper_right_ll, grid_res, depth_layers, time_varying_depth=True): """ Sets up the grids to interpolate to and from. Part of the work is reducing the domain, then comes the trick step of interpolating onto vertical layers which vary in time and may create discontiguous areas of the model domain. Parameters ---------- lower_left_ll : 1d array A two entry array of the lon/lat of the lower left corner of the regular grid to be interpolated onto upper_right_ll : 1d array The corresponding lon/lat of the upper right corner grid_res : float The resolution in degrees of the regular grid depth layers : array Array of the depths to interpolate onto. These are positive down starting from the top of the free surface or the zero mean depending on the setting of time_varying_depth """ if self._noisy: print('Rank {}: Loading grid'.format(self.rank)) # Check ther regular grid limits fit with the grid resolution specified if not round((upper_right_ll[0] - lower_left_ll[0]) % grid_res, 10) % grid_res == 0: print( 'Longitudes not divisible by grid resolution, extending grid') upper_right_ll[0] = round( np.ceil((upper_right_ll[0] - lower_left_ll[0]) / grid_res) * grid_res, 10) + lower_left_ll[0] if not round((upper_right_ll[1] - lower_left_ll[1]) % grid_res, 10) % grid_res == 0: print('Latitudes not divisible by grid resolution, extending grid') upper_right_ll[1] = round( np.ceil((upper_right_ll[1] - lower_left_ll[1]) / grid_res) * grid_res, 10) + lower_left_ll[1] self.regular_grid.lons = np.linspace( lower_left_ll[0], upper_right_ll[0], (upper_right_ll[0] - lower_left_ll[0]) / grid_res + 1) self.regular_grid.lats = np.linspace( lower_left_ll[1], upper_right_ll[1], (upper_right_ll[1] - lower_left_ll[1]) / grid_res + 1) self.regular_grid.mesh_lons, self.regular_grid.mesh_lats = np.meshgrid( self.regular_grid.lons, self.regular_grid.lats) self.regular_grid.dep_lays = np.asarray(depth_layers) # And load the fvcom grid, reducing to the interpolation area only fvcom_grid_fr = FileReader(self.fvcom_file, ['zeta'], dims={'time': self.time_indices}) fvcom_nodes_reduce = np.logical_and( np.logical_and(fvcom_grid_fr.grid.lon >= lower_left_ll[0], fvcom_grid_fr.grid.lon <= upper_right_ll[0]), np.logical_and(fvcom_grid_fr.grid.lat >= lower_left_ll[1], fvcom_grid_fr.grid.lat <= upper_right_ll[1])) fvcom_nodes_reduce = np.squeeze(np.argwhere(fvcom_nodes_reduce)) ## extend the node list to include attached nodes outside the area, this should make the interpolation better fvcom_nodes = np.isin(fvcom_grid_fr.grid.triangles, fvcom_nodes_reduce) self.fvcom_grid.nodes = np.unique(fvcom_grid_fr.grid.triangles[ np.asarray(np.sum(fvcom_nodes, axis=1), dtype=bool), :]) self.fvcom_grid.triangles, self.fvcom_grid.elements = reduce_triangulation( fvcom_grid_fr.grid.triangles, self.fvcom_grid.nodes, return_elements=True) self.fvcom_grid.ll = np.asarray([ fvcom_grid_fr.grid.lon[self.fvcom_grid.nodes], fvcom_grid_fr.grid.lat[self.fvcom_grid.nodes] ]).T self.fvcom_grid.elements_ll = np.asarray([ fvcom_grid_fr.grid.lonc[self.fvcom_grid.elements], fvcom_grid_fr.grid.latc[self.fvcom_grid.elements] ]).T self.regular_grid.initial_mask = mask_to_fvcom_meshgrid( self.fvcom_grid.ll, self.fvcom_grid.triangles, self.regular_grid.mesh_lons, self.regular_grid.mesh_lats) self.fvcom_grid.h = fvcom_grid_fr.grid.h[self.fvcom_grid.nodes] self.fvcom_grid.zeta = fvcom_grid_fr.data.zeta[:, self.fvcom_grid.nodes] self.fvcom_grid.zeta_center = node_to_centre( fvcom_grid_fr.data.zeta, fvcom_grid_fr)[:, self.fvcom_grid.elements] self.fvcom_grid.sigma = fvcom_grid_fr.grid.siglay[:, self.fvcom_grid. nodes] self.fvcom_grid.h_center = fvcom_grid_fr.grid.h_center[ self.fvcom_grid.elements] self.fvcom_grid.sigma_center = fvcom_grid_fr.grid.siglay_center[:, self. fvcom_grid . elements] if time_varying_depth: if self._noisy: print('Rank {}: Calculating time varying mask'.format( self.rank)) fvcom_dep_lays = -unstructured_grid_depths( self.fvcom_grid.h, self.fvcom_grid.zeta, self.fvcom_grid.sigma) ## change to depth from free surface since I *think* this is what cmems does? self.fvcom_grid.dep_lays = fvcom_dep_lays - np.tile( np.min(fvcom_dep_lays, axis=1)[:, np.newaxis, :], [1, fvcom_dep_lays.shape[1], 1]) self.fvcom_grid.total_depth = np.max(self.fvcom_grid.dep_lays, axis=1) self.regular_grid.mask = np.ones([ len(self.time_indices), len(self.regular_grid.lons), len(self.regular_grid.lats), len(self.regular_grid.dep_lays) ], dtype=bool) for this_t in np.arange(0, len(self.time_indices)): # retriangulate for each depth layer, can be multiple if there are split regions and interpolate for this_depth_lay_ind, this_depth_lay in enumerate( self.regular_grid.dep_lays): if self._noisy: print('Rank {}: Time step {}, Depth {}'.format( self.rank, this_t, this_depth_lay_ind)) this_depth_layer_nodes = np.where( self.fvcom_grid.total_depth[ this_t, :] >= this_depth_lay)[0] if this_depth_layer_nodes.size: this_depth_tri = reduce_triangulation( self.fvcom_grid.triangles, this_depth_layer_nodes) this_td_mask = np.ones( self.regular_grid.mesh_lons.shape, dtype=bool) this_td_mask[ ~self.regular_grid.initial_mask] = mask_to_fvcom( self.fvcom_grid.ll[this_depth_layer_nodes], this_depth_tri, self.regular_grid. mesh_lons[~self.regular_grid.initial_mask], self.regular_grid. mesh_lats[~self.regular_grid.initial_mask], split_domain_check=True).T self.regular_grid.mask[ this_t, :, :, this_depth_lay_ind] = this_td_mask.T else: print('Not implemented yet')
def test_get_multipe_latc(self): latc = np.array((52.8649059, 52.90310308)) F = FileReader(self.stub.ncfile.name, dims={'nele': [29, 34]}) test.assert_almost_equal(F.grid.latc, latc, decimal=5)
def test_get_time_with_string(self): time_dims = ['2001-02-12 09:00:00.00000', '2001-02-14 12:00:00.00000'] returned_indices = [26, 78] F = FileReader(self.stub.ncfile.name, dims={'time': time_dims}) test.assert_equal(F._dims['time'], returned_indices)
def test_get_multipe_lat(self): lat = np.array((56.89470906, 58.49899088)) F = FileReader(self.stub.ncfile.name, dims={'node': [29, 34]}) test.assert_almost_equal(F.grid.lat, lat, decimal=5)
def test_get_multipe_lonc(self): lonc = np.array((-4.67915533, 0.61115498)) F = FileReader(self.stub.ncfile.name, dims={'nele': [0, 5]}) test.assert_almost_equal(F.grid.lonc, lonc, decimal=5)
def test_get_multipe_lon(self): lon = np.array((-5.78687373, -3.26585943)) F = FileReader(self.stub.ncfile.name, dims={'node': [0, 5]}) test.assert_almost_equal(F.grid.lon, lon, decimal=5)
def test_get_single_latc(self): latc = 52.864905897403958 F = FileReader(self.stub.ncfile.name, dims={'nele': [29]}) test.assert_almost_equal(F.grid.latc, latc, decimal=5)
def test_get_single_lonc(self): lonc = -4.67915533 F = FileReader(self.stub.ncfile.name, dims={'nele': [0]}) test.assert_almost_equal(F.grid.lonc, lonc, decimal=5)
def test_get_single_lat(self): lat = 56.89470906 F = FileReader(self.stub.ncfile.name, dims={'node': [29]}) test.assert_almost_equal(F.grid.lat, lat, decimal=5)
def test_get_single_lon(self): lon = -5.78687373 F = FileReader(self.stub.ncfile.name, dims={'node': [0]}) test.assert_almost_equal(F.grid.lon, lon, decimal=5)
def InterpolateRegular(self, variable, mode='nodes'): """ Actually do the interpolation """ if mode in ['nodes', 'surface']: self.fvcom_grid.select_points = self.fvcom_grid.nodes self.fvcom_grid.select_ll = self.fvcom_grid.ll else: self.fvcom_grid.select_points = self.fvcom_grid.elements self.fvcom_grid.select_ll = self.fvcom_grid.elements_ll if mode in ['nodes', 'surface']: self.fvcom_grid.select_dep_lays = self.fvcom_grid.dep_lays else: fvcom_dep_lays = -unstructured_grid_depths( self.fvcom_grid.h_center, self.fvcom_grid.zeta_center, self.fvcom_grid.sigma_center) ## change to depth from free surface since I *think* this is what cmems does? self.fvcom_grid.select_dep_lays = fvcom_dep_lays - np.tile( np.min(fvcom_dep_lays, axis=1)[:, np.newaxis, :], [1, fvcom_dep_lays.shape[1], 1]) if mode == 'surface': reg_grid_data = np.zeros([ len(self.time_indices), len(self.regular_grid.lons), len(self.regular_grid.lats) ]) else: reg_grid_data = np.zeros([ len(self.time_indices), len(self.regular_grid.lons), len(self.regular_grid.lats), len(self.regular_grid.dep_lays) ]) reg_grid_data[:] = np.nan fvcom_data = getattr( FileReader(self.fvcom_file, [variable], dims={ 'time': self.time_indices }).data, variable)[..., self.fvcom_grid.select_points] for this_t in np.arange(0, len(self.time_indices)): if self._noisy: print('Rank {}: Interpolating time step {} for {}'.format( self.rank, this_t, variable)) if mode == 'surface': depth_lay_data = fvcom_data[this_t, :] interp_data = self._Interpolater(depth_lay_data).T interp_data[self.regular_grid.mask[this_t, :, :, 0]] = np.nan reg_grid_data[this_t, :, :] = interp_data else: depth_lay_data = np.zeros([ len(self.regular_grid.dep_lays), len(self.fvcom_grid.select_points) ]) for i in np.arange(0, len(self.fvcom_grid.select_points)): depth_lay_data[:, i] = np.interp( self.regular_grid.dep_lays, self.fvcom_grid.select_dep_lays[this_t, :, i], fvcom_data[this_t, :, i], left=np.nan, right=np.nan) # Replace surface data as it can't interpolate properly there if self.regular_grid.dep_lays[0] == 0: depth_lay_data[0, :] = fvcom_data[this_t, 0, :] for this_dep_lay_ind, this_dep_lay in enumerate( self.regular_grid.dep_lays): interp_data = self._Interpolater( depth_lay_data[this_dep_lay_ind, :]).T interp_data[self.regular_grid.mask[ this_t, :, :, this_dep_lay_ind]] = np.nan reg_grid_data[this_t, :, :, this_dep_lay_ind] = interp_data return reg_grid_data
def test_non_temporal_variable_with_dimension(self): h = np.asarray([1.64808428, 12.75706577, 18.34670639, 24.29236031]) F = FileReader(self.stub.ncfile.name, variables=['h'], dims={'node': np.arange(4)}) test.assert_almost_equal(F.data.h, h)