def test_supplying_attributes(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') attrs = { 'y': { '_CoordinateAxisType': 'Lat', '_FillValue': -9999.9, 'missing_value': -9999.9, } } with OrthogonalMultidimensionalTimeseries(self.single) as s: df = s.to_dataframe() with OrthogonalMultidimensionalTimeseries.from_dataframe( df, single_tmp, attributes=attrs) as result_ncd: assert 'station' in result_ncd.dimensions assert result_ncd.variables['y']._CoordinateAxisType == 'Lat' with self.assertRaises(AttributeError): result_ncd.variables['y'].missing_value with self.assertRaises(AttributeError): result_ncd.variables['y']._FillValue test_is_mine(OrthogonalMultidimensionalTimeseries, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_crt_dataframe_oot_A(self): axes = { 't': 'time', 'x': 'lon', 'y': 'lat', 'z': 'depth', 'sample': 'sample' } fid, tmpnc = tempfile.mkstemp(suffix='.nc') with ContiguousRaggedTrajectory(self.oot_A) as ncd: df = ncd.to_dataframe(axes=axes) df = df.sort_values(['trajectory', 'time']) attrs = get_calculated_attributes(df, axes=axes) with ContiguousRaggedTrajectory.from_dataframe( df, tmpnc, axes=axes, mode='a') as result_ncd: assert 'sample' in result_ncd.dimensions assert result_ncd.dimensions['sample'].size == 6610 assert 'trajectory' in result_ncd.dimensions # This is removing null trajectories that have no data. Not much to do about this # because there is no way to store this empty trajectory in a dataframe. assert result_ncd.dimensions['trajectory'].size == 507 result_ncd.apply_meta(attrs) test_is_mine(ContiguousRaggedTrajectory, tmpnc) # Try to load it again os.close(fid) os.remove(tmpnc)
def test_timeseries_omp_dataframe(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') with OrthogonalMultidimensionalTimeseries(self.single) as s: df = s.to_dataframe() with OrthogonalMultidimensionalTimeseries.from_dataframe( df, single_tmp) as result_ncd: assert 'station' in result_ncd.dimensions test_is_mine(OrthogonalMultidimensionalTimeseries, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_imp_dataframe(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') with IncompleteMultidimensionalProfile(self.multi) as ncd: df = ncd.to_dataframe() with IncompleteMultidimensionalProfile.from_dataframe( df, single_tmp) as result_ncd: assert 'profile' in result_ncd.dimensions test_is_mine(IncompleteMultidimensionalProfile, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_omp_dataframe_multi_unique_dims(self): CFDataset.load(self.multi) fid, multi_tmp = tempfile.mkstemp(suffix='.nc') with OrthogonalMultidimensionalProfile(self.multi) as ncd: df = ncd.to_dataframe() with self.assertRaises(NotImplementedError): with OrthogonalMultidimensionalProfile.from_dataframe(df, multi_tmp, unique_dims=True) as result_ncd: assert 'profile_dim' in result_ncd.dimensions test_is_mine(OrthogonalMultidimensionalProfile, multi_tmp) # Try to load it again os.close(fid) os.remove(multi_tmp)
def test_timeseries_omt_reduce_dims(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') with OrthogonalMultidimensionalTimeseries(self.single) as s: df = s.to_dataframe() with OrthogonalMultidimensionalTimeseries.from_dataframe( df, single_tmp, reduce_dims=True) as result_ncd: assert 'station' not in result_ncd.dimensions assert np.ma.allclose(result_ncd.variables['pH'][:].flatten(), self.ph) test_is_mine(OrthogonalMultidimensionalTimeseries, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_omtp_single_but_multi_file(self): filepath = os.path.join(os.path.dirname(__file__), 'resources', 'om-multi-format-but-one-station.nc') with OrthogonalMultidimensionalTimeseriesProfile(filepath) as ncd: fid, tmpfile = tempfile.mkstemp(suffix='.nc') df = ncd.to_dataframe(clean_rows=False) with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile) as result_ncd: assert 'station' in result_ncd.dimensions test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile, reduce_dims=True) as result_ncd: # Should remove the station dim since there is only one station assert 'station' not in result_ncd.dimensions test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile, unlimited=True) as result_ncd: assert 'station' in result_ncd.dimensions assert result_ncd.dimensions['t'].isunlimited() is True test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd: # Should remove the station dim since there is only one station assert 'station' not in result_ncd.dimensions assert result_ncd.dimensions['t'].isunlimited() is True test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again os.close(fid) os.remove(tmpfile)
def test_imt_multi_not_string(self): filepath = os.path.join(os.path.dirname(__file__), 'resources', 'im-multiple-nonstring.nc') CFDataset.load(filepath).close() with IncompleteMultidimensionalTrajectory(filepath) as ncd: fid, tmpfile = tempfile.mkstemp(suffix='.nc') df = ncd.to_dataframe(clean_rows=False) with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile) as result_ncd: assert 'trajectory' in result_ncd.dimensions test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, reduce_dims=True) as result_ncd: # Could not reduce dims since there was more than one trajectory assert 'trajectory' not in result_ncd.dimensions test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, unlimited=True) as result_ncd: assert result_ncd.dimensions['obs'].isunlimited() is True test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd: # Could not reduce dims since there was more than one trajectory assert 'trajectory' not in result_ncd.dimensions assert result_ncd.dimensions['obs'].isunlimited() is True test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again os.close(fid) os.remove(tmpfile)
def test_crt_dataframe_multiple_unique_dims(self): axes = { 't': 'time', 'x': 'lon', 'y': 'lat', 'z': 'z', } fid, tmpnc = tempfile.mkstemp(suffix='.nc') with ContiguousRaggedTrajectory(self.multi) as ncd: df = ncd.to_dataframe(axes=axes) with ContiguousRaggedTrajectory.from_dataframe( df, tmpnc, axes=axes, unique_dims=True) as result_ncd: assert 'trajectory_dim' in result_ncd.dimensions test_is_mine(ContiguousRaggedTrajectory, tmpnc) # Try to load it again os.close(fid) os.remove(tmpnc)
def test_imt_change_axis_names(self): new_axis = {'t': 'time', 'x': 'lon', 'y': 'lat', 'z': 'depth'} filepath = os.path.join(os.path.dirname(__file__), 'resources', 'im-multiple.nc') with IncompleteMultidimensionalTrajectory(filepath) as ncd: fid, tmpfile = tempfile.mkstemp(suffix='.nc') df = ncd.to_dataframe(clean_rows=False, axes=new_axis) with IncompleteMultidimensionalTrajectory.from_dataframe( df, tmpfile, axes=new_axis) as result_ncd: assert 'trajectory' in result_ncd.dimensions assert 'time' in result_ncd.variables assert 'lon' in result_ncd.variables assert 'lat' in result_ncd.variables assert 'depth' in result_ncd.variables test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
def test_crtp_dataframe_missing_time(self): axes = { 't': 'precise_time', 'x': 'precise_lon', 'y': 'precise_lat', 'z': 'depth', } fid, tmpnc = tempfile.mkstemp(suffix='.nc') with ContiguousRaggedTrajectoryProfile(self.missing_time) as ncd: df = ncd.to_dataframe(axes=axes) with ContiguousRaggedTrajectoryProfile.from_dataframe( df, tmpnc, axes=axes) as result_ncd: assert 'profile' in result_ncd.dimensions assert 'trajectory' in result_ncd.dimensions test_is_mine(ContiguousRaggedTrajectoryProfile, tmpnc) # Try to load it again os.close(fid) os.remove(tmpnc)
def test_timeseries_omt_dataframe_multi(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') with OrthogonalMultidimensionalTimeseries(self.multi) as s: df = s.to_dataframe() with OrthogonalMultidimensionalTimeseries.from_dataframe( df, single_tmp) as result_ncd: assert 'station' in result_ncd.dimensions assert np.ma.allclose( result_ncd.variables['temperature'][0, 0:7].flatten(), [ 18.61804, 13.2165, 39.30018, 17.00865, 24.95154, 35.99525, 24.33436 ], ) test_is_mine(OrthogonalMultidimensionalTimeseries, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_timeseries_omt_no_z(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') with OrthogonalMultidimensionalTimeseries(self.single) as s: df = s.to_dataframe() axes = {'z': None} df.drop(columns=['z'], inplace=True) with OrthogonalMultidimensionalTimeseries.from_dataframe( df, single_tmp, axes=axes, ) as result_ncd: assert 'station' in result_ncd.dimensions assert 'z' not in result_ncd.variables assert np.ma.allclose(result_ncd.variables['pH'][:].flatten(), self.ph) test_is_mine(OrthogonalMultidimensionalTimeseries, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_write_nc(self): fid, single_tmp = tempfile.mkstemp(suffix='.nc') axes = { 't': 'time', 'x': 'longitude', 'y': 'latitude', 'z': 'z', 'profile': 'stationid' } with IncompleteMultidimensionalProfile.from_dataframe(self.df, single_tmp, axes=axes, mode='a') as ncd: ncd.renameDimension('stationid', 'profile') test_is_mine(IncompleteMultidimensionalProfile, single_tmp) # Try to load it again os.close(fid) os.remove(single_tmp)
def test_omtp_change_axis_names(self): filepath = os.path.join(os.path.dirname(__file__), 'resources', 'om-multiple.nc') new_axis = {'t': 'time', 'x': 'lon', 'y': 'lat', 'z': 'depth'} with OrthogonalMultidimensionalTimeseriesProfile(filepath) as ncd: fid, tmpfile = tempfile.mkstemp(suffix='.nc') df = ncd.to_dataframe(clean_rows=False, axes=new_axis) with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe( df, tmpfile, axes=new_axis) as result_ncd: assert 'station' in result_ncd.dimensions assert 'time' in result_ncd.variables assert 'lon' in result_ncd.variables assert 'lat' in result_ncd.variables assert 'depth' in result_ncd.variables test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again os.close(fid) os.remove(tmpfile)
def test_omtp_multi(self): filepath = os.path.join(os.path.dirname(__file__), 'resources', 'om-multiple.nc') with OrthogonalMultidimensionalTimeseriesProfile(filepath) as ncd: fid, tmpfile = tempfile.mkstemp(suffix='.nc') df = ncd.to_dataframe(clean_rows=False) with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe( df, tmpfile) as result_ncd: assert 'station' in result_ncd.dimensions test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe( df, tmpfile, reduce_dims=True) as result_ncd: # Even though we pass reduce_dims, there are two stations so it is not reduced assert 'station' in result_ncd.dimensions test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe( df, tmpfile, unlimited=True) as result_ncd: assert 'station' in result_ncd.dimensions assert result_ncd.dimensions['t'].isunlimited() is True test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe( df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd: # Even though we pass reduce_dims, there are two stations so it is not reduced assert 'station' in result_ncd.dimensions assert result_ncd.dimensions['t'].isunlimited() is True test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again os.close(fid) os.remove(tmpfile)
def test_rtp_single(self): filepath = os.path.join(os.path.dirname(__file__), 'resources', 'r-ctd-single.nc') with RaggedTimeseriesProfile(filepath) as ncd: fid, tmpfile = tempfile.mkstemp(suffix='.nc') df = ncd.to_dataframe(clean_rows=False) with RaggedTimeseriesProfile.from_dataframe(df, tmpfile) as result_ncd: assert 'station' in result_ncd.dimensions test_is_mine(RaggedTimeseriesProfile, tmpfile) # Try to load it again with RaggedTimeseriesProfile.from_dataframe( df, tmpfile, unique_dims=True) as result_ncd: assert 'station_dim' in result_ncd.dimensions test_is_mine(RaggedTimeseriesProfile, tmpfile) # Try to load it again with RaggedTimeseriesProfile.from_dataframe( df, tmpfile, reduce_dims=True) as result_ncd: # Even though we pass reduce_dims, there are two stations so it is not reduced assert 'station' not in result_ncd.dimensions assert 'profile' in result_ncd.dimensions test_is_mine(RaggedTimeseriesProfile, tmpfile) # Try to load it again with RaggedTimeseriesProfile.from_dataframe( df, tmpfile, unlimited=True) as result_ncd: assert 'station' in result_ncd.dimensions assert 'profile' in result_ncd.dimensions assert result_ncd.dimensions['obs'].isunlimited() is True test_is_mine(RaggedTimeseriesProfile, tmpfile) # Try to load it again with RaggedTimeseriesProfile.from_dataframe( df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd: assert 'station' not in result_ncd.dimensions assert 'profile' in result_ncd.dimensions assert result_ncd.dimensions['obs'].isunlimited() is True test_is_mine(RaggedTimeseriesProfile, tmpfile) # Try to load it again with RaggedTimeseriesProfile.from_dataframe( df, tmpfile, unique_dims=True, reduce_dims=False, unlimited=True) as result_ncd: assert 'station_dim' in result_ncd.dimensions assert 'profile_dim' in result_ncd.dimensions assert result_ncd.dimensions['obs_dim'].isunlimited() is True test_is_mine(RaggedTimeseriesProfile, tmpfile) # Try to load it again os.close(fid) os.remove(tmpfile)
def test_crt_load(fp): test_is_mine(ContiguousRaggedTrajectory, fp)