def test_supplying_attributes(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
attrs = {
'y': {
'_CoordinateAxisType': 'Lat',
'_FillValue': -9999.9,
'missing_value': -9999.9,
}
}
with OrthogonalMultidimensionalTimeseries(self.single) as s:
df = s.to_dataframe()
with OrthogonalMultidimensionalTimeseries.from_dataframe(
df, single_tmp, attributes=attrs) as result_ncd:
assert 'station' in result_ncd.dimensions
assert result_ncd.variables['y']._CoordinateAxisType == 'Lat'
with self.assertRaises(AttributeError):
result_ncd.variables['y'].missing_value
with self.assertRaises(AttributeError):
result_ncd.variables['y']._FillValue
test_is_mine(OrthogonalMultidimensionalTimeseries,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_crt_dataframe_oot_A(self):
axes = {
't': 'time',
'x': 'lon',
'y': 'lat',
'z': 'depth',
'sample': 'sample'
}
fid, tmpnc = tempfile.mkstemp(suffix='.nc')
with ContiguousRaggedTrajectory(self.oot_A) as ncd:
df = ncd.to_dataframe(axes=axes)
df = df.sort_values(['trajectory', 'time'])
attrs = get_calculated_attributes(df, axes=axes)
with ContiguousRaggedTrajectory.from_dataframe(
df, tmpnc, axes=axes, mode='a') as result_ncd:
assert 'sample' in result_ncd.dimensions
assert result_ncd.dimensions['sample'].size == 6610
assert 'trajectory' in result_ncd.dimensions
# This is removing null trajectories that have no data. Not much to do about this
# because there is no way to store this empty trajectory in a dataframe.
assert result_ncd.dimensions['trajectory'].size == 507
result_ncd.apply_meta(attrs)
test_is_mine(ContiguousRaggedTrajectory,
tmpnc) # Try to load it again
os.close(fid)
os.remove(tmpnc)
def test_timeseries_omp_dataframe(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
with OrthogonalMultidimensionalTimeseries(self.single) as s:
df = s.to_dataframe()
with OrthogonalMultidimensionalTimeseries.from_dataframe(
df, single_tmp) as result_ncd:
assert 'station' in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalTimeseries,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_imp_dataframe(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
with IncompleteMultidimensionalProfile(self.multi) as ncd:
df = ncd.to_dataframe()
with IncompleteMultidimensionalProfile.from_dataframe(
df, single_tmp) as result_ncd:
assert 'profile' in result_ncd.dimensions
test_is_mine(IncompleteMultidimensionalProfile,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_omp_dataframe_multi_unique_dims(self):
CFDataset.load(self.multi)
fid, multi_tmp = tempfile.mkstemp(suffix='.nc')
with OrthogonalMultidimensionalProfile(self.multi) as ncd:
df = ncd.to_dataframe()
with self.assertRaises(NotImplementedError):
with OrthogonalMultidimensionalProfile.from_dataframe(df, multi_tmp, unique_dims=True) as result_ncd:
assert 'profile_dim' in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalProfile, multi_tmp) # Try to load it again
os.close(fid)
os.remove(multi_tmp)
def test_timeseries_omt_reduce_dims(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
with OrthogonalMultidimensionalTimeseries(self.single) as s:
df = s.to_dataframe()
with OrthogonalMultidimensionalTimeseries.from_dataframe(
df, single_tmp, reduce_dims=True) as result_ncd:
assert 'station' not in result_ncd.dimensions
assert np.ma.allclose(result_ncd.variables['pH'][:].flatten(),
self.ph)
test_is_mine(OrthogonalMultidimensionalTimeseries,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_omtp_single_but_multi_file(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources', 'om-multi-format-but-one-station.nc')
with OrthogonalMultidimensionalTimeseriesProfile(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False)
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile) as result_ncd:
assert 'station' in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile, reduce_dims=True) as result_ncd:
# Should remove the station dim since there is only one station
assert 'station' not in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile, unlimited=True) as result_ncd:
assert 'station' in result_ncd.dimensions
assert result_ncd.dimensions['t'].isunlimited() is True
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd:
# Should remove the station dim since there is only one station
assert 'station' not in result_ncd.dimensions
assert result_ncd.dimensions['t'].isunlimited() is True
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile, tmpfile) # Try to load it again
os.close(fid)
os.remove(tmpfile)
def test_imt_multi_not_string(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources', 'im-multiple-nonstring.nc')
CFDataset.load(filepath).close()
with IncompleteMultidimensionalTrajectory(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False)
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile) as result_ncd:
assert 'trajectory' in result_ncd.dimensions
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, reduce_dims=True) as result_ncd:
# Could not reduce dims since there was more than one trajectory
assert 'trajectory' not in result_ncd.dimensions
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, unlimited=True) as result_ncd:
assert result_ncd.dimensions['obs'].isunlimited() is True
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
with IncompleteMultidimensionalTrajectory.from_dataframe(df, tmpfile, reduce_dims=True, unlimited=True) as result_ncd:
# Could not reduce dims since there was more than one trajectory
assert 'trajectory' not in result_ncd.dimensions
assert result_ncd.dimensions['obs'].isunlimited() is True
test_is_mine(IncompleteMultidimensionalTrajectory, tmpfile) # Try to load it again
os.close(fid)
os.remove(tmpfile)
def test_crt_dataframe_multiple_unique_dims(self):
axes = {
't': 'time',
'x': 'lon',
'y': 'lat',
'z': 'z',
}
fid, tmpnc = tempfile.mkstemp(suffix='.nc')
with ContiguousRaggedTrajectory(self.multi) as ncd:
df = ncd.to_dataframe(axes=axes)
with ContiguousRaggedTrajectory.from_dataframe(
df, tmpnc, axes=axes, unique_dims=True) as result_ncd:
assert 'trajectory_dim' in result_ncd.dimensions
test_is_mine(ContiguousRaggedTrajectory,
tmpnc) # Try to load it again
os.close(fid)
os.remove(tmpnc)
def test_imt_change_axis_names(self):
new_axis = {'t': 'time', 'x': 'lon', 'y': 'lat', 'z': 'depth'}
filepath = os.path.join(os.path.dirname(__file__), 'resources',
'im-multiple.nc')
with IncompleteMultidimensionalTrajectory(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False, axes=new_axis)
with IncompleteMultidimensionalTrajectory.from_dataframe(
df, tmpfile, axes=new_axis) as result_ncd:
assert 'trajectory' in result_ncd.dimensions
assert 'time' in result_ncd.variables
assert 'lon' in result_ncd.variables
assert 'lat' in result_ncd.variables
assert 'depth' in result_ncd.variables
test_is_mine(IncompleteMultidimensionalTrajectory,
tmpfile) # Try to load it again
def test_crtp_dataframe_missing_time(self):
axes = {
't': 'precise_time',
'x': 'precise_lon',
'y': 'precise_lat',
'z': 'depth',
}
fid, tmpnc = tempfile.mkstemp(suffix='.nc')
with ContiguousRaggedTrajectoryProfile(self.missing_time) as ncd:
df = ncd.to_dataframe(axes=axes)
with ContiguousRaggedTrajectoryProfile.from_dataframe(
df, tmpnc, axes=axes) as result_ncd:
assert 'profile' in result_ncd.dimensions
assert 'trajectory' in result_ncd.dimensions
test_is_mine(ContiguousRaggedTrajectoryProfile,
tmpnc) # Try to load it again
os.close(fid)
os.remove(tmpnc)
def test_timeseries_omt_dataframe_multi(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
with OrthogonalMultidimensionalTimeseries(self.multi) as s:
df = s.to_dataframe()
with OrthogonalMultidimensionalTimeseries.from_dataframe(
df, single_tmp) as result_ncd:
assert 'station' in result_ncd.dimensions
assert np.ma.allclose(
result_ncd.variables['temperature'][0, 0:7].flatten(),
[
18.61804, 13.2165, 39.30018, 17.00865, 24.95154,
35.99525, 24.33436
],
)
test_is_mine(OrthogonalMultidimensionalTimeseries,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_timeseries_omt_no_z(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
with OrthogonalMultidimensionalTimeseries(self.single) as s:
df = s.to_dataframe()
axes = {'z': None}
df.drop(columns=['z'], inplace=True)
with OrthogonalMultidimensionalTimeseries.from_dataframe(
df,
single_tmp,
axes=axes,
) as result_ncd:
assert 'station' in result_ncd.dimensions
assert 'z' not in result_ncd.variables
assert np.ma.allclose(result_ncd.variables['pH'][:].flatten(),
self.ph)
test_is_mine(OrthogonalMultidimensionalTimeseries,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_write_nc(self):
fid, single_tmp = tempfile.mkstemp(suffix='.nc')
axes = {
't': 'time',
'x': 'longitude',
'y': 'latitude',
'z': 'z',
'profile': 'stationid'
}
with IncompleteMultidimensionalProfile.from_dataframe(self.df,
single_tmp,
axes=axes,
mode='a') as ncd:
ncd.renameDimension('stationid', 'profile')
test_is_mine(IncompleteMultidimensionalProfile,
single_tmp) # Try to load it again
os.close(fid)
os.remove(single_tmp)
def test_omtp_change_axis_names(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources',
'om-multiple.nc')
new_axis = {'t': 'time', 'x': 'lon', 'y': 'lat', 'z': 'depth'}
with OrthogonalMultidimensionalTimeseriesProfile(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False, axes=new_axis)
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(
df, tmpfile, axes=new_axis) as result_ncd:
assert 'station' in result_ncd.dimensions
assert 'time' in result_ncd.variables
assert 'lon' in result_ncd.variables
assert 'lat' in result_ncd.variables
assert 'depth' in result_ncd.variables
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile,
tmpfile) # Try to load it again
os.close(fid)
os.remove(tmpfile)
def test_omtp_multi(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources',
'om-multiple.nc')
with OrthogonalMultidimensionalTimeseriesProfile(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False)
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(
df, tmpfile) as result_ncd:
assert 'station' in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile,
tmpfile) # Try to load it again
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(
df, tmpfile, reduce_dims=True) as result_ncd:
# Even though we pass reduce_dims, there are two stations so it is not reduced
assert 'station' in result_ncd.dimensions
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile,
tmpfile) # Try to load it again
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(
df, tmpfile, unlimited=True) as result_ncd:
assert 'station' in result_ncd.dimensions
assert result_ncd.dimensions['t'].isunlimited() is True
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile,
tmpfile) # Try to load it again
with OrthogonalMultidimensionalTimeseriesProfile.from_dataframe(
df, tmpfile, reduce_dims=True,
unlimited=True) as result_ncd:
# Even though we pass reduce_dims, there are two stations so it is not reduced
assert 'station' in result_ncd.dimensions
assert result_ncd.dimensions['t'].isunlimited() is True
test_is_mine(OrthogonalMultidimensionalTimeseriesProfile,
tmpfile) # Try to load it again
os.close(fid)
os.remove(tmpfile)
def test_rtp_single(self):
filepath = os.path.join(os.path.dirname(__file__), 'resources',
'r-ctd-single.nc')
with RaggedTimeseriesProfile(filepath) as ncd:
fid, tmpfile = tempfile.mkstemp(suffix='.nc')
df = ncd.to_dataframe(clean_rows=False)
with RaggedTimeseriesProfile.from_dataframe(df,
tmpfile) as result_ncd:
assert 'station' in result_ncd.dimensions
test_is_mine(RaggedTimeseriesProfile,
tmpfile) # Try to load it again
with RaggedTimeseriesProfile.from_dataframe(
df, tmpfile, unique_dims=True) as result_ncd:
assert 'station_dim' in result_ncd.dimensions
test_is_mine(RaggedTimeseriesProfile,
tmpfile) # Try to load it again
with RaggedTimeseriesProfile.from_dataframe(
df, tmpfile, reduce_dims=True) as result_ncd:
# Even though we pass reduce_dims, there are two stations so it is not reduced
assert 'station' not in result_ncd.dimensions
assert 'profile' in result_ncd.dimensions
test_is_mine(RaggedTimeseriesProfile,
tmpfile) # Try to load it again
with RaggedTimeseriesProfile.from_dataframe(
df, tmpfile, unlimited=True) as result_ncd:
assert 'station' in result_ncd.dimensions
assert 'profile' in result_ncd.dimensions
assert result_ncd.dimensions['obs'].isunlimited() is True
test_is_mine(RaggedTimeseriesProfile,
tmpfile) # Try to load it again
with RaggedTimeseriesProfile.from_dataframe(
df, tmpfile, reduce_dims=True,
unlimited=True) as result_ncd:
assert 'station' not in result_ncd.dimensions
assert 'profile' in result_ncd.dimensions
assert result_ncd.dimensions['obs'].isunlimited() is True
test_is_mine(RaggedTimeseriesProfile,
tmpfile) # Try to load it again
with RaggedTimeseriesProfile.from_dataframe(
df,
tmpfile,
unique_dims=True,
reduce_dims=False,
unlimited=True) as result_ncd:
assert 'station_dim' in result_ncd.dimensions
assert 'profile_dim' in result_ncd.dimensions
assert result_ncd.dimensions['obs_dim'].isunlimited() is True
test_is_mine(RaggedTimeseriesProfile,
tmpfile) # Try to load it again
os.close(fid)
os.remove(tmpfile)
def test_crt_load(fp):
test_is_mine(ContiguousRaggedTrajectory, fp)
