def test_roundtrip_coordinates(self):
original = Dataset({'foo': ('x', [0, 1])}, {
'x': [2, 3],
'y': ('a', [42]),
'z': ('x', [4, 5])
})
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(original, actual)
expected = original.drop('foo')
with self.roundtrip(expected) as actual:
self.assertDatasetIdentical(expected, actual)
expected = original.copy()
expected.attrs['coordinates'] = 'something random'
with self.assertRaisesRegexp(ValueError, 'cannot serialize'):
with self.roundtrip(expected):
pass
expected = original.copy(deep=True)
expected['foo'].attrs['coordinates'] = 'something random'
with self.assertRaisesRegexp(ValueError, 'cannot serialize'):
with self.roundtrip(expected):
pass
def test_roundtrip_strings_with_fill_value(self):
values = np.array(['ab', 'cdef', np.nan], dtype=object)
encoding = {'_FillValue': np.string_('X'), 'dtype': np.dtype('S1')}
original = Dataset({'x': ('t', values, {}, encoding)})
expected = original.copy(deep=True)
expected['x'][:2] = values[:2].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
original = Dataset({'x': ('t', values, {}, {'_FillValue': '\x00'})})
if not isinstance(self, Only32BitTypes):
# these stores can save unicode strings
expected = original.copy(deep=True)
if isinstance(self, BaseNetCDF4Test):
# netCDF4 can't keep track of an empty _FillValue for VLEN
# variables
expected['x'][-1] = ''
elif (isinstance(
self,
(NetCDF3ViaNetCDF4DataTest, NetCDF4ClassicViaNetCDF4DataTest))
or (has_netCDF4 and type(self) is GenericNetCDFDataTest)):
# netCDF4 can't keep track of an empty _FillValue for nc3, either:
# https://github.com/Unidata/netcdf4-python/issues/273
expected['x'][-1] = np.string_('')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def test_roundtrip_strings_with_fill_value(self):
values = np.array(['ab', 'cdef', np.nan], dtype=object)
encoding = {'_FillValue': np.string_('X'), 'dtype': np.dtype('S1')}
original = Dataset({'x': ('t', values, {}, encoding)})
expected = original.copy(deep=True)
expected['x'][:2] = values[:2].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
original = Dataset({'x': ('t', values, {}, {'_FillValue': '\x00'})})
if not isinstance(self, Only32BitTypes):
# these stores can save unicode strings
expected = original.copy(deep=True)
if isinstance(self, BaseNetCDF4Test):
# netCDF4 can't keep track of an empty _FillValue for VLEN
# variables
expected['x'][-1] = ''
elif (isinstance(self, (NetCDF3ViaNetCDF4DataTest,
NetCDF4ClassicViaNetCDF4DataTest)) or
(has_netCDF4 and type(self) is GenericNetCDFDataTest)):
# netCDF4 can't keep track of an empty _FillValue for nc3, either:
# https://github.com/Unidata/netcdf4-python/issues/273
expected['x'][-1] = np.string_('')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def test_coords_modify(self):
data = Dataset({'x': ('x', [-1, -2]),
'y': ('y', [0, 1, 2]),
'foo': (['x', 'y'], np.random.randn(2, 3))},
{'a': ('x', [4, 5]), 'b': -10})
actual = data.copy(deep=True)
actual.coords['x'] = ('x', ['a', 'b'])
self.assertArrayEqual(actual['x'], ['a', 'b'])
actual = data.copy(deep=True)
actual.coords['z'] = ('z', ['a', 'b'])
self.assertArrayEqual(actual['z'], ['a', 'b'])
with self.assertRaisesRegexp(ValueError, 'conflicting sizes'):
data.coords['x'] = ('x', [-1])
actual = data.copy()
del actual.coords['b']
expected = data.reset_coords('b', drop=True)
self.assertDatasetIdentical(expected, actual)
with self.assertRaises(KeyError):
del data.coords['not_found']
with self.assertRaises(KeyError):
del data.coords['foo']
def test_dataset_math(self):
# more comprehensive tests with multiple dataset variables
obs = Dataset(
{
'tmin': ('x', np.arange(5)),
'tmax': ('x', 10 + np.arange(5))
}, {
'x': ('x', 0.5 * np.arange(5)),
'loc': ('x', range(-2, 3))
})
actual = 2 * obs['tmax']
expected = DataArray(2 * (10 + np.arange(5)), obs.coords, name='tmax')
self.assertDataArrayIdentical(actual, expected)
actual = obs['tmax'] - obs['tmin']
expected = DataArray(10 * np.ones(5), obs.coords)
self.assertDataArrayIdentical(actual, expected)
sim = Dataset({
'tmin': ('x', 1 + np.arange(5)),
'tmax': ('x', 11 + np.arange(5)),
# does *not* include 'loc' as a coordinate
'x': ('x', 0.5 * np.arange(5))
})
actual = sim['tmin'] - obs['tmin']
expected = DataArray(np.ones(5), obs.coords, name='tmin')
self.assertDataArrayIdentical(actual, expected)
actual = -obs['tmin'] + sim['tmin']
self.assertDataArrayIdentical(actual, expected)
actual = sim['tmin'].copy()
actual -= obs['tmin']
self.assertDataArrayIdentical(actual, expected)
actual = sim.copy()
actual['tmin'] = sim['tmin'] - obs['tmin']
expected = Dataset(
{
'tmin': ('x', np.ones(5)),
'tmax': ('x', sim['tmax'].values)
}, obs.coords)
self.assertDatasetIdentical(actual, expected)
actual = sim.copy()
actual['tmin'] -= obs['tmin']
self.assertDatasetIdentical(actual, expected)
def test_roundtrip_object_dtype(self):
floats = np.array([0.0, 0.0, 1.0, 2.0, 3.0], dtype=object)
floats_nans = np.array([np.nan, np.nan, 1.0, 2.0, 3.0], dtype=object)
letters = np.array(['ab', 'cdef', 'g'], dtype=object)
letters_nans = np.array(['ab', 'cdef', np.nan], dtype=object)
all_nans = np.array([np.nan, np.nan], dtype=object)
original = Dataset({
'floats': ('a', floats),
'floats_nans': ('a', floats_nans),
'letters': ('b', letters),
'letters_nans': ('b', letters_nans),
'all_nans': ('c', all_nans),
'nan': ([], np.nan)
})
expected = original.copy(deep=True)
if isinstance(self, Only32BitTypes):
# for netCDF3 tests, expect the results to come back as characters
expected['letters_nans'] = expected['letters_nans'].astype('S')
expected['letters'] = expected['letters'].astype('S')
with self.roundtrip(original) as actual:
try:
self.assertDatasetIdentical(expected, actual)
except AssertionError:
# Most stores use '' for nans in strings, but some don't
# first try the ideal case (where the store returns exactly)
# the original Dataset), then try a more realistic case.
# ScipyDataTest, NetCDF3ViaNetCDF4DataTest and NetCDF4DataTest
# all end up using this case.
expected['letters_nans'][-1] = ''
self.assertDatasetIdentical(expected, actual)
def test_roundtrip_object_dtype(self):
floats = np.array([0.0, 0.0, 1.0, 2.0, 3.0], dtype=object)
floats_nans = np.array([np.nan, np.nan, 1.0, 2.0, 3.0], dtype=object)
letters = np.array(['ab', 'cdef', 'g'], dtype=object)
letters_nans = np.array(['ab', 'cdef', np.nan], dtype=object)
all_nans = np.array([np.nan, np.nan], dtype=object)
original = Dataset({'floats': ('a', floats),
'floats_nans': ('a', floats_nans),
'letters': ('b', letters),
'letters_nans': ('b', letters_nans),
'all_nans': ('c', all_nans),
'nan': ([], np.nan)})
if PY3 and type(self) is ScipyDataTest:
# see the note under test_zero_dimensional_variable
del original['nan']
expected = original.copy(deep=True)
if type(self) in [NetCDF3ViaNetCDF4DataTest, ScipyDataTest]:
# for netCDF3 tests, expect the results to come back as characters
expected['letters_nans'] = expected['letters_nans'].astype('S')
expected['letters'] = expected['letters'].astype('S')
with self.roundtrip(original) as actual:
try:
self.assertDatasetIdentical(expected, actual)
except AssertionError:
# Most stores use '' for nans in strings, but some don't
# first try the ideal case (where the store returns exactly)
# the original Dataset), then try a more realistic case.
# ScipyDataTest, NetCDF3ViaNetCDF4DataTest and NetCDF4DataTest
# all end up using this case.
expected['letters_nans'][-1] = ''
self.assertDatasetIdentical(expected, actual)
def test_roundtrip_object_dtype(self):
floats = np.array([0.0, 0.0, 1.0, 2.0, 3.0], dtype=object)
floats_nans = np.array([np.nan, np.nan, 1.0, 2.0, 3.0], dtype=object)
letters = np.array(['ab', 'cdef', 'g'], dtype=object)
letters_nans = np.array(['ab', 'cdef', np.nan], dtype=object)
all_nans = np.array([np.nan, np.nan], dtype=object)
original = Dataset({
'floats': ('a', floats),
'floats_nans': ('a', floats_nans),
'letters': ('b', letters),
'letters_nans': ('b', letters_nans),
'all_nans': ('c', all_nans),
'nan': ([], np.nan)
})
if PY3 and type(self) is ScipyDataTest:
# see the note under test_zero_dimensional_variable
del original['nan']
expected = original.copy(deep=True)
expected['letters_nans'][-1] = ''
if type(self) is not NetCDF4DataTest:
# for netCDF3 tests, expect the results to come back as characters
expected['letters_nans'] = expected['letters_nans'].astype('S')
expected['letters'] = expected['letters'].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def test_rename_inplace(self):
times = pd.date_range('2000-01-01', periods=3)
data = Dataset({'z': ('x', [2, 3, 4]), 't': ('t', times)})
copied = data.copy()
renamed = data.rename({'x': 'y'})
data.rename({'x': 'y'}, inplace=True)
self.assertDatasetIdentical(data, renamed)
self.assertFalse(data.equals(copied))
self.assertEquals(data.dims, {'y': 3, 't': 3})
# check virtual variables
self.assertArrayEqual(data['t.dayofyear'], [1, 2, 3])
def test_dataset_math(self):
# more comprehensive tests with multiple dataset variables
obs = Dataset({'tmin': ('x', np.arange(5)),
'tmax': ('x', 10 + np.arange(5))},
{'x': ('x', 0.5 * np.arange(5)),
'loc': ('x', range(-2, 3))})
actual = 2 * obs['tmax']
expected = DataArray(2 * (10 + np.arange(5)), obs.coords, name='tmax')
self.assertDataArrayIdentical(actual, expected)
actual = obs['tmax'] - obs['tmin']
expected = DataArray(10 * np.ones(5), obs.coords)
self.assertDataArrayIdentical(actual, expected)
sim = Dataset({'tmin': ('x', 1 + np.arange(5)),
'tmax': ('x', 11 + np.arange(5)),
# does *not* include 'loc' as a coordinate
'x': ('x', 0.5 * np.arange(5))})
actual = sim['tmin'] - obs['tmin']
expected = DataArray(np.ones(5), obs.coords, name='tmin')
self.assertDataArrayIdentical(actual, expected)
actual = -obs['tmin'] + sim['tmin']
self.assertDataArrayIdentical(actual, expected)
actual = sim['tmin'].copy()
actual -= obs['tmin']
self.assertDataArrayIdentical(actual, expected)
actual = sim.copy()
actual['tmin'] = sim['tmin'] - obs['tmin']
expected = Dataset({'tmin': ('x', np.ones(5)),
'tmax': ('x', sim['tmax'].values)},
obs.coords)
self.assertDatasetIdentical(actual, expected)
actual = sim.copy()
actual['tmin'] -= obs['tmin']
self.assertDatasetIdentical(actual, expected)
def test_roundtrip_strings_with_fill_value(self):
values = np.array(['ab', 'cdef', np.nan], dtype=object)
encoding = {'_FillValue': np.string_('X'), 'dtype': np.dtype('S1')}
original = Dataset({'x': ('t', values, {}, encoding)})
expected = original.copy(deep=True)
expected['x'][:2] = values[:2].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
original = Dataset({'x': ('t', values, {}, {'_FillValue': '\x00'})})
if type(self) is NetCDF4DataTest:
# NetCDF4 should still write a VLEN (unicode) string
expected = original.copy(deep=True)
# the netCDF4 library can't keep track of an empty _FillValue for
# VLEN variables:
expected['x'][-1] = ''
elif type(self) is NetCDF3ViaNetCDF4DataTest:
# netCDF4 can't keep track of an empty _FillValue for nc3, either:
# https://github.com/Unidata/netcdf4-python/issues/273
expected['x'][-1] = np.string_('')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def test_roundtrip_coordinates(self):
original = Dataset({'foo': ('x', [0, 1])},
{'x': [2, 3], 'y': ('a', [42]), 'z': ('x', [4, 5])})
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(original, actual)
expected = original.drop('foo')
with self.roundtrip(expected) as actual:
self.assertDatasetIdentical(expected, actual)
expected = original.copy()
expected.attrs['coordinates'] = 'something random'
with self.assertRaisesRegexp(ValueError, 'cannot serialize'):
with self.roundtrip(expected):
pass
expected = original.copy(deep=True)
expected['foo'].attrs['coordinates'] = 'something random'
with self.assertRaisesRegexp(ValueError, 'cannot serialize'):
with self.roundtrip(expected):
pass
def test_roundtrip_strings_with_fill_value(self):
values = np.array(['ab', 'cdef', np.nan], dtype=object)
encoding = {'_FillValue': np.string_('X'), 'dtype': np.dtype('S1')}
original = Dataset({'x': ('t', values, {}, encoding)})
expected = original.copy(deep=True)
expected['x'][:2] = values[:2].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
original = Dataset({'x': ('t', values, {}, {'_FillValue': '\x00'})})
if type(self) is NetCDF4DataTest:
# NetCDF4 should still write a VLEN (unicode) string
expected = original.copy(deep=True)
# the netCDF4 library can't keep track of an empty _FillValue for
# VLEN variables:
expected['x'][-1] = ''
elif type(self) is NetCDF3ViaNetCDF4DataTest:
# netCDF4 can't keep track of an empty _FillValue for nc3, either:
# https://github.com/Unidata/netcdf4-python/issues/273
expected['x'][-1] = np.string_('')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def test_roundtrip_object_dtype(self):
floats = np.array([0.0, 0.0, 1.0, 2.0, 3.0], dtype=object)
floats_nans = np.array([np.nan, np.nan, 1.0, 2.0, 3.0], dtype=object)
letters = np.array(['ab', 'cdef', 'g'], dtype=object)
letters_nans = np.array(['ab', 'cdef', np.nan], dtype=object)
all_nans = np.array([np.nan, np.nan], dtype=object)
original = Dataset({'floats': ('a', floats),
'floats_nans': ('a', floats_nans),
'letters': ('b', letters),
'letters_nans': ('b', letters_nans),
'all_nans': ('c', all_nans),
'nan': ([], np.nan)})
if PY3 and type(self) is ScipyDataTest:
# see the note under test_zero_dimensional_variable
del original['nan']
expected = original.copy(deep=True)
expected['letters_nans'][-1] = ''
if type(self) is not NetCDF4DataTest:
# for netCDF3 tests, expect the results to come back as characters
expected['letters_nans'] = expected['letters_nans'].astype('S')
expected['letters'] = expected['letters'].astype('S')
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(expected, actual)
def test_coord_math(self):
ds = Dataset({'x': ('x', 1 + np.arange(3))})
expected = ds.copy()
expected['x2'] = ('x', np.arange(3))
actual = ds['x'] - 1
self.assertDataArrayEqual(expected['x2'], actual)
def test_coord_math(self):
ds = Dataset({'x': ('x', 1 + np.arange(3))})
expected = ds.copy()
expected['x2'] = ('x', np.arange(3))
actual = ds['x'] - 1
self.assertDataArrayEqual(expected['x2'], actual)
