def _test_convert_mismatch_helper(self, spec_type, value_types): data = 1 spec = DatasetSpec('an example dataset', spec_type, name='data') for dtype in value_types: value = self._get_type(dtype)(data) # convert data to given dtype with self.subTest(dtype=dtype): s = np.dtype(self._get_type(spec_type)) g = np.dtype(self._get_type(dtype)) msg = "expected %s, received %s - must supply %s" % (s.name, g.name, s.name) with self.assertRaisesWith(ValueError, msg): ObjectMapper.convert_dtype(spec, value)
def test_ascii_spec(self): ascii_spec_types = ['ascii', 'bytes'] for spec_type in ascii_spec_types: with self.subTest(spec_type=spec_type): spec = DatasetSpec('an example dataset', spec_type, name='data') value = 'a' ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, b'a') self.assertIs(type(ret), bytes) self.assertEqual(ret_dtype, 'ascii') value = b'a' ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, b'a') self.assertIs(type(ret), bytes) self.assertEqual(ret_dtype, 'ascii') value = ['a', 'b'] ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertListEqual(ret, [b'a', b'b']) self.assertIs(type(ret[0]), bytes) self.assertEqual(ret_dtype, 'ascii') value = np.array(['a', 'b']) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) np.testing.assert_array_equal(ret, np.array(['a', 'b'], dtype='S1')) self.assertEqual(ret_dtype, 'ascii') value = np.array(['a', 'b'], dtype='S1') ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) np.testing.assert_array_equal(ret, value) self.assertEqual(ret_dtype, 'ascii') value = [] ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertListEqual(ret, value) self.assertEqual(ret_dtype, 'ascii') value = 1 msg = "Expected unicode or ascii string, got <class 'int'>" with self.assertRaisesWith(ValueError, msg): ObjectMapper.convert_dtype(spec, value) value = DataChunkIterator(np.array(['a', 'b'])) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion self.assertIs(ret, value) self.assertEqual(ret_dtype, 'ascii') value = DataChunkIterator(np.array(['a', 'b'], dtype='S1')) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion self.assertIs(ret, value) self.assertEqual(ret_dtype, 'ascii')
def test_numeric_spec(self): spec_type = 'numeric' spec = DatasetSpec('an example dataset', spec_type, name='data') value = np.uint64(4) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret), np.uint64) self.assertEqual(ret_dtype, np.uint64) value = DataChunkIterator(data=[1, 2, 3]) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(ret.dtype.type, np.dtype(int).type) self.assertIs(type(ret.data[0]), int) self.assertEqual(ret_dtype, np.dtype(int).type) value = ['a', 'b'] msg = "Cannot convert from <class 'str'> to 'numeric' specification dtype." with self.assertRaisesWith(ValueError, msg): ObjectMapper.convert_dtype(spec, value) value = np.array(['a', 'b']) msg = "Cannot convert from <class 'numpy.str_'> to 'numeric' specification dtype." with self.assertRaisesWith(ValueError, msg): ObjectMapper.convert_dtype(spec, value) value = [] msg = "Cannot infer dtype of empty list or tuple. Please use numpy array with specified dtype." with self.assertRaisesWith(ValueError, msg): ObjectMapper.convert_dtype(spec, value)
def test_dci_input(self): spec = DatasetSpec('an example dataset', 'int64', name='data') value = DataChunkIterator(np.array([1, 2, 3], dtype=np.int32)) msg = "Spec 'data': Value with data type int32 is being converted to data type int64 as specified." with self.assertWarnsWith(UserWarning, msg): ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion self.assertIs(ret, value) self.assertEqual(ret_dtype, np.int64) spec = DatasetSpec('an example dataset', 'int16', name='data') value = DataChunkIterator(np.array([1, 2, 3], dtype=np.int32)) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion self.assertIs(ret, value) self.assertEqual(ret_dtype, np.int32) # increase precision
def test_bool_spec(self): spec_type = 'bool' spec = DatasetSpec('an example dataset', spec_type, name='data') value = np.bool_(True) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret), np.bool_) self.assertEqual(ret_dtype, np.bool_) value = True ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret), np.bool_) self.assertEqual(ret_dtype, np.bool_)
def test_compound_type(self): """Test that convert_dtype passes through arguments if spec dtype is a list without any validation.""" spec_type = [DtypeSpec('an int field', 'f1', 'int'), DtypeSpec('a float field', 'f2', 'float')] spec = DatasetSpec('an example dataset', spec_type, name='data') value = ['a', 1, 2.2] res, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertListEqual(res, value) self.assertListEqual(ret_dtype, spec_type)
def _test_keep_higher_precision_helper(self, spec_type, value_types): data = 1 spec = DatasetSpec('an example dataset', spec_type, name='data') for dtype in value_types: value = self._get_type(dtype)(data) match = (value, self._get_type(dtype)) with self.subTest(dtype=dtype): ret = ObjectMapper.convert_dtype(spec, value) self.assertTupleEqual(ret, match) self.assertIs(ret[0].dtype.type, match[1])
def _test_convert_alias(self, spec_type, value_types): data = 1 spec = DatasetSpec('an example dataset', spec_type, name='data') match = (self._get_type(spec_type)(data), self._get_type(spec_type)) for dtype in value_types: value = self._get_type(dtype)(data) # convert data to given dtype with self.subTest(dtype=dtype): ret = ObjectMapper.convert_dtype(spec, value) self.assertTupleEqual(ret, match) self.assertIs(ret[0].dtype.type, match[1])
def test_isodatetime_spec(self): spec_type = 'isodatetime' spec = DatasetSpec('an example dataset', spec_type, name='data') # NOTE: datetime.isoformat is called on all values with a datetime spec before conversion # see ObjectMapper.get_attr_value value = datetime.isoformat(datetime(2020, 11, 10)) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, b'2020-11-10T00:00:00') self.assertIs(type(ret), bytes) self.assertEqual(ret_dtype, 'ascii')
def _test_convert_higher_precision_helper(self, spec_type, value_types): data = 1 spec = DatasetSpec('an example dataset', spec_type, name='data') match = (self._get_type(spec_type)(data), self._get_type(spec_type)) for dtype in value_types: value = self._get_type(dtype)(data) # convert data to given dtype with self.subTest(dtype=dtype): s = np.dtype(self._get_type(spec_type)) g = np.dtype(self._get_type(dtype)) msg = ("Spec 'data': Value with data type %s is being converted to data type %s as specified." % (g.name, s.name)) with self.assertWarnsWith(UserWarning, msg): ret = ObjectMapper.convert_dtype(spec, value) self.assertTupleEqual(ret, match) self.assertIs(ret[0].dtype.type, match[1])
def test_override_type_none_to_bool(self): spec = DatasetSpec('an example dataset', None, name='data') res = ObjectMapper.convert_dtype(spec, True, 'bool') self.assertTupleEqual(res, (True, np.bool_))
def test_override_type_numeric_to_uint_list(self): spec = DatasetSpec('an example dataset', 'numeric', name='data') res = ObjectMapper.convert_dtype(spec, np.uint32((1, 2, 3)), 'uint8') np.testing.assert_array_equal(res[0], np.uint32((1, 2, 3))) self.assertEqual(res[1], np.uint32)
def test_override_type_numeric_to_uint(self): spec = DatasetSpec('an example dataset', 'numeric', name='data') res = ObjectMapper.convert_dtype(spec, np.uint32(1), 'uint8') self.assertTupleEqual(res, (np.uint32(1), np.uint32))
def test_override_type_int_restrict_precision(self): spec = DatasetSpec('an example dataset', 'int8', name='data') res = ObjectMapper.convert_dtype(spec, np.int64(1), 'int64') self.assertTupleEqual(res, (np.int64(1), np.int64))
def test_no_spec(self): spec_type = None spec = DatasetSpec('an example dataset', spec_type, name='data') value = [1, 2, 3] ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertListEqual(ret, value) self.assertIs(type(ret[0]), int) self.assertEqual(ret_dtype, int) value = np.uint64(4) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret), np.uint64) self.assertEqual(ret_dtype, np.uint64) value = 'hello' ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret), str) self.assertEqual(ret_dtype, 'utf8') value = b'hello' ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret), bytes) self.assertEqual(ret_dtype, 'ascii') value = np.array(['aa', 'bb']) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) np.testing.assert_array_equal(ret, value) self.assertEqual(ret_dtype, 'utf8') value = np.array(['aa', 'bb'], dtype='S2') ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) np.testing.assert_array_equal(ret, value) self.assertEqual(ret_dtype, 'ascii') value = DataChunkIterator(data=[1, 2, 3]) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(ret.dtype.type, np.dtype(int).type) self.assertIs(type(ret.data[0]), int) self.assertEqual(ret_dtype, np.dtype(int).type) value = DataChunkIterator(data=['a', 'b']) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(ret.dtype.type, np.str_) self.assertIs(type(ret.data[0]), str) self.assertEqual(ret_dtype, 'utf8') value = H5DataIO(np.arange(30).reshape(5, 2, 3)) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(ret.data.dtype.type, np.dtype(int).type) self.assertEqual(ret_dtype, np.dtype(int).type) value = H5DataIO(['foo', 'bar']) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret.data[0]), str) self.assertEqual(ret_dtype, 'utf8') value = H5DataIO([b'foo', b'bar']) ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) self.assertEqual(ret, value) self.assertIs(type(ret.data[0]), bytes) self.assertEqual(ret_dtype, 'ascii') value = [] msg = "Cannot infer dtype of empty list or tuple. Please use numpy array with specified dtype." with self.assertRaisesWith(ValueError, msg): ObjectMapper.convert_dtype(spec, value)
def test_value_none(self): spec = DatasetSpec('an example dataset', 'int', name='data') self.assertTupleEqual(ObjectMapper.convert_dtype(spec, None), (None, 'int')) spec = DatasetSpec('an example dataset', RefSpec(reftype='object', target_type='int'), name='data') self.assertTupleEqual(ObjectMapper.convert_dtype(spec, None), (None, 'object'))