def test_sparse_array_not_dense(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(1, 8), tile=2),
t.Dim(ctx, domain=(1, 8), tile=2))
att = t.Attr(ctx, "val", dtype='f8')
T = t.SparseArray(ctx, self.path("foo"), domain=dom, attrs=(att, ))
T.dump()
self.assertTrue(T.name == self.path("foo"))
self.assertTrue(T.sparse)
def test_unique_attributes(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, "d1", (1, 4), 2, dtype='u8'),
t.Dim(ctx, "d2", (1, 4), 2, dtype='u8'))
attr1 = t.Attr(ctx, "foo", dtype=float)
attr2 = t.Attr(ctx, "foo", dtype=int)
with self.assertRaises(t.TileDBError):
t.ArraySchema(ctx, "foobar", domain=dom, attrs=(attr1, attr2))
def test_simple_2d_sparse_vector(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 3), tile=4, dtype=int),
t.Dim(ctx, domain=(0, 3), tile=4, dtype=int))
attr = t.Attr(ctx, dtype=float)
T = t.SparseArray(ctx, self.path("foo"), domain=dom, attrs=(attr, ))
values = np.array([3, 4], dtype=float)
T[[1, 2], [1, 2]] = values
assert_array_equal(T[[1, 2], [1, 2]], values)
def test_domain(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, "d1", (1, 4), 2, dtype='u8'),
t.Dim(ctx, "d2", (1, 4), 2, dtype='u8'))
dom.dump()
self.assertEqual(dom.ndim, 2)
self.assertEqual(dom.rank, dom.ndim)
self.assertEqual(dom.dtype, np.dtype("uint64"))
self.assertEqual(dom.shape, (4, 4))
# check that we can iterate over the dimensions
dim_names = [dim.name for dim in dom]
self.assertEqual(["d1", "d2"], dim_names)
def test_index_2d(self):
A = np.arange(10000).reshape((1000, 10))
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 999), tile=100),
t.Dim(ctx, domain=(0, 9), tile=2))
att = t.Attr(ctx, dtype=A.dtype)
T = t.DenseArray(ctx, self.path("foo"), dom, (att, ))
T[:] = A
for idx in self.good_index_1d:
self._test_index(A, T, idx)
for idx in self.bad_index_2d:
with self.assertRaises(IndexError):
T[idx]
def test_multiple_attributes(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 7), tile=8, dtype=int))
attr_int = t.Attr(ctx, "ints", dtype=int)
attr_float = t.Attr(ctx, "floats", dtype=float)
T = t.DenseArray(ctx,
self.path("foo"),
domain=dom,
attrs=(attr_int, attr_float))
V_ints = np.array([0, 1, 2, 3, 4, 6, 7, 5])
V_floats = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 7.0, 5.0])
V = {"ints": V_ints, "floats": V_floats}
T[:] = V
R = T[:]
assert_array_equal(V["ints"], R["ints"])
assert_array_equal(V["floats"], R["floats"])
# check error ncells length
V["ints"] = V["ints"][1:2].copy()
with self.assertRaises(t.TileDBError):
T[:] = V
# check error attribute does not exist
V["foo"] = V["ints"].astype(np.int8)
with self.assertRaises(t.TileDBError):
T[:] = V
def test_dense_array_fp_domain_error(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(1, 8), tile=2,
dtype=np.float64))
att = t.Attr(ctx, "val", dtype=np.float64)
with self.assertRaises(t.TileDBError):
t.DenseArray(ctx, self.path("foo"), domain=dom, attrs=(att, ))
def test_sparse_unordered_fp_domain(self):
ctx = t.Ctx()
dom = t.Domain(
ctx, t.Dim(ctx, "x", domain=(0.0, 10.0), tile=2.0, dtype=float))
attr = t.Attr(ctx, dtype=float)
T = t.SparseArray(ctx, self.path("foo"), domain=dom, attrs=(attr, ))
values = np.array([3.3, 2.7])
T[[4.2, 2.5]] = values
assert_array_equal(T[[2.5, 4.2]], values[::-1])
def test_ncell_attributes(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 9), tile=10, dtype=int))
attr = t.Attr(ctx, dtype=[("", np.int32), ("", np.int32)])
T = t.DenseArray(ctx, self.path("foo"), domain=dom, attrs=(attr, ))
A = np.ones((10, ), dtype=[("", np.int32), ("", np.int32)])
self.assertEqual(A.dtype, attr.dtype)
T[:] = A
assert_array_equal(A, T[:])
assert_array_equal(A[:5], T[:5])
def test_read_write(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 2), tile=3))
att = t.Attr(ctx, dtype='i8')
arr = t.DenseArray(ctx, self.path("foo"), domain=dom, attrs=[att])
A = np.array([1, 2, 3])
arr.write_direct(A)
arr.dump()
assert_array_equal(arr.read_direct(), A)
self.assertEqual(arr.ndim, A.ndim)
def test_multiple_attributes(self):
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(1, 10), tile=10, dtype=int),
t.Dim(ctx, domain=(1, 10), tile=10, dtype=int))
attr_int = t.Attr(ctx, "ints", dtype=int)
attr_float = t.Attr(ctx, "floats", dtype="float")
T = t.SparseArray(ctx,
self.path("foo"),
domain=dom,
attrs=(
attr_int,
attr_float,
))
I = np.array([1, 1, 1, 2, 3, 3, 3, 4])
J = np.array([1, 2, 4, 3, 1, 6, 7, 5])
V_ints = np.array([0, 1, 2, 3, 4, 6, 7, 5])
V_floats = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 7.0, 5.0])
V = {"ints": V_ints, "floats": V_floats}
T[I, J] = V
R = T[I, J]
assert_array_equal(V["ints"], R["ints"])
assert_array_equal(V["floats"], R["floats"])
# check error attribute does not exist
# TODO: should this be an attribute error?
V["foo"] = V["ints"].astype(np.int8)
with self.assertRaises(t.TileDBError):
T[I, J] = V
# check error ncells length
V["ints"] = V["ints"][1:2].copy()
with self.assertRaises(AttributeError):
T[I, J] = V
def test_subarray(self):
ctx = t.Ctx()
dom = t.Domain(ctx,
t.Dim(ctx, "x", domain=(1, 10000), tile=100, dtype=int))
att = t.Attr(ctx, "", dtype=float)
T = t.SparseArray(ctx, self.path("foo"), domain=dom, attrs=(att, ))
self.assertIsNone(T.nonempty_domain())
T[[50, 60, 100]] = [1.0, 2.0, 3.0]
self.assertEqual(((50, 100), ), T.nonempty_domain())
# retrieve just valid coordinates in subarray T[40:60]
assert_array_equal(T[40:61]["coords"]["x"], [50, 60])
def test_index_1d(self):
A = np.arange(1050, dtype=int)
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 1049), tile=100))
att = t.Attr(ctx, dtype=int)
T = t.DenseArray(ctx, self.path("foo"), domain=dom, attrs=(att, ))
T[:] = A
for idx in self.good_index_1d:
self._test_index(A, T, idx)
for idx in self.bad_index_1d:
with self.assertRaises(IndexError):
T[idx]
def test_array_1d_set_scalar(self):
A = np.zeros(50)
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 49), tile=10))
att = t.Attr(ctx, dtype=A.dtype)
T = t.DenseArray(ctx, self.path("foo"), dom, (att, ))
T[:] = A
for value in (-1, 0, 1, 10):
A[5:25] = value
T[5:25] = value
assert_array_equal(A, T[:])
A[:] = value
T[:] = value
assert_array_equal(A, T[:])
def test_array_interface(self):
# Tests that __array__ interface works
ctx = t.Ctx()
A1 = np.arange(1, 10)
arr1 = t.DenseArray.from_numpy(ctx, self.path("arr1"), A1)
A2 = np.array(arr1)
assert_array_equal(A1, A2)
# Test that __array__ interface throws an error when number of attributes > 1
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 2), tile=3))
foo = t.Attr(ctx, "foo", dtype='i8')
bar = t.Attr(ctx, "bar", dtype='i8')
arr2 = t.DenseArray(ctx,
self.path("arr2"),
domain=dom,
attrs=(foo, bar))
with self.assertRaises(ValueError):
np.array(arr2)
def test_minimal_dimension(self):
ctx = t.Ctx()
dim = t.Dim(ctx, domain=(0, 4))
self.assertEqual(dim.name, "", "dimension name is empty")
self.assertEqual(dim.shape, (5, ))
self.assertEqual(dim.tile, None, "tiled extent is None (void)")
def test_domain_dims_not_same_type(self):
ctx = t.Ctx()
with self.assertRaises(TypeError):
t.Domain(ctx, t.Dim(ctx, "d1", (1, 4), 2, dtype=int),
t.Dim(ctx, "d2", (1, 4), 2, dtype=float))
def test_array_2d(self):
A = np.arange(10000).reshape((1000, 10))
ctx = t.Ctx()
dom = t.Domain(ctx, t.Dim(ctx, domain=(0, 999), tile=100),
t.Dim(ctx, domain=(0, 9), tile=2))
att = t.Attr(ctx, dtype=A.dtype)
T = t.DenseArray(ctx, self.path("foo"), dom, (att, ))
self.assertEqual(len(A), len(T))
self.assertEqual(A.ndim, T.ndim)
self.assertEqual(A.shape, T.shape)
self.assertEqual(1, T.nattr)
self.assertEqual(A.dtype, T.attr(0).dtype)
# check that the non-empty domain is None
self.assertIsNone(T.nonempty_domain())
# Set data
T[:] = A
assert_array_equal(A, T[:])
# check the non-empty domain spans the whole domain
self.assertEqual(((0, 999), (0, 9)), T.nonempty_domain())
# check array-like
assert_array_equal(A, np.array(T))
# slicing
assert_array_equal(A, T[:])
assert_array_equal(A, T[...])
assert_array_equal(A, T[slice(None)])
# slice first dimension
assert_array_equal(A[:10], T[:10])
assert_array_equal(A[:10], T[:10])
assert_array_equal(A[10:20], T[10:20])
assert_array_equal(A[-10:], T[-10:])
assert_array_equal(A[:10, :], T[:10, :])
assert_array_equal(A[10:20, :], T[10:20, :])
assert_array_equal(A[-10:, :], T[-10:, :])
assert_array_equal(A[:10, ...], T[:10, ...])
assert_array_equal(A[10:20, ...], T[10:20, ...])
assert_array_equal(A[-10:, ...], T[-10:, ...])
assert_array_equal(A[:10, :, ...], T[:10, :, ...])
assert_array_equal(A[10:20, :, ...], T[10:20, :, ...])
assert_array_equal(A[-10:, :, ...], T[-10:, :, ...])
# slice second dimension
assert_array_equal(A[:, :2], T[:, :2])
assert_array_equal(A[:, 2:4], T[:, 2:4])
assert_array_equal(A[:, -2:], T[:, -2:])
assert_array_equal(A[..., :2], T[..., :2])
assert_array_equal(A[..., 2:4], T[..., 2:4])
assert_array_equal(A[..., -2:], T[..., -2:])
assert_array_equal(A[:, ..., :2], T[:, ..., :2])
assert_array_equal(A[:, ..., 2:4], T[:, ..., 2:4])
assert_array_equal(A[:, ..., -2:], T[:, ..., -2:])
# slice both dimensions
assert_array_equal(A[:10, :2], T[:10, :2])
assert_array_equal(A[10:20, 2:4], T[10:20, 2:4])
assert_array_equal(A[-10:, -2:], T[-10:, -2:])
# slice across tile boundries
assert_array_equal(A[:110], T[:110])
assert_array_equal(A[190:310], T[190:310])
assert_array_equal(A[-110:], T[-110:])
assert_array_equal(A[:110, :], T[:110, :])
assert_array_equal(A[190:310, :], T[190:310, :])
assert_array_equal(A[-110:, :], T[-110:, :])
assert_array_equal(A[:, :3], T[:, :3])
assert_array_equal(A[:, 3:7], T[:, 3:7])
assert_array_equal(A[:, -3:], T[:, -3:])
assert_array_equal(A[:110, :3], T[:110, :3])
assert_array_equal(A[190:310, 3:7], T[190:310, 3:7])
assert_array_equal(A[-110:, -3:], T[-110:, -3:])
# single row/col/item
assert_array_equal(A[0], T[0])
assert_array_equal(A[-1], T[-1])
assert_array_equal(A[:, 0], T[:, 0])
assert_array_equal(A[:, -1], T[:, -1])
self.assertEqual(A[0, 0], T[0, 0])
self.assertEqual(A[-1, -1], T[-1, -1])
# too many indices
with self.assertRaises(IndexError):
T[:, :, :]
with self.assertRaises(IndexError):
T[0, :, :]
with self.assertRaises(IndexError):
T[:, 0, :]
with self.assertRaises(IndexError):
T[:, :, 0]
with self.assertRaises(IndexError):
T[0, 0, 0]
# only single ellipsis allowed
with self.assertRaises(IndexError):
T[..., ...]
# check partial assignment
B = np.arange(10000, 20000).reshape((1000, 10))
T[190:310, 3:7] = B[190:310, 3:7]
assert_array_equal(A[:190], T[:190])
assert_array_equal(A[:, :3], T[:, :3])
assert_array_equal(B[190:310, 3:7], T[190:310, 3:7])
assert_array_equal(A[310:], T[310:])
assert_array_equal(A[:, 7:], T[:, 7:])
def test_array_1d(self):
A = np.arange(1050)
ctx = t.Ctx()
dom = t.Domain(ctx,
t.Dim(ctx, domain=(0, 1049), tile=100, dtype=np.int64))
att = t.Attr(ctx, dtype=A.dtype)
T = t.DenseArray(ctx, self.path("foo"), domain=dom, attrs=(att, ))
self.assertEqual(len(A), len(T))
self.assertEqual(A.ndim, T.ndim)
self.assertEqual(A.shape, T.shape)
self.assertEqual(1, T.nattr)
self.assertEqual(A.dtype, T.attr(0).dtype)
# check empty array
B = T[:]
self.assertEqual(A.shape, B.shape)
self.assertEqual(A.dtype, B.dtype)
self.assertIsNone(T.nonempty_domain())
# check set array
T[:] = A
self.assertEqual(((0, 1049), ), T.nonempty_domain())
# check slicing
assert_array_equal(A, np.array(T))
assert_array_equal(A, T[:])
assert_array_equal(A, T[...])
assert_array_equal(A, T[slice(None)])
assert_array_equal(A[:10], T[:10])
assert_array_equal(A[10:20], T[10:20])
assert_array_equal(A[-10:], T[-10:])
# ellipsis
assert_array_equal(A[:10, ...], T[:10, ...])
assert_array_equal(A[10:50, ...], T[10:50, ...])
assert_array_equal(A[-50:, ...], T[-50:, ...])
assert_array_equal(A[..., :10], T[..., :10])
assert_array_equal(A[..., 10:20], T[..., 10:20])
assert_array_equal(A[..., -50:], T[..., -50:])
# across tiles
assert_array_equal(A[:150], T[:150])
assert_array_equal(A[-250:], T[-250:])
# point index
self.assertEqual(A[0], T[0])
self.assertEqual(A[-1], T[-1])
# point index with all index types
self.assertEqual(A[123], T[np.int8(123)])
self.assertEqual(A[123], T[np.uint8(123)])
self.assertEqual(A[123], T[np.int16(123)])
self.assertEqual(A[123], T[np.uint16(123)])
self.assertEqual(A[123], T[np.int64(123)])
self.assertEqual(A[123], T[np.uint64(123)])
self.assertEqual(A[123], T[np.int32(123)])
self.assertEqual(A[123], T[np.uint32(123)])
# basic step
assert_array_equal(A[:50:2], T[:50:2])
assert_array_equal(A[:2:50], T[:2:50])
assert_array_equal(A[10:-1:50], T[10:-1:50])
# indexing errors
with self.assertRaises(IndexError):
T[:, :]
with self.assertRaises(IndexError):
T[:, 50]
with self.assertRaises(IndexError):
T[50, :]
with self.assertRaises(IndexError):
T[0, 0]
# check single ellipsis
with self.assertRaises(IndexError):
T[..., 1:5, ...]
# check partial assignment
B = np.arange(1e5, 2e5).astype(A.dtype)
T[190:310] = B[190:310]
assert_array_equal(A[:190], T[:190])
assert_array_equal(B[190:310], T[190:310])
assert_array_equal(A[310:], T[310:])
def test_dimension(self):
ctx = t.Ctx()
dim = t.Dim(ctx, "d1", domain=(0, 3), tile=2)
self.assertEqual(dim.name, "d1")
self.assertEqual(dim.shape, (4, ))
self.assertEqual(dim.tile, 2)