def test_dim_start_float():
ctx = tiledb.Ctx()
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx,
name="i",
domain=(0.0, 6.0),
tile=6,
dtype=np.float64),
ctx=ctx,
)
schema = tiledb.ArraySchema(
ctx=ctx,
domain=dom,
sparse=True,
attrs=[tiledb.Attr(ctx=ctx, name='a', dtype=np.float32)])
tempdir = tempfile.mkdtemp()
try:
# create tiledb array
tiledb.SparseArray.create(tempdir, schema)
with pytest.raises(ValueError):
fromtiledb(tempdir, ctx=ctx)
finally:
shutil.rmtree(tempdir)
def testFromTileDB(self):
ctx = tiledb.Ctx()
for sparse in (True, False):
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx, name="i", domain=(1, 30), tile=7, dtype=np.int32),
tiledb.Dim(ctx=ctx, name="j", domain=(1, 20), tile=3, dtype=np.int32),
tiledb.Dim(ctx=ctx, name="k", domain=(1, 10), tile=4, dtype=np.int32),
ctx=ctx,
)
schema = tiledb.ArraySchema(ctx=ctx, domain=dom, sparse=sparse,
attrs=[tiledb.Attr(ctx=ctx, name='a', dtype=np.float32)])
tempdir = tempfile.mkdtemp()
try:
# create tiledb array
array_type = tiledb.DenseArray if not sparse else tiledb.SparseArray
array_type.create(tempdir, schema)
tensor = fromtiledb(tempdir)
self.assertIsInstance(tensor.op, TensorTileDBDataSource)
self.assertEqual(tensor.op.issparse(), sparse)
self.assertEqual(tensor.shape, (30, 20, 10))
self.assertEqual(tensor.extra_params.raw_chunk_size, (7, 3, 4))
self.assertIsNone(tensor.op.tiledb_config)
self.assertEqual(tensor.op.tiledb_uri, tempdir)
self.assertIsNone(tensor.op.tiledb_key)
self.assertIsNone(tensor.op.tiledb_timestamp)
tensor = tensor.tiles()
self.assertEqual(len(tensor.chunks), 105)
self.assertIsInstance(tensor.chunks[0].op, TensorTileDBDataSource)
self.assertEqual(tensor.chunks[0].op.issparse(), sparse)
self.assertEqual(tensor.chunks[0].shape, (7, 3, 4))
self.assertIsNone(tensor.chunks[0].op.tiledb_config)
self.assertEqual(tensor.chunks[0].op.tiledb_uri, tempdir)
self.assertIsNone(tensor.chunks[0].op.tiledb_key)
self.assertIsNone(tensor.chunks[0].op.tiledb_timestamp)
self.assertEqual(tensor.chunks[0].op.tiledb_dim_starts, (1, 1, 1))
# test axis_offsets of chunk op
self.assertEqual(tensor.chunks[0].op.axis_offsets, (0, 0, 0))
self.assertEqual(tensor.chunks[1].op.axis_offsets, (0, 0, 4))
self.assertEqual(tensor.cix[0, 2, 2].op.axis_offsets, (0, 6, 8))
self.assertEqual(tensor.cix[0, 6, 2].op.axis_offsets, (0, 18, 8))
self.assertEqual(tensor.cix[4, 6, 2].op.axis_offsets, (28, 18, 8))
tensor2 = fromtiledb(tempdir, ctx=ctx)
self.assertEqual(tensor2.op.tiledb_config, ctx.config().dict())
tensor2 = tensor2.tiles()
self.assertEqual(tensor2.chunks[0].op.tiledb_config, ctx.config().dict())
finally:
shutil.rmtree(tempdir)
def test_from_tile_db():
ctx = tiledb.Ctx()
for sparse in (True, False):
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx,
name="i",
domain=(1, 30),
tile=7,
dtype=np.int32),
tiledb.Dim(ctx=ctx,
name="j",
domain=(1, 20),
tile=3,
dtype=np.int32),
tiledb.Dim(ctx=ctx,
name="k",
domain=(1, 10),
tile=4,
dtype=np.int32),
ctx=ctx,
)
schema = tiledb.ArraySchema(
ctx=ctx,
domain=dom,
sparse=sparse,
attrs=[tiledb.Attr(ctx=ctx, name='a', dtype=np.float32)])
tempdir = tempfile.mkdtemp()
try:
# create tiledb array
array_type = tiledb.DenseArray if not sparse else tiledb.SparseArray
array_type.create(tempdir, schema)
tensor = fromtiledb(tempdir)
assert isinstance(tensor.op, TensorTileDBDataSource)
assert tensor.op.issparse() == sparse
assert tensor.shape == (30, 20, 10)
assert tensor.extra_params.raw_chunk_size == (7, 3, 4)
assert tensor.op.tiledb_config is None
assert tensor.op.tiledb_uri == tempdir
assert tensor.op.tiledb_key is None
assert tensor.op.tiledb_timestamp is None
tensor = tile(tensor)
assert len(tensor.chunks) == 105
assert isinstance(tensor.chunks[0].op, TensorTileDBDataSource)
assert tensor.chunks[0].op.issparse() == sparse
assert tensor.chunks[0].shape == (7, 3, 4)
assert tensor.chunks[0].op.tiledb_config is None
assert tensor.chunks[0].op.tiledb_uri == tempdir
assert tensor.chunks[0].op.tiledb_key is None
assert tensor.chunks[0].op.tiledb_timestamp is None
assert tensor.chunks[0].op.tiledb_dim_starts == (1, 1, 1)
# test axis_offsets of chunk op
assert tensor.chunks[0].op.axis_offsets == (0, 0, 0)
assert tensor.chunks[1].op.axis_offsets == (0, 0, 4)
assert tensor.cix[0, 2, 2].op.axis_offsets == (0, 6, 8)
assert tensor.cix[0, 6, 2].op.axis_offsets == (0, 18, 8)
assert tensor.cix[4, 6, 2].op.axis_offsets == (28, 18, 8)
tensor2 = fromtiledb(tempdir, ctx=ctx)
assert tensor2.op.tiledb_config == ctx.config().dict()
tensor2 = tile(tensor2)
assert tensor2.chunks[0].op.tiledb_config == ctx.config().dict()
finally:
shutil.rmtree(tempdir)
def testReadTileDBExecution(self):
ctx = tiledb.Ctx()
tempdir = tempfile.mkdtemp()
try:
# create TileDB dense array
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx, domain=(1, 100), tile=30, dtype=np.int32),
tiledb.Dim(ctx=ctx, domain=(0, 90), tile=22, dtype=np.int32),
tiledb.Dim(ctx=ctx, domain=(0, 9), tile=8, dtype=np.int32),
ctx=ctx,
)
schema = tiledb.ArraySchema(
ctx=ctx,
domain=dom,
sparse=False,
attrs=[tiledb.Attr(ctx=ctx, dtype=np.float64)])
tiledb.DenseArray.create(tempdir, schema)
expected = np.random.rand(100, 91, 10)
with tiledb.DenseArray(uri=tempdir, ctx=ctx, mode='w') as arr:
arr.write_direct(expected)
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected, result)
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# create 2-d TileDB sparse array
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx, domain=(0, 99), tile=30, dtype=np.int32),
tiledb.Dim(ctx=ctx, domain=(2, 11), tile=8, dtype=np.int32),
ctx=ctx,
)
schema = tiledb.ArraySchema(
ctx=ctx,
domain=dom,
sparse=True,
attrs=[tiledb.Attr(ctx=ctx, dtype=np.float64)])
tiledb.SparseArray.create(tempdir, schema)
expected = sps.rand(100, 10, density=0.01)
with tiledb.SparseArray(uri=tempdir, ctx=ctx, mode='w') as arr:
I, J = expected.row, expected.col + 2
arr[I, J] = {arr.attr(0).name: expected.data}
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected.toarray(), result.toarray())
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# create 1-d TileDB sparse array
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx, domain=(1, 100), tile=30, dtype=np.int32),
ctx=ctx,
)
schema = tiledb.ArraySchema(
ctx=ctx,
domain=dom,
sparse=True,
attrs=[tiledb.Attr(ctx=ctx, dtype=np.float64)])
tiledb.SparseArray.create(tempdir, schema)
expected = sps.rand(1, 100, density=0.05)
with tiledb.SparseArray(uri=tempdir, ctx=ctx, mode='w') as arr:
I = expected.col + 1
arr[I] = expected.data
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected.toarray()[0], result.toarray())
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# create TileDB dense array with column-major
dom = tiledb.Domain(
tiledb.Dim(ctx=ctx, domain=(1, 100), tile=30, dtype=np.int32),
tiledb.Dim(ctx=ctx, domain=(0, 90), tile=22, dtype=np.int32),
tiledb.Dim(ctx=ctx, domain=(0, 9), tile=8, dtype=np.int32),
ctx=ctx,
)
schema = tiledb.ArraySchema(
ctx=ctx,
domain=dom,
sparse=False,
cell_order='F',
attrs=[tiledb.Attr(ctx=ctx, dtype=np.float64)])
tiledb.DenseArray.create(tempdir, schema)
expected = np.asfortranarray(np.random.rand(100, 91, 10))
with tiledb.DenseArray(uri=tempdir, ctx=ctx, mode='w') as arr:
arr.write_direct(expected)
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected, result)
self.assertTrue(result.flags['F_CONTIGUOUS'])
self.assertFalse(result.flags['C_CONTIGUOUS'])
finally:
shutil.rmtree(tempdir)
