def test_put_array_in_s3_with_compression(self, tmpdir):
import datacube.drivers.s3.storage.s3aio as s3aio
s = s3aio.S3LIO(True, False, str(tmpdir))
x = np.arange(4 * 4 * 4, dtype=np.uint8).reshape((4, 4, 4))
key_map = s.put_array_in_s3(x, (2, 2, 2), "base_name", 'arrayio')
e = np.empty(4 * 4 * 4, dtype=np.uint8).reshape((4, 4, 4))
keys = [a[0] for a in key_map]
idx = [a[1] for a in key_map]
e = s.assemble_array_from_s3(e, idx, 'arrayio', keys, np.uint8)
assert np.array_equal(x, e)
def test_regular_index(self):
import datacube.drivers.s3.storage.s3aio as s3aio
s = s3aio.S3LIO()
i = s.regular_index((-35 + 2 * 0.128, 149 + 2 * 0.128),
((-35, -34), (149, 150)), (4000, 4000))
assert i == [1024, 1024]
i = s.regular_index((3, 1, 1), (5, 5, 5), (3, 3, 3))
assert i == [1, 0, 0]
i = s.regular_index((3, 1, 1), (5, 5, 5), (3, 3, 3), True)
assert i == 4
def test_get_data_without_compression(self, tmpdir):
import datacube.drivers.s3.storage.s3aio as s3aio
s = s3aio.S3LIO(False, False, str(tmpdir))
x = np.arange(4 * 4 * 4, dtype=np.uint8).reshape((4, 4, 4))
key_map = s.put_array_in_s3(x, (2, 2, 2), "base_name", 'arrayio')
d = s.get_data_unlabeled('base_name', (4, 4, 4), (2, 2, 2), np.int8,
(slice(1, 3), slice(1, 3), slice(1, 3)),
'arrayio')
assert np.array_equal(x[1:3, 1:3, 1:3], d)
d = s.get_data_unlabeled_mp('base_name', (4, 4, 4), (2, 2, 2), np.int8,
(slice(1, 3), slice(1, 3), slice(1, 3)),
'arrayio')
assert np.array_equal(x[1:3, 1:3, 1:3], d)
def test_chunk_indices_1d(self):
import datacube.drivers.s3.storage.s3aio as s3aio
s = s3aio.S3LIO()
assert list(s.chunk_indices_1d(0, 10, 2)) == [
slice(0, 2, None),
slice(2, 4, None),
slice(4, 6, None),
slice(6, 8, None),
slice(8, 10, None)
]
assert list(s.chunk_indices_1d(0, 10, 2, slice(2, 8))) == [
slice(2, 4, None),
slice(4, 6, None),
slice(6, 8, None)
]
assert list(s.chunk_indices_1d(0, 10, 2, slice(2, 8),
True)) == [2, 2, 2]
def test_chunk_indices_nd(self):
import datacube.drivers.s3.storage.s3aio as s3aio
s = s3aio.S3LIO()
x = np.arange(4 * 4 * 4, dtype=np.uint8).reshape((4, 4, 4))
assert (list(s.chunk_indices_nd(x.shape, (2, 2, 2))) == [
(slice(0, 2, None), slice(0, 2, None), slice(0, 2, None)),
(slice(0, 2, None), slice(0, 2, None), slice(2, 4, None)),
(slice(0, 2, None), slice(2, 4, None), slice(0, 2, None)),
(slice(0, 2, None), slice(2, 4, None), slice(2, 4, None)),
(slice(2, 4, None), slice(0, 2, None), slice(0, 2, None)),
(slice(2, 4, None), slice(0, 2, None), slice(2, 4, None)),
(slice(2, 4, None), slice(2, 4, None), slice(0, 2, None)),
(slice(2, 4, None), slice(2, 4, None), slice(2, 4, None))
])
assert list(
s.chunk_indices_nd(
x.shape, (2, 2, 2),
(slice(1, 3), slice(1, 3), slice(1, 3)))) == [
(slice(1, 2, None), slice(1, 2, None), slice(1, 2, None)),
(slice(1, 2, None), slice(1, 2, None), slice(2, 3, None)),
(slice(1, 2, None), slice(2, 3, None), slice(1, 2, None)),
(slice(1, 2, None), slice(2, 3, None), slice(2, 3, None)),
(slice(2, 3, None), slice(1, 2, None), slice(1, 2, None)),
(slice(2, 3, None), slice(1, 2, None), slice(2, 3, None)),
(slice(2, 3, None), slice(2, 3, None), slice(1, 2, None)),
(slice(2, 3, None), slice(2, 3, None), slice(2, 3, None))
]
assert list(
s.chunk_indices_nd(x.shape, (2, 2, 2),
(slice(1, 3), slice(1, 3), slice(1, 3)),
True)) == [(1, 1, 1), (1, 1, 1), (1, 1, 1),
(1, 1, 1), (1, 1, 1), (1, 1, 1),
(1, 1, 1), (1, 1, 1)]
def test_create_s3lio(self, tmpdir):
import datacube.drivers.s3.storage.s3aio as s3aio
s = s3aio.S3LIO()