def test_write_to_shape_match(cleanup_blocks):
b = Block((1, 2, 3), (5, 6, 7), fill='random', file_name='test.bin')
b.read()
by, _, _ = b.write_to(b)
assert (by == math.prod(b.shape))
with open(b.file_name, 'rb') as f:
data = f.read()
assert (data == b.data.bytes())
os.remove(b.file_name)
def test_block_offsets():
b = Block((1, 2, 3), (5, 6, 7))
assert ((b.block_offsets(b)) == (b.origin, b.shape, (0, 209)))
c = Block((0, 0, 0), (4, 4, 4))
assert (c.block_offsets(b) == (b.origin, (3, 2, 1),
(27, 27, 31, 31, 43, 43, 47, 47, 59, 59, 63,
63)))
d = Block((1, 2, 2), (4, 4, 4))
assert (c.block_offsets(d) == (d.origin, (3, 2, 2),
(26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62,
63)))
e = Block((1, 2, 1), (4, 4, 4))
assert (c.block_offsets(e) == (e.origin, (3, 2, 3),
(25, 27, 29, 31, 41, 43, 45, 47, 57, 59, 61,
63)))
def __get_blocks(self, fill):
'''
Initialize and return the list of blocks in this partition
Arguments:
fill: argument passed to Block
'''
# Blocks have to be created
blocks = {}
shape = self.array.shape
# Warning: read order of blocks in repartition
# depends on this key order...
ni = int(shape[0] / self.shape[0])
nj = int(shape[1] / self.shape[1])
nk = int(shape[2] / self.shape[2])
size = math.prod(self.shape)
blocks = {
(i * self.shape[0], j * self.shape[1], k * self.shape[2]): Block(
(i * self.shape[0], j * self.shape[1], k * self.shape[2]),
self.shape,
fill=fill,
file_name=(f'{self.name}_block_'
f'{size*(k+j*nk+i*nj*nk)}.bin'))
for i in range(ni) for j in range(nj) for k in range(nk)
}
return blocks
def test_offset():
b = Block((1, 1, 1), (4, 5, 6))
assert (b.offset((1, 1, 3)) == 2)
assert (b.offset((2, 2, 2)) == 1 + 6 + 30)
c = Block((1, 2, 3), (5, 6, 7))
assert (c.offset((1, 2, 9)) == 6)
assert (c.offset((1, 7, 3)) == 35)
assert (c.offset((1, 7, 9)) == 41)
assert (c.offset((2, 2, 3)) == 42)
assert (c.offset((5, 7, 9)) == math.prod(c.shape) - 1)
def merge_blocks(block_list):
'''
Assume block list merges in a cuboid block.
Assume all blocks are empty.
Return the merged blocks.
'''
message = 'Cannot merge non-empty blocks'
assert (all(b.data.mem_usage() == 0 for b in block_list)), message
origin = tuple(min([b.origin[i] for b in block_list]) for i in (0, 1, 2))
end = tuple(
max([b.origin[i] + b.shape[i] for b in block_list]) for i in (0, 1, 2))
shape = tuple(end[i] - origin[i] for i in (0, 1, 2))
b = Block(origin, shape, data=bytearray())
return b
def test_read_from_shape_mismatch(cleanup_blocks):
b = Block((1, 2, 3), (5, 6, 7), file_name='test.bin')
c = Block((1, 2, 3), (5, 2, 7), fill='random', file_name='block1.bin')
d = Block((1, 4, 3), (5, 4, 7), fill='random', file_name='block2.bin')
l, _, _ = b.read_from(c)
m, _, _ = b.read_from(d)
assert (l + m == math.prod(b.shape))
# Check content
c.read()
b.read()
d.read()
assert (c.data.bytes()[:10] == b.data.bytes()[:10])
assert (d.data.bytes()[-10:] == b.data.bytes()[-10:])
for fn in (c.file_name, d.file_name):
os.remove(fn)
def test_read_from_disjoint():
b = Block((4, 5, 6), (2, 4, 6))
c = Block((40, 50, 60), (2, 4, 6))
assert (b.read_from(c) == (0, 0))
def test_delete():
file_name = 'test.bin'
b = Block((1, 1, 1), (4, 5, 6), fill='zeros', file_name=file_name)
assert (os.path.isfile(file_name))
b.delete()
assert (not os.path.isfile(file_name))
def test_write_to_read_from(cleanup_blocks):
b = Block((1, 2, 3), (5, 6, 7), fill='random', file_name='test.bin')
c = Block((1, 2, 3), (5, 2, 7), file_name='block1.bin')
d = Block((1, 4, 3), (5, 4, 7), file_name='block2.bin')
b.read()
l, _, _ = b.write_to(c)
m, _, _ = b.write_to(d)
original_data = bytearray()
original_data[:] = b.data.bytes()
b.clear()
b.read_from(c)
b.read_from(d)
assert (b.data.bytes() == original_data)
for fn in (c.file_name, d.file_name):
os.remove(fn)
def test_zeros(cleanup_blocks):
b = Block((4, 5, 6), (2, 4, 6), fill='zeros', file_name='block.bin')
b.read()
assert (b.data.bytes() == bytearray(math.prod(b.shape)))
def get_F_blocks(write_block, out_blocks, get_data=False, dry_run=False):
'''
Assuming out_blocks are of uniform size
'''
# F0 is where the write_block origin is
origin = write_block.origin
shape = write_block.shape
out_ends = partition_to_end_coords(out_blocks)
# TODO: turn to list comprehension
block_ends = list(origin)
for d in (0, 1, 2):
ends = sorted(out_ends[d])
for o in ends:
if o > origin[d] and o <= origin[d] + shape[d] - 1:
block_ends[d] = o
shape = [block_ends[i] - origin[i] + 1 for i in range(len(origin))]
F0 = Block(origin, shape)
if get_data:
F0 = write_block.get_data_block(F0, dry_run)
# F1
origin = (F0.origin[0], F0.origin[1], F0.origin[2] + F0.shape[2])
shape = [F0.shape[0], F0.shape[1], write_block.shape[2] - F0.shape[2]]
F1 = Block(origin, shape)
if get_data:
F1 = write_block.get_data_block(F1, dry_run)
# F2
origin = (F0.origin[0], F0.origin[1] + F0.shape[1], F0.origin[2])
shape = [F0.shape[0], write_block.shape[1] - F0.shape[1], F0.shape[2]]
F2 = Block(origin, shape)
if get_data:
F2 = write_block.get_data_block(F2, dry_run)
# F3
origin = (F0.origin[0], F0.origin[1] + F0.shape[1],
F0.origin[2] + F0.shape[2])
shape = [
F0.shape[0], write_block.shape[1] - F0.shape[1],
write_block.shape[2] - F0.shape[2]
]
F3 = Block(origin, shape)
if get_data:
F3 = write_block.get_data_block(F3, dry_run)
# F4
origin = (F0.origin[0] + F0.shape[0], F0.origin[1], F0.origin[2])
shape = [write_block.shape[0] - F0.shape[0], F0.shape[1], F0.shape[2]]
F4 = Block(origin, shape)
if get_data:
F4 = write_block.get_data_block(F4, dry_run)
# F5
origin = (F0.origin[0] + F0.shape[0], F0.origin[1],
F0.origin[2] + F0.shape[2])
shape = [
write_block.shape[0] - F0.shape[0], F0.shape[1],
write_block.shape[2] - F0.shape[2]
]
F5 = Block(origin, shape)
if get_data:
F5 = write_block.get_data_block(F5, dry_run)
# F6
origin = (F0.origin[0] + F0.shape[0], F0.origin[1] + F0.shape[1],
F0.origin[2])
shape = [
write_block.shape[0] - F0.shape[0], write_block.shape[1] - F0.shape[1],
F0.shape[2]
]
F6 = Block(origin, shape)
if get_data:
F6 = write_block.get_data_block(F6, dry_run)
# F7
origin = (F0.origin[0] + F0.shape[0], F0.origin[1] + F0.shape[1],
F0.origin[2] + F0.shape[2])
shape = [
write_block.shape[0] - F0.shape[0], write_block.shape[1] - F0.shape[1],
write_block.shape[2] - F0.shape[2]
]
F7 = Block(origin, shape)
if get_data:
F7 = write_block.get_data_block(F7, dry_run)
# Remove empty blocks and return
f_blocks = [
f if not f.empty() else None for f in [F0, F1, F2, F3, F4, F5, F6, F7]
]
return f_blocks