def overlap_slice(curr_buff, buff, blocks_shape):
""" Utility function for buffering
curr_buff: current buffer containing 1+ entire rows
buff: buffer containing a entire row
"""
end_of_buffer = curr_buff[-1]
start_of_row = buff[0]
i_1 = numeric_to_3d_pos(end_of_buffer, blocks_shape, order='F')[0]
i_2 = numeric_to_3d_pos(start_of_row, blocks_shape, order='F')[0]
if i_1 != i_2:
return True
return False
def get_buff_to_vols(R, B, O, buffers_volumes, buffers_partition):
""" Outputs a dictionary associating buffer_index to list of Volumes indexed as in paper.
"""
logger.debug("== Function == get_buff_to_vols")
buff_to_vols = dict()
for buffer_index in buffers_volumes.keys():
_3d_index = numeric_to_3d_pos(buffer_index,
buffers_partition,
order='F')
T = list()
for dim in range(len(buffers_volumes[buffer_index].p1)):
C = ((_3d_index[dim] + 1) * B[dim]) % O[dim]
if C == 0 and B[dim] != O[dim]: # particular case
C = O[dim]
T.append(B[dim] - C)
volumes_list = get_main_volumes(B, T) # get coords in basis of buffer
volumes_list = volumes_list + compute_hidden_volumes(
T, O) # still in basis of buffer
add_offsets(volumes_list, _3d_index, B) # convert coords in basis of R
buff_to_vols[buffer_index] = volumes_list
logger.debug("Associated buffer n°%s to volumes:", buffer_index)
for v in volumes_list:
v.print()
logger.debug("End\n")
return buff_to_vols
def test_get_volumes():
""" test getmain and gethidden
"""
R = (1, 120, 120)
B = (1, 60, 60)
O = (1, 40, 40)
logger.debug("FUNCTION test_get_volumes ---")
from dask_io.optimizer.utils.utils import numeric_to_3d_pos
from dask_io.optimizer.cases.resplit_utils import get_blocks_shape
buffers_partition = get_blocks_shape(R, B)
for bufferindex in range(4):
logger.debug("buffer %s", bufferindex)
_3d_index = numeric_to_3d_pos(bufferindex,
buffers_partition,
order='F')
T = list()
for dim in range(3):
nb = _3d_index[dim] + 1
logger.debug("nb:%s", nb)
C = (nb * B[dim]) % O[dim]
if C == 0 and B[dim] != O[dim]:
C = O[dim]
T.append(B[dim] - C)
logger.debug("C: %s", C)
logger.debug("T: %s", T)
main_volumes = get_main_volumes(B, T)
assert len(main_volumes) == 4
def get_buffer_slices_from_original_array(load, shape, original_array_chunk):
start = min(load)
end = max(load)
all_block_num_indexes = range(start, end + 1)
all_block_3d_indexes = [
numeric_to_3d_pos(num_pos, shape, order='F')
for num_pos in all_block_num_indexes
]
mini = [None, None, None]
maxi = [None, None, None]
for _3d_index in all_block_3d_indexes:
for i in range(3):
if (mini[i] is None) or (_3d_index[i] < mini[i]):
mini[i] = _3d_index[i]
if maxi[i] is None or (_3d_index[i] + 1 > maxi[i]):
maxi[i] = _3d_index[i] + 1
mini = [e * d for e, d in zip(mini, original_array_chunk)]
maxi = [e * d for e, d in zip(maxi, original_array_chunk)]
return (slice(mini[0], maxi[0],
None), slice(mini[1], maxi[1],
None), slice(mini[2], maxi[2], None))
def origarr_to_buffer_slices(dicts, proxy, buffer_key, slices, chunk_shape):
buffer_id, _ = buffer_key[0].split('-')
origarr_name = 'array-original' + '-' + buffer_id
origarr_obj = dicts['origarr_to_obj'][origarr_name]
img_nb_blocks_per_dim = dicts['origarr_to_blocks_shape'][origarr_name]
block_id, start_block, end_block = buffer_key
start_pos = numeric_to_3d_pos(start_block, img_nb_blocks_per_dim, 'F')
offset = [x * i for x, i in zip(start_pos, chunk_shape)]
new_slices = list()
for i, s in enumerate(slices):
start = s.start - offset[i]
stop = s.stop - offset[i]
new_slices.append(slice(start, stop, s.step))
slices = (new_slices[0], new_slices[1], new_slices[2])
return slices
def apply_store(B, O, R, volumestokeep, reconstructed_array):
""" Apply store, using the keep strategy.
"""
# creations of data for dask store function
d_arrays, d_regions = compute_zones(B, O, R, volumestokeep)
out_files = list() # to keep outfiles open during processing
sources = list()
targets = list()
regions = list()
for outfile_index in sorted(d_arrays.keys()):
sliceslistoflist = d_arrays[outfile_index]
# create file
outfiles_partition = get_blocks_shape(R, O)
_3d_pos = numeric_to_3d_pos(outfile_index,
outfiles_partition,
order='F')
i, j, k = _3d_pos
out_filename = f'{i}_{j}_{k}.hdf5'
out_file = h5py.File(os.path.join(outdir_path, out_filename), 'w')
out_files.append(out_file)
# create dset
dset = out_file.create_dataset('/data', shape=O, dtype=np.float16)
for i, st in enumerate(sliceslistoflist):
tmp_array = reconstructed_array[st[0], st[1], st[2]]
# print("Volume to be stored shape: ", tmp_array.shape)
reg = d_regions[outfile_index][i]
tmp_array = tmp_array.rechunk(tmp_array.shape)
sources.append(tmp_array)
targets.append(dset)
regions.append(reg)
return da.store(sources, targets, regions=regions,
compute=False), out_files
_type, R, O, I, B, volumestokeep = int(case["type"]), tuple(
case["R"]), tuple(case["O"]), tuple(case["I"]), tuple(
case["B"]), case["volumestokeep"]
print(
f'Current run ------ \nType: {_type}\nR: {R},\nO: {O},\nI: {I}\nvolumestokeep: {volumestokeep}'
)
buffers_partition = get_blocks_shape(R, B)
buffers_volumes = get_named_volumes(buffers_partition, B)
# find omega and theta max
omega_max = [0, 0, 0]
T_max = [0, 0, 0]
for buffer_index in buffers_volumes.keys():
_3d_index = numeric_to_3d_pos(buffer_index,
buffers_partition,
order='F')
T, Cs = get_theta(buffers_volumes, buffer_index, _3d_index, O, B)
for i in range(3):
if Cs[i] > omega_max[i]:
omega_max[i] = Cs[i]
if T[i] > T_max[i]:
T_max[i] = T[i]
print("Omega max: ", omega_max)
nb_bytes_per_voxel = 2
buffersize = B[0] * B[1] * B[2]
n = R[2] / B[2]
N = R[1] / B[1] * R[2] / B[2]
def get_buff_to_vols(R, B, O, buffers_volumes, buffers_partition):
""" Outputs a dictionary associating buffer_index to list of Volumes indexed as in paper.
"""
def get_theta(buffers_volumes, buffer_index, _3d_index, O, B):
T = list()
Cs = list()
for dim in range(len(buffers_volumes[buffer_index].p1)):
if B[dim] < O[dim]:
C = 0
else:
C = ((_3d_index[dim]+1) * B[dim]) % O[dim]
print(f'{((_3d_index[dim]+1) * B[dim])}mod{O[dim]} = {C}')
if C == 0 and B[dim] != O[dim]: # particular case
C = O[dim]
if C < 0:
raise ValueError("modulo should not return negative value")
Cs.append(C)
T.append(B[dim] - C)
print(f'C: {Cs}')
print(f'theta: {T}')
return T
def first_sanity_check(buffers_volumes, buffer_index, volumes_list):
""" see if volumes coordinates found are inside buffer
"""
xs, ys, zs = list(), list(), list()
for volume in volumes_list:
x1, y1, z1 = volume.p1
x2, y2, z2 = volume.p2
xs.append(x1)
xs.append(x2)
ys.append(y1)
ys.append(y2)
zs.append(z1)
zs.append(z2)
err = -1
buff_vol = buffers_volumes[buffer_index]
if not min(xs) == buff_vol.p1[0]:
err = 0
if not min(ys) == buff_vol.p1[1]:
err = 1
if not min(zs) == buff_vol.p1[2]:
err = 2
if not max(xs) == buff_vol.p2[0]:
err = 3
if not max(ys) == buff_vol.p2[1]:
err = 4
if not max(zs) == buff_vol.p2[2]:
err = 5
if err > -1:
print(f'buffer lower corner: {buff_vol.p1}')
print(f'volumes lower corner: {(min(xs), min(ys), min(zs))}')
print(f'buffer upper corner: {buff_vol.p2}')
print(f'volumes upper corner: {(max(xs), max(ys), max(zs))}')
raise ValueError("[get_buff_to_vols] Error " + str(err))
def second_sanity_check(B, O, volumes_list):
""" see if sum of all volumes equals the volume of the buffer
+ see if each volume is <= volume of an output file as a volume cannot be bigger than an output file
"""
volumes_volume = 0
buffer_volume = B[0]*B[1]*B[2]
outfile_volume = O[0]*O[1]*O[2]
for volume in volumes_list:
x1, y1, z1 = volume.p1
x2, y2, z2 = volume.p2
vol = (x2-x1)*(y2-y1)*(z2-z1)
if vol > outfile_volume:
print(f'Outfile volume: {outfile_volume}')
print(f'Volume considered: {vol}')
raise ValueError("A volume should not be bigger than outfile")
volumes_volume += vol
if buffer_volume != volumes_volume:
print(f'Buffer volume: {buffer_volume}')
print(f'Sum of volumes: {volumes_volume}')
raise ValueError("sum of volumes should be equal to buffer volume")
logger.debug("== Function == get_buff_to_vols")
print("== Function == get_buff_to_vols")
buff_to_vols = dict()
rows = list()
for buffer_index in buffers_volumes.keys():
print(f'\nProcessing buffer {buffer_index}')
if DEBUG:
buffers_volumes[buffer_index].print()
_3d_index = numeric_to_3d_pos(buffer_index, buffers_partition, order='F')
T = get_theta(buffers_volumes, buffer_index, _3d_index, O, B)
volumes_list = get_main_volumes(B, T) # get coords in basis of buffer
add_offsets(volumes_list, _3d_index, B) # convert coords in basis of R - WARNING: important to be in this order, we need basis R for split_main_volumes
if DEBUG:
print('Main volumes found:')
for v in volumes_list:
v.print()
volumes_list = split_main_volumes(volumes_list, O) # seek for hidden volumes in main volumes
if DEBUG:
print('Split volumes found:')
for v in volumes_list:
v.print()
# hidd_vols = compute_hidden_volumes(T, O)
# add_offsets(hidd_vols, _3d_index, B)
# print('Hidden volumes found:')
# for v in hidd_vols:
# v.print()
# volumes_list = volumes_list + hidd_vols # still in basis of buffer
if DEBUG:
print('\nVolumes found:')
for v in volumes_list:
v.print()
print('\n')
first_sanity_check(buffers_volumes, buffer_index, volumes_list)
second_sanity_check(B, O, volumes_list)
buff_to_vols[buffer_index] = volumes_list
# debug csv file
for v in volumes_list:
rows.append((
(v.p1[1], v.p1[2]),
v.p2[1] - v.p1[1],
v.p2[2] - v.p1[2],
))
# debug csv file
columns = [
'bl_corner',
'width',
'height'
]
csv_path = '/tmp/compute_zones_buffervolumes.csv'
csv_out, writer = create_csv_file(csv_path, columns, delimiter=',', mode='w+')
for row in set(rows):
writer.writerow(row)
csv_out.close()
logger.debug("End\n")
return buff_to_vols