class TestConfig():
def __init__(self, params, paths):
self.params = params
self.create_split_case(params, paths)
self.create_merge_case(params, paths)
def create_split_case(self, params, paths):
try:
if params["hardware"] == "hdd":
self.hardware_path = paths["hdd_path"]
else:
self.hardware_path = paths["ssd_path"]
self.cuboid_filepath = os.path.join(
self.hardware_path, params["cuboid_name"] + ".hdf5")
self.splitcase = Split(self.cuboid_filepath, params["chunk_shape"])
self.splitcase.split_hdf5_multiple(self.hardware_path,
nb_blocks=None)
except Exception as e:
print(traceback.format_exc())
print("Something went wrong while creating case config.")
exit(1)
def create_merge_case(self, params, paths):
try:
if params["hardware"] == "hdd":
self.hardware_path = paths["hdd_path"]
else:
self.hardware_path = paths["ssd_path"]
self.merge_filepath = os.path.join(self.hardware_path,
"merged.hdf5")
self.mergecase = Merge(self.merge_filepath)
self.mergecase.merge_hdf5_multiple(self.hardware_path,
data_key='/data',
store=True)
except Exception as e:
print(traceback.format_exc())
print("Something went wrong while creating case config.")
exit(1)
def print_config(self):
print(f'\n-------------------')
print(f'Test configuration')
print(f'-------------------')
print(f'\nTest configurations:')
print(f'\tHardware: {self.params["hardware"]}')
print(f'\tCuboid name: {self.params["cuboid_name"]}')
print(f'\tCuboid shape: "{self.params["array_shape"]}"')
print(f'\tChunk shape: "{self.params["chunk_shape"]}"')
print(f'\tChunk type: "{self.params["chunk_type"]}"')
print(f'\nDask configuration:')
print(f'\tOptimization enabled: {self.params["optimized"]}')
print(f'\tBuffer size: {self.params["buffer_size"]} bytes')
print(f'\tNb threads: {self.params["nthreads"]}')
return
def split(datadir, filepath, cs, split_files=True):
"""
Arguments:
----------
split_files: if true then perform a split into multiple files, if false then perform a split inside one hdf5 file
"""
print("Splitting...")
splitcase = Split(filepath, chunk_shapes[cs])
if split_files:
splitcase.split_hdf5_multiple(datadir, nb_blocks=None)
else:
out_filepath = os.path.join(datadir, "split.hdf5")
splitcase.split_hdf5(out_filepath, nb_blocks=None)
arr = splitcase.get()
try:
with dask.config.set(scheduler='single-threaded'):
tsplit = run(arr)
splitcase.clean()
return tsplit
except Exception as e:
print(e, "\nOops something went wrong... Aborting.")
splitcase.clean()
sys.exit(1)