def get_write_handle(self):
# FIXME: make example executable? involves temporary directory, source dataset, ...
# TODO: Direct I/O writing support
"""
Get a handle to write to this partition. Current rules:
1) You can only write a complete partition at once, which is then immutable afterwards.
If you want to change something, you need to write the whole partition again.
2) Once the `with`-block is exited successfully, the data is written down to disk. If
there is an error while writing the data, the partition will not be moved into its final
place and it will be missing from the data set. There cannot be a "parially written"
partition.
Example
-------
>>> with dest_part.get_write_handle() as wh: # doctest: +SKIP
... for tile in wh.write_tiles(source_part.get_tiles()):
... pass # do something with `tile`
"""
tmp_path = os.path.dirname(self._path)
return WriteHandle(
path=self._path,
tmp_base_path=tmp_path,
dtype=self.dtype,
part_slice=self.slice,
)
def test_write_handle(tmpdir_factory):
"""
test the common "happy path":
"""
datadir = tmpdir_factory.mktemp('write_handle_tests')
full_path = os.path.join(datadir, "f1")
part_slice = Slice(
shape=Shape((32, 64, 64), sig_dims=2),
origin=(16, 0, 0),
)
tile_slice = Slice(
shape=Shape((3, 64, 64), sig_dims=2),
origin=(19, 0, 0),
)
tile_data = np.random.randn(3, 64, 64).astype("float32")
tile = DataTile(
tile_data,
tile_slice=tile_slice,
scheme_idx=0,
)
wh = WriteHandle(full_path, datadir, part_slice, dtype='float32')
tmp_file_name = ""
with wh:
wh.write_tile(tile)
tmp_file_name = wh._tmp_file.name
assert os.path.exists(tmp_file_name)
# check some internals:
assert wh._dest is None
assert wh._tmp_file is None
# the temporary file should no longer exist in case of success
assert not os.path.exists(tmp_file_name)
# ... buf our dest fname should:
assert os.path.exists(full_path)
assert os.path.isfile(full_path)
# check if data is written correctly:
read_data = np.fromfile(full_path,
dtype="float32").reshape(part_slice.shape)
assert np.allclose(read_data[3:6, ...], tile_data)
def test_write_handle_exception(tmpdir_factory):
datadir = tmpdir_factory.mktemp('write_handle_tests')
full_path = os.path.join(datadir, "f1")
part_slice = Slice(
shape=Shape((32, 64, 64), sig_dims=2),
origin=(16, 0, 0),
)
tile_slice = Slice(
shape=Shape((3, 64, 64), sig_dims=2),
origin=(19, 0, 0),
)
tile_data = np.random.randn(3, 64, 64).astype("float32")
tile = DataTile(
tile_data,
tile_slice=tile_slice,
scheme_idx=0,
)
wh = WriteHandle(full_path, datadir, part_slice, dtype='float32')
tmp_file_name = ""
with pytest.raises(Exception):
with wh:
wh.write_tile(tile)
tmp_file_name = wh._tmp_file.name
assert os.path.exists(tmp_file_name)
raise Exception("nope")
# check some internals:
assert wh._dest is None
assert wh._tmp_file is None
# the temporary file should no longer exist in case of exception
assert not os.path.exists(tmp_file_name)
# and neither should the full destination path
assert not os.path.exists(full_path)