def get_write_handle(self): # FIXME: make example executable? involves temporary directory, source dataset, ... # TODO: Direct I/O writing support """ Get a handle to write to this partition. Current rules: 1) You can only write a complete partition at once, which is then immutable afterwards. If you want to change something, you need to write the whole partition again. 2) Once the `with`-block is exited successfully, the data is written down to disk. If there is an error while writing the data, the partition will not be moved into its final place and it will be missing from the data set. There cannot be a "parially written" partition. Example ------- >>> with dest_part.get_write_handle() as wh: # doctest: +SKIP ... for tile in wh.write_tiles(source_part.get_tiles()): ... pass # do something with `tile` """ tmp_path = os.path.dirname(self._path) return WriteHandle( path=self._path, tmp_base_path=tmp_path, dtype=self.dtype, part_slice=self.slice, )
def test_write_handle(tmpdir_factory): """ test the common "happy path": """ datadir = tmpdir_factory.mktemp('write_handle_tests') full_path = os.path.join(datadir, "f1") part_slice = Slice( shape=Shape((32, 64, 64), sig_dims=2), origin=(16, 0, 0), ) tile_slice = Slice( shape=Shape((3, 64, 64), sig_dims=2), origin=(19, 0, 0), ) tile_data = np.random.randn(3, 64, 64).astype("float32") tile = DataTile( tile_data, tile_slice=tile_slice, scheme_idx=0, ) wh = WriteHandle(full_path, datadir, part_slice, dtype='float32') tmp_file_name = "" with wh: wh.write_tile(tile) tmp_file_name = wh._tmp_file.name assert os.path.exists(tmp_file_name) # check some internals: assert wh._dest is None assert wh._tmp_file is None # the temporary file should no longer exist in case of success assert not os.path.exists(tmp_file_name) # ... buf our dest fname should: assert os.path.exists(full_path) assert os.path.isfile(full_path) # check if data is written correctly: read_data = np.fromfile(full_path, dtype="float32").reshape(part_slice.shape) assert np.allclose(read_data[3:6, ...], tile_data)
def test_write_handle_exception(tmpdir_factory): datadir = tmpdir_factory.mktemp('write_handle_tests') full_path = os.path.join(datadir, "f1") part_slice = Slice( shape=Shape((32, 64, 64), sig_dims=2), origin=(16, 0, 0), ) tile_slice = Slice( shape=Shape((3, 64, 64), sig_dims=2), origin=(19, 0, 0), ) tile_data = np.random.randn(3, 64, 64).astype("float32") tile = DataTile( tile_data, tile_slice=tile_slice, scheme_idx=0, ) wh = WriteHandle(full_path, datadir, part_slice, dtype='float32') tmp_file_name = "" with pytest.raises(Exception): with wh: wh.write_tile(tile) tmp_file_name = wh._tmp_file.name assert os.path.exists(tmp_file_name) raise Exception("nope") # check some internals: assert wh._dest is None assert wh._tmp_file is None # the temporary file should no longer exist in case of exception assert not os.path.exists(tmp_file_name) # and neither should the full destination path assert not os.path.exists(full_path)