def test_closed(blocksize):
raw = io.BytesIO()
b = BlockBuffer(raw, blocksize)
b.close()
assert b.closed
assert raw.closed
with pytest.raises(ValueError, match="I/O operation on closed file."):
b.size
with pytest.raises(ValueError, match="I/O operation on closed file."):
b.seekable()
with pytest.raises(ValueError, match="I/O operation on closed file."):
b.readable()
with pytest.raises(ValueError, match="I/O operation on closed file."):
b.tell()
with pytest.raises(ValueError, match="I/O operation on closed file."):
b.seek(0)
with pytest.raises(ValueError, match="I/O operation on closed file."):
b.read()
# closing twice works
b.close()
def get_physical_partition_stats(metapartitions, store):
"""
Get statistics for partition.
.. hint::
To get the metapartitions pre-aligned, use ``concat_partitions_on_primary_index=True`` during dispatch.
Parameters
----------
metapartitions: Iterable[kartothek.io_components.metapartition.MetaPartition]
Iterable of metapartitions belonging to the same physical partition.
store: Union[simplekv.KeyValueStore, Callable[[], simplekv.KeyValueStore]]
KV store.
Returns
-------
stats: Dict[str, int]
Statistics for the current partition.
"""
if callable(store):
store = store()
files = 0
blobsize = 0
rows = 0
for mp in metapartitions:
for f in mp.files.values():
files += 1
fp = BlockBuffer(store.open(f))
try:
fp_parquet = pq.ParquetFile(fp)
rows += fp_parquet.metadata.num_rows
blobsize += fp.size
finally:
fp.close()
return {
"blobsize": blobsize,
"files": files,
"partitions": 1,
"rows": rows
}
def test_real_file(tmpdir, blocksize):
path = tmpdir.join("test_real_file.bin").strpath
with open(path, "wb") as fp:
fp.write(b"foxbar")
real_file = open(path, "rb")
b = BlockBuffer(real_file, blocksize)
assert not b.closed
assert b.size == 6
assert b.seekable() is True
assert b.readable() is True
assert b.tell() == 0
assert b.seek(1) == 1
assert b.read() == b"oxbar"
# final close
b.close()
# closing twice works
b.close()
