def test_cache(tmpdir):
path = str(tmpdir.join('test.txt'))
data = b'1234567890'
with open(path, 'wb') as f:
f.write(data)
fake_path = 's3://vaex/test.hdf5?profile_name=foo'
with open(path, 'rb') as fp:
cache = vaex.file.cache.CachedFile(fp,
fake_path,
str(tmpdir),
block_size=2)
assert cache.tell() == 0
assert cache.read(1) == b'1'
assert cache.block_reads == 1
assert cache.reads == 1
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 0
assert cache.tell() == 1
assert cache.read(1) == b'2'
assert cache.block_reads == 1
assert cache.reads == 1
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 0
assert cache.tell() == 2
cache.seek(4)
assert cache.read(1) == b'5'
assert cache.block_reads == 2
assert cache.reads == 2
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.data_file.data[3] == 0
assert cache.data_file.data[4] == ord('5')
assert cache.data_file.data[5] == ord('6')
assert cache.data_file.data[6] == 0
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 1
assert cache.mask_file.data[3] == 0
assert cache.tell() == 5
cache.seek(8)
assert cache.read(2) == b'90'
assert cache.block_reads == 3
assert cache.reads == 3
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.data_file.data[3] == 0
assert cache.data_file.data[4] == ord('5')
assert cache.data_file.data[5] == ord('6')
assert cache.data_file.data[6] == 0
assert cache.data_file.data[7] == 0
assert cache.data_file.data[8] == ord('9')
assert cache.data_file.data[9] == ord('0')
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 1
assert cache.mask_file.data[3] == 0
assert cache.mask_file.data[4] == 1
cache.seek(1)
assert cache.read(8) == b'23456789'
assert cache.block_reads == 5
assert cache.reads == 5
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == ord('3')
assert cache.data_file.data[3] == ord('4')
assert cache.data_file.data[4] == ord('5')
assert cache.data_file.data[5] == ord('6')
assert cache.data_file.data[6] == ord('7')
assert cache.data_file.data[7] == ord('8')
assert cache.data_file.data[8] == ord('9')
assert cache.data_file.data[9] == ord('0')
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 1
assert cache.mask_file.data[2] == 1
assert cache.mask_file.data[3] == 1
assert cache.mask_file.data[4] == 1
# reset
cache.data_file.data[:] = ord('x')
cache.mask_file.data[:] = 0
cache.block_reads = 0
cache.reads = 0
cache.seek(3)
assert cache.read(2) == b'45'
assert cache.mask_file.data.tolist() == [0, 1, 1, 0, 0]
assert cache.block_reads == 2
assert cache.reads == 1
cache.seek(0)
# this should do two reads, once of the first block, and then the last two at once
assert cache.read(10) == b'1234567890'
assert cache.block_reads == 5
assert cache.reads == 3
assert cache.mask_file.data.tolist() == [1, 1, 1, 1, 1]
def test_cache(tmpdir):
path = str(tmpdir.join('test.txt'))
data = b'1234567890'
with open(path, 'wb') as f:
f.write(data)
fake_path = 's3://vaex/test.hdf5?profile_name=foo'
with open(path, 'rb') as fp:
cache = vaex.file.cache.CachedFile(fp,
fake_path,
str(tmpdir),
block_size=2)
assert cache.tell() == 0
assert cache.read(1) == b'1'
assert cache.block_reads == 1
assert cache.reads == 1
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 0
assert cache.tell() == 1
assert cache.read(1) == b'2'
assert cache.block_reads == 1
assert cache.reads == 1
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 0
assert cache.tell() == 2
cache.seek(4)
assert cache.read(1) == b'5'
assert cache.block_reads == 2
assert cache.reads == 2
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.data_file.data[3] == 0
assert cache.data_file.data[4] == ord('5')
assert cache.data_file.data[5] == ord('6')
assert cache.data_file.data[6] == 0
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 1
assert cache.mask_file.data[3] == 0
assert cache.tell() == 5
cache.seek(8)
assert cache.read(2) == b'90'
assert cache.block_reads == 3
assert cache.reads == 3
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == 0
assert cache.data_file.data[3] == 0
assert cache.data_file.data[4] == ord('5')
assert cache.data_file.data[5] == ord('6')
assert cache.data_file.data[6] == 0
assert cache.data_file.data[7] == 0
assert cache.data_file.data[8] == ord('9')
assert cache.data_file.data[9] == ord('0')
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 0
assert cache.mask_file.data[2] == 1
assert cache.mask_file.data[3] == 0
assert cache.mask_file.data[4] == 1
cache.seek(1)
assert cache.read(8) == b'23456789'
assert cache.block_reads == 5
assert cache.reads == 5
assert cache.data_file.data[0] == ord('1')
assert cache.data_file.data[1] == ord('2')
assert cache.data_file.data[2] == ord('3')
assert cache.data_file.data[3] == ord('4')
assert cache.data_file.data[4] == ord('5')
assert cache.data_file.data[5] == ord('6')
assert cache.data_file.data[6] == ord('7')
assert cache.data_file.data[7] == ord('8')
assert cache.data_file.data[8] == ord('9')
assert cache.data_file.data[9] == ord('0')
assert cache.mask_file.data[0] == 1
assert cache.mask_file.data[1] == 1
assert cache.mask_file.data[2] == 1
assert cache.mask_file.data[3] == 1
assert cache.mask_file.data[4] == 1