def test_read_bitpacked():
results = np.empty(1000000, dtype=np.int32)
with open(os.path.join(TEST_DATA, 'bitpack')) as f:
for counter, l in enumerate(f):
if counter > count:
break
raw, head, wid, res = eval(l)
i = encoding.NumpyIO(raw)
o = encoding.NumpyIO(results.view('uint8'))
encoding.read_bitpacked(i, head, wid, o, itemsize=4)
assert (res == results[:len(res)]).all()
assert o.tell() == len(res) * 4
def test_read_bitpacked_uint():
results = np.empty(1000000, dtype=np.uint8)
with open(os.path.join(TEST_DATA, 'bitpack')) as f:
for counter, l in enumerate(f):
if counter > count:
break
raw, head, wid, res = eval(l)
if wid > 7:
continue
print(wid)
i = encoding.NumpyIO(raw)
o = encoding.NumpyIO(results)
encoding.read_bitpacked(i, head, wid, o, itemsize=1)
assert (res == results[:len(res)]).all()
assert o.tell() == len(res)
def test_hybrid():
results = np.empty(1000000, dtype=np.int32)
with open(os.path.join(TEST_DATA, 'hybrid')) as f:
for counter, l in enumerate(f):
if counter > count // 20:
break
(data, width, length, res) = eval(l)
i = encoding.NumpyIO(data)
o = encoding.NumpyIO(results.view('uint8'))
encoding.read_rle_bit_packed_hybrid(i,
width,
length or 0,
o,
itemsize=4)
out = np.frombuffer(o.so_far(), dtype="int32")
cond = (res == out).all()
assert cond
def test_uvarint():
with open(os.path.join(TEST_DATA, 'uvarint')) as f:
for i, l in enumerate(f):
if i > count:
break
data, res = eval(l)
i = encoding.NumpyIO(data)
o = encoding.read_unsigned_var_int(i)
assert (res == o)
def test_read_data():
with open(os.path.join(TEST_DATA, 'read_data')) as f:
for i, l in enumerate(f):
(data, fo_encoding, value_count, bit_width, res) = eval(l)
i = encoding.NumpyIO(data)
out = core.read_data(i, fo_encoding, value_count, bit_width)
for o, r in zip(out, res):
# result from old version is sometimes 1 value too long
assert o == r
def test_hybrid_extra_bytes():
results = np.empty(1000000, dtype=np.int32)
with open(os.path.join(TEST_DATA, 'hybrid')) as f:
for i, l in enumerate(f):
if i > count // 20:
break
(data, width, length, res) = eval(l)
if length is not None:
data2 = data + b'extra bytes'
else:
continue
i = encoding.NumpyIO(data2)
o = encoding.NumpyIO(results.view("uint8"))
encoding.read_rle_bit_packed_hybrid(i,
width,
length,
o,
itemsize=4)
out = np.frombuffer(o.so_far(), dtype="int32")
cond = (res == out).all()
assert cond
assert i.tell() == len(data)