def test_roundtrip():
_rt(columns.VarBytesColumn(),
[b("a"), b("ccc"), b("bbb"),
b("e"), b("dd")], b(""))
_rt(columns.FixedBytesColumn(5), [
b("aaaaa"), b("eeeee"),
b("ccccc"), b("bbbbb"),
b("eeeee")
],
b("\x00") * 5)
_rt(columns.RefBytesColumn(),
[b("a"), b("ccc"),
b("bb"), b("ccc"),
b("a"), b("bb")], b(""))
_rt(columns.RefBytesColumn(3),
[b("aaa"), b("bbb"),
b("ccc"), b("aaa"),
b("bbb"), b("ccc")],
b("\x00") * 3)
_rt(columns.StructColumn("ifH", (0, 0.0, 0)), [(100, 1.5, 15000),
(-100, -5.0, 0),
(5820, 6.5, 462),
(-57829, -1.5, 6),
(0, 0, 0)], (0, 0.0, 0))
numcol = columns.NumericColumn
_rt(numcol("b"), [10, -20, 30, -25, 15], 0)
_rt(numcol("B"), [10, 20, 30, 25, 15], 0)
_rt(numcol("h"), [1000, -2000, 3000, -15000, 32000], 0)
_rt(numcol("H"), [1000, 2000, 3000, 15000, 50000], 0)
_rt(numcol("i"), [2**16, -(2**20), 2**24, -(2**28), 2**30], 0)
_rt(numcol("I"), [2**16, 2**20, 2**24, 2**28, 2**31 & 0xFFFFFFFF], 0)
_rt(numcol("q"), [10, -20, 30, -25, 15], 0)
_rt(numcol("Q"), [2**35, 2**40, 2**48, 2**52, 2**63], 0)
_rt(numcol("f"), [1.5, -2.5, 3.5, -4.5, 1.25], 0)
_rt(numcol("d"), [1.5, -2.5, 3.5, -4.5, 1.25], 0)
c = columns.BitColumn(compress_at=10)
_rt(c, [bool(random.randint(0, 1)) for _ in xrange(70)], False)
_rt(c, [bool(random.randint(0, 1)) for _ in xrange(90)], False)
c = columns.PickleColumn(columns.VarBytesColumn())
_rt(c, [None, True, False, 100, -7, "hello"], None)
c = columns.VarBytesListColumn()
_rt(c, [[b('garnet'), b('amethyst')], [b('pearl')]], [])
c = columns.VarBytesListColumn()
c = columns.FixedBytesListColumn(4)
_rt(c, [[b('garn'), b('amet')], [b('pear')]], [])
def test_column_field():
schema = fields.Schema(a=fields.TEXT(sortable=True),
b=fields.COLUMN(columns.RefBytesColumn()))
with TempIndex(schema, "columnfield") as ix:
with ix.writer(codec=W3Codec()) as w:
w.add_document(a=u("alfa bravo"), b=b("charlie delta"))
w.add_document(a=u("bravo charlie"), b=b("delta echo"))
w.add_document(a=u("charlie delta"), b=b("echo foxtrot"))
with ix.reader() as r:
assert r.has_column("a")
assert r.has_column("b")
cra = r.column_reader("a")
assert cra[0] == u("alfa bravo")
assert type(cra[0]) == text_type
crb = r.column_reader("b")
assert crb[0] == b("charlie delta")
assert type(crb[0]) == bytes_type
def test_ref_switch():
import warnings
col = columns.RefBytesColumn()
def rw(size):
st = RamStorage()
f = st.create_file("test")
cw = col.writer(f)
for i in xrange(size):
cw.add(i, hex(i).encode("latin1"))
cw.finish(size)
length = f.tell()
f.close()
f = st.open_file("test")
cr = col.reader(f, 0, length, size)
for i in xrange(size):
v = cr[i]
# Column ignores additional unique values after 65535
if i <= 65535 - 1:
assert v == hex(i).encode("latin1")
else:
assert v == b('')
f.close()
rw(255)
# warnings.catch_warnings is not available in Python 2.5
if hasattr(warnings, "catch_warnings"):
# Column warns on additional unique values after 65535
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
rw(65537)
assert len(w) == 2
assert issubclass(w[-1].category, UserWarning)