def test_multiple_child_reader_recursive_ack(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('parent.child.grandchild1')
db.register_reader('parent.child.grandchild2')
with db.reader('parent.child.grandchild1') as reader:
for i in range(5, 10):
reader.recursive_ack(reader[i])
with db.reader('parent.child.grandchild1') as reader:
assert list(reader) == entries[:5]
with db.reader('parent.child.grandchild2') as reader:
for i in range(5):
reader.recursive_ack(reader[i])
with db.reader('parent.child.grandchild2') as reader:
assert list(reader) == entries[5:]
with db.reader('parent.child') as reader:
assert not list(reader)
with db.reader('parent') as reader:
assert not list(reader)
def test_save_do_not_replace(tmpdir):
from binlog.model import Model
env = lmdb.open(str(tmpdir), max_dbs=1)
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=0)
assert m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
first = cursor.get(struct.pack("!Q", 0))
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=1)
assert not m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
second = cursor.get(struct.pack("!Q", 0))
assert first == second
def test_purge_with_not_found(acked):
from binlog.databases import Entries
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for pk in acked:
reader.ack(reader[pk])
# Delete everything
with db.data(write=True) as res:
with Entries.cursor(res) as cursor:
for pk in range(100):
cursor.pop(pk)
removed, not_found = db.purge(chunk_size=10)
assert removed == 0
# Because purge now uses ANDS the Entries cursor with
# `common_acked`, "not_found" always will be 0.
assert not_found == 0
def test_purge_with_multiple_reader(acked_list):
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
for i, acked in enumerate(acked_list):
db.register_reader('myreader_%d' % i)
with db.reader('myreader_%d' % i) as reader:
for pk in acked:
reader.ack(reader[pk])
common = reduce(op.and_, acked_list)
removed, not_found = db.purge()
assert removed == len(common)
assert not_found == 0
with db.reader() as reader:
for pk in range(100):
if pk in common:
with pytest.raises(IndexError):
reader[pk]
else:
assert reader[pk]
def test_clone_reader_inherits_progress(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('reader1')
with db.reader('reader1') as reader:
for i in range(5):
assert reader.ack(i)
with db.reader('reader1') as reader:
for a, b in zip(reader, range(5, 10)):
assert a["idx"] == b
db.clone_reader('reader1', 'reader2')
with db.reader('reader2') as reader:
for a, b in zip(reader, range(5, 10)):
assert a["idx"] == b
with db.reader('reader1') as reader:
for i in reader:
assert reader.ack(i)
with db.reader('reader2') as reader:
for a, b in zip(reader, range(5, 10)):
assert a["idx"] == b
def test_reader_filter_exact(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i, even=(i % 2 == 0)) for i in range(100)]
db.bulk_create(entries)
with db.reader() as r:
for a, b in zip_longest(r.filter(even=True), range(0, 100, 2)):
assert a.pk == b
def test_slice_step_cannot_be_zero(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(4)]
db.bulk_create(entries)
with db.reader() as reader:
with pytest.raises(ValueError):
reader[::0]
def test_not_nested_inside_nested_connection(tmpdir):
with Model.open(tmpdir) as conn1:
with Model.open(tmpdir) as conn2_a:
assert conn1 is conn2_a
with Model.open(tmpdir) as conn2_b:
assert conn1 is conn2_b
def test_reader_reads_bulk_create(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for current, expected in zip_longest(reader, entries):
assert current == expected
def test_slice_negative_start_stop_step(tmpdir, start, stop, step):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(4)]
db.bulk_create(entries)
with db.reader() as reader:
expected = list(entries[start:stop:step])
current = list(reader[start:stop:step])
assert expected == current
def test_reader_index(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for e in entries:
assert e == reader[e['idx']]
def test_bulk_create(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(data=i) for i in range(10)]
db.bulk_create(entries)
for entry in entries:
assert entry.saved
assert entry.pk == entry['data']
def test_bulk_create_multiple_times(tmpdir):
with Model.open(tmpdir) as db:
entries1 = [Model(data=i) for i in range(10)]
db.bulk_create(entries1)
entries2 = [Model(data=i) for i in range(10, 20)]
db.bulk_create(entries2)
for entry in entries2:
assert entry.saved
assert entry.pk == entry['data']
def test_reader_filter_exact_multiple(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i,
fizz=(i % 3 == 0),
buzz=(i % 5 == 0)) for i in range(100)]
db.bulk_create(entries)
with db.reader() as r:
for a, b in zip_longest(r.filter(fizz=True, buzz=True),
[x for x in range(0, 100)
if x % 3 == x % 5 == 0]):
assert a.pk == b
def test_reader_negative_index(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
positive_indexes = range(len(entries))
negative_indexes = [(1 + i) * -1
for i in reversed(positive_indexes)]
for pos, neg in zip(positive_indexes, negative_indexes):
assert reader[pos] == reader[neg]
def test_save_store_data(tmpdir):
from binlog.model import Model
env = lmdb.open(str(tmpdir), max_dbs=1)
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=0)
assert m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
assert cursor.get(struct.pack("!Q", 0))
def test_save_store_data(tmpdir):
from binlog.model import Model
env = lmdb.open(str(tmpdir), max_dbs=1)
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=0)
assert m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
assert cursor.get(struct.pack("!Q", 0))
def test_reader_reads_except_acked(acks):
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for pk in acks:
reader.ack(reader[pk])
expected = set(range(100)) - set(acks)
current = {e.pk for e in reader}
assert expected == current
def test_cant_remove_if_is_not_acked(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
assert db.remove(e0) is False
def test_bulk_create_index_collision(tmpdir):
from binlog.exceptions import IntegrityError
env = lmdb.open(os.path.join(str(tmpdir),
Model._meta["data_env_directory"]), max_dbs=1)
with env.begin(write=True) as txn:
entries_db = env.open_db(Model._meta["entries_db_name"].encode("utf-8"),
txn=txn)
with txn.cursor(entries_db) as cursor:
raw = cursor.put(struct.pack("!Q", 5),
b'I am not supposed to be here!')
with Model.open(tmpdir) as db:
entries = [Model(data=i) for i in range(10)]
with pytest.raises(IntegrityError):
db.bulk_create(entries)
def test_cant_remove_if_is_not_acked(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
assert db.remove(e0) is False
def test_ack_on_anonymous_reader(tmpdir):
with Model.open(tmpdir) as db:
entry = db.create(test='data')
with db.reader() as reader:
with pytest.raises(RuntimeError):
reader.ack(reader[0])
def test_reader_unknown_negative_index(tmpdir):
with Model.open(tmpdir) as db:
db.create(test='test')
db.register_reader('myreader')
with db.reader('myreader') as reader:
with pytest.raises(IndexError):
reader[-2]
def test_reader_filter_exact(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i, even=(i % 2 == 0)) for i in range(100)]
db.bulk_create(entries)
with db.reader() as r:
for a, b in zip_longest(r.filter(even=True), range(0, 100, 2)):
assert a.pk == b
def test_can_ack_with_pk(tmpdir):
with Model.open(tmpdir) as db:
entry = db.create(test='data')
db.register_reader('myreader')
with db.reader('myreader') as reader:
reader.ack(0)
assert 0 in reader.registry
def test_cannot_use_same_connection_from_multiple_threads(tmpdir, io_method):
conn = Model.open(tmpdir)
with ThreadPoolExecutor(max_workers=1) as pool:
res = pool.submit(getattr(conn, io_method))
with pytest.raises(BadUsageError):
res.result()
def test_reader_unknown_negative_index(tmpdir):
with Model.open(tmpdir) as db:
db.create(test='test')
db.register_reader('myreader')
with db.reader('myreader') as reader:
with pytest.raises(IndexError):
reader[-2]
def test_create_is_incremental(tmpdir):
with Model.open(tmpdir) as db:
entry0 = db.create()
entry1 = db.create()
assert entry0.pk == 0
assert entry1.pk == 1
def test_purge_without_readers(tmpdir):
with Model.open(tmpdir) as db:
db.create(test='data')
removed, not_found = db.purge()
assert removed == not_found == 0
def test_list_readers(readers):
assume(len(readers & RESERVED_READER_NAMES) == 0)
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
for name in readers:
db.register_reader(name)
assert set(db.list_readers()) == readers
def test_list_readers(readers):
assume(len(readers & RESERVED_READER_NAMES) == 0)
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
for name in readers:
db.register_reader(name)
assert set(db.list_readers()) == readers
def test_connection_is_context_manager(tmpdir):
with patch('lmdb.open') as lmdbopen:
with Model.open(tmpdir) as conn:
assert not conn.closed
lmdbopen.assert_any_call(str(tmpdir) + "/readers", max_dbs=1)
lmdbopen.assert_any_call(str(tmpdir) + "/data", max_dbs=2)
assert conn.closed
def test_ack_is_not_effective_before_commit(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('myreader')
with db.reader('myreader') as reader1:
reader1.ack(e0)
with db.reader('myreader') as reader2:
assert list(reader2)
def test_reader_index(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for e in entries:
assert e == reader[e['idx']]
def test_can_remove_if_is_acked(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
with db.reader('reader1') as reader:
reader.ack(e0)
assert db.remove(e0) is True
def test_reader_reads_bulk_create(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for current, expected in zip_longest(reader, entries):
assert current == expected
def test_can_remove_if_is_acked(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
with db.reader('reader1') as reader:
reader.ack(e0)
assert db.remove(e0) is True
def test_list_readers_ignore_hints(tmpdir):
with Model.open(tmpdir) as db:
db.register_reader("myreader")
with db.reader("myreader") as r1:
# This will create the `hints` db.
r1._save_hint(0)
assert db.list_readers() == ["myreader"]
def test_child_reader(tmpdir):
with Model.open(tmpdir) as db:
db.register_reader('parent.child')
# SHOULD NOT RAISE
with db.reader('parent'):
pass
with db.reader('parent.child'):
pass
def test_remove_unsuccess(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
with db.reader('reader1') as reader1:
reader1.ack(e0)
db.remove(e0)
assert not db.remove(e0)
def test_create_returns_entry(tmpdir):
doc = {'key1': 'value1', 'key2': 'value2'}
with Model.open(tmpdir) as db:
entry = db.create(**doc)
assert entry == doc, entry
assert entry.pk == 0
assert entry.saved
def test_remove_unsuccess(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
with db.reader('reader1') as reader1:
reader1.ack(e0)
db.remove(e0)
assert not db.remove(e0)
def test_event_is_acked(tmpdir):
with Model.open(tmpdir) as db:
entry = db.create(test='data')
db.register_reader('myreader')
with db.reader('myreader') as reader:
e = reader[0]
assert e.pk not in reader.registry
reader.ack(reader[0])
assert e.pk in reader.registry
def test_reader_reads_create(entries):
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
for e in entries:
db.create(**e)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for current, expected in zip_longest(reader, entries):
assert current == expected
def test_empty_iterator(tmpdir):
with Model.open(tmpdir) as db:
# Create and delete one register
db.create(test="data")
with db.data(write=True) as res:
with Entries.cursor(res) as cursor:
assert cursor.pop(0)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for _ in reader:
assert False, "SHOULD be empty"
def test_remove_delete_entry(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
db.register_reader('reader1')
with db.reader('reader1') as reader1:
reader1.ack(e0)
db.remove(e0)
db.register_reader('reader2')
with db.reader('reader2') as reader2:
assert not list(reader2)
def test_ReadonlyError_is_masked(tmpdir):
with Model.open(tmpdir) as db:
db.register_reader('myreader')
with db.reader('myreader') as reader:
with pytest.raises(IndexError):
reader[0]
with pytest.raises(StopIteration):
next(reader.__iter__())
with pytest.raises(StopIteration):
next(reader.__reversed__())
def test_multiple_instances_of_same_reader_can_ack_safe4(tmpdir):
with Model.open(tmpdir) as db:
e0 = db.create(test='data0')
e1 = db.create(test='data1')
db.register_reader('myreader')
with db.reader('myreader') as reader1:
with db.reader('myreader') as reader2:
reader2.ack(e1)
reader1.ack(e0)
with db.reader('myreader') as reader:
assert not list(reader)
def test_acked_event_persist_after_reader_is_closed(tmpdir):
with Model.open(tmpdir) as db:
entry = db.create(test='data')
db.register_reader('myreader')
with db.reader('myreader') as reader:
e = reader[0]
reader.ack(e)
# Reader commits registry on exit
with db.reader('myreader') as reader:
assert e.pk in reader.registry
