def test_save_do_not_replace(tmpdir):
from binlog.model import Model
env = lmdb.open(str(tmpdir), max_dbs=1)
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=0)
assert m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
first = cursor.get(struct.pack("!Q", 0))
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=1)
assert not m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
second = cursor.get(struct.pack("!Q", 0))
assert first == second
def test_bulk_create_multiple_times(tmpdir):
with Model.open(tmpdir) as db:
entries1 = [Model(data=i) for i in range(10)]
db.bulk_create(entries1)
entries2 = [Model(data=i) for i in range(10, 20)]
db.bulk_create(entries2)
for entry in entries2:
assert entry.saved
assert entry.pk == entry['data']
def test_purge_with_multiple_reader(acked_list):
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
for i, acked in enumerate(acked_list):
db.register_reader('myreader_%d' % i)
with db.reader('myreader_%d' % i) as reader:
for pk in acked:
reader.ack(reader[pk])
common = reduce(op.and_, acked_list)
removed, not_found = db.purge()
assert removed == len(common)
assert not_found == 0
with db.reader() as reader:
for pk in range(100):
if pk in common:
with pytest.raises(IndexError):
reader[pk]
else:
assert reader[pk]
def test_multiple_child_reader_recursive_ack(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('parent.child.grandchild1')
db.register_reader('parent.child.grandchild2')
with db.reader('parent.child.grandchild1') as reader:
for i in range(5, 10):
reader.recursive_ack(reader[i])
with db.reader('parent.child.grandchild1') as reader:
assert list(reader) == entries[:5]
with db.reader('parent.child.grandchild2') as reader:
for i in range(5):
reader.recursive_ack(reader[i])
with db.reader('parent.child.grandchild2') as reader:
assert list(reader) == entries[5:]
with db.reader('parent.child') as reader:
assert not list(reader)
with db.reader('parent') as reader:
assert not list(reader)
def test_purge_with_not_found(acked):
from binlog.databases import Entries
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for pk in acked:
reader.ack(reader[pk])
# Delete everything
with db.data(write=True) as res:
with Entries.cursor(res) as cursor:
for pk in range(100):
cursor.pop(pk)
removed, not_found = db.purge(chunk_size=10)
assert removed == 0
# Because purge now uses ANDS the Entries cursor with
# `common_acked`, "not_found" always will be 0.
assert not_found == 0
def test_clone_reader_inherits_progress(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('reader1')
with db.reader('reader1') as reader:
for i in range(5):
assert reader.ack(i)
with db.reader('reader1') as reader:
for a, b in zip(reader, range(5, 10)):
assert a["idx"] == b
db.clone_reader('reader1', 'reader2')
with db.reader('reader2') as reader:
for a, b in zip(reader, range(5, 10)):
assert a["idx"] == b
with db.reader('reader1') as reader:
for i in reader:
assert reader.ack(i)
with db.reader('reader2') as reader:
for a, b in zip(reader, range(5, 10)):
assert a["idx"] == b
def test_reader_filter_exact(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i, even=(i % 2 == 0)) for i in range(100)]
db.bulk_create(entries)
with db.reader() as r:
for a, b in zip_longest(r.filter(even=True), range(0, 100, 2)):
assert a.pk == b
def test_slice_step_cannot_be_zero(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(4)]
db.bulk_create(entries)
with db.reader() as reader:
with pytest.raises(ValueError):
reader[::0]
def test_slice_negative_start_stop_step(tmpdir, start, stop, step):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(4)]
db.bulk_create(entries)
with db.reader() as reader:
expected = list(entries[start:stop:step])
current = list(reader[start:stop:step])
assert expected == current
def test_reader_index(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for e in entries:
assert e == reader[e['idx']]
def test_reader_reads_bulk_create(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for current, expected in zip_longest(reader, entries):
assert current == expected
def test_bulk_create(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(data=i) for i in range(10)]
db.bulk_create(entries)
for entry in entries:
assert entry.saved
assert entry.pk == entry['data']
def test_reader_filter_exact_multiple(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i,
fizz=(i % 3 == 0),
buzz=(i % 5 == 0)) for i in range(100)]
db.bulk_create(entries)
with db.reader() as r:
for a, b in zip_longest(r.filter(fizz=True, buzz=True),
[x for x in range(0, 100)
if x % 3 == x % 5 == 0]):
assert a.pk == b
def test_model_have_meta_with_defaults():
from binlog.model import Model
b = Model()
assert b._meta['config_db_name'] == 'Config'
assert b._meta['entries_db_name'] == 'Entries'
assert b._meta['checkpoints_db_name'] == 'Checkpoints'
assert b._meta['index_db_format'] == ('{model._meta[entries_db_name]}'
'__idx__'
'{index_name}')
assert b._meta['readers_env_directory'] == 'readers'
assert b._meta['data_env_directory'] == 'data'
def test_reader_negative_index(tmpdir):
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(10)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
positive_indexes = range(len(entries))
negative_indexes = [(1 + i) * -1
for i in reversed(positive_indexes)]
for pos, neg in zip(positive_indexes, negative_indexes):
assert reader[pos] == reader[neg]
def test_save_store_data(tmpdir):
from binlog.model import Model
env = lmdb.open(str(tmpdir), max_dbs=1)
with env.begin(write=True) as txn:
db = env.open_db(b'test', txn=txn)
m = Model(data=0)
assert m.save(0, db, txn)
with env.begin(write=False) as txn:
db = env.open_db(b'test', txn=txn)
with txn.cursor(db) as cursor:
assert cursor.get(struct.pack("!Q", 0))
def test_reader_reads_except_acked(acks):
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for pk in acks:
reader.ack(reader[pk])
expected = set(range(100)) - set(acks)
current = {e.pk for e in reader}
assert expected == current
def test_bulk_create_index_collision(tmpdir):
from binlog.exceptions import IntegrityError
env = lmdb.open(os.path.join(str(tmpdir),
Model._meta["data_env_directory"]), max_dbs=1)
with env.begin(write=True) as txn:
entries_db = env.open_db(Model._meta["entries_db_name"].encode("utf-8"),
txn=txn)
with txn.cursor(entries_db) as cursor:
raw = cursor.put(struct.pack("!Q", 5),
b'I am not supposed to be here!')
with Model.open(tmpdir) as db:
entries = [Model(data=i) for i in range(10)]
with pytest.raises(IntegrityError):
db.bulk_create(entries)
def test_purge_with_one_reader(acked):
with TemporaryDirectory() as tmpdir:
with Model.open(tmpdir) as db:
entries = [Model(idx=i) for i in range(100)]
db.bulk_create(entries)
db.register_reader('myreader')
with db.reader('myreader') as reader:
for pk in acked:
reader.ack(reader[pk])
removed, not_found = db.purge()
assert removed == len(acked)
assert not_found == 0
with db.reader() as reader:
for pk in range(100):
if pk in acked:
with pytest.raises(IndexError):
reader[pk]
else:
assert reader[pk]
def test_reader_filter_exact_index_and_nonindex(tmpdir):
from binlog.index import NumericIndex
class MyModel(Model):
fizz = NumericIndex(mandatory=True)
with MyModel.open(tmpdir) as db:
entries = [Model(idx=i,
fizz=int(i % 3 == 0),
buzz=int(i % 5 == 0)) for i in range(100)]
assert db.bulk_create(entries) == len(entries)
with db.reader() as r:
current = r.filter(fizz=1, buzz=1)
expected = [x for x in range(0, 100) if x % 3 == x % 5 == 0]
for a, b in zip_longest(current, expected):
if a is None or b is None:
assert False, (a, b)
else:
assert a.pk == b
def test_ack_on_unsaved_event(tmpdir):
with Model.open(tmpdir) as db:
db.register_reader('myreader')
with db.reader('myreader') as reader:
with pytest.raises(ValueError):
reader.ack(Model(test='data'))