def test_save_do_not_replace(tmpdir): from binlog.model import Model env = lmdb.open(str(tmpdir), max_dbs=1) with env.begin(write=True) as txn: db = env.open_db(b'test', txn=txn) m = Model(data=0) assert m.save(0, db, txn) with env.begin(write=False) as txn: db = env.open_db(b'test', txn=txn) with txn.cursor(db) as cursor: first = cursor.get(struct.pack("!Q", 0)) with env.begin(write=True) as txn: db = env.open_db(b'test', txn=txn) m = Model(data=1) assert not m.save(0, db, txn) with env.begin(write=False) as txn: db = env.open_db(b'test', txn=txn) with txn.cursor(db) as cursor: second = cursor.get(struct.pack("!Q", 0)) assert first == second
def test_bulk_create_multiple_times(tmpdir): with Model.open(tmpdir) as db: entries1 = [Model(data=i) for i in range(10)] db.bulk_create(entries1) entries2 = [Model(data=i) for i in range(10, 20)] db.bulk_create(entries2) for entry in entries2: assert entry.saved assert entry.pk == entry['data']
def test_purge_with_multiple_reader(acked_list): with TemporaryDirectory() as tmpdir: with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(100)] db.bulk_create(entries) for i, acked in enumerate(acked_list): db.register_reader('myreader_%d' % i) with db.reader('myreader_%d' % i) as reader: for pk in acked: reader.ack(reader[pk]) common = reduce(op.and_, acked_list) removed, not_found = db.purge() assert removed == len(common) assert not_found == 0 with db.reader() as reader: for pk in range(100): if pk in common: with pytest.raises(IndexError): reader[pk] else: assert reader[pk]
def test_multiple_child_reader_recursive_ack(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(10)] db.bulk_create(entries) db.register_reader('parent.child.grandchild1') db.register_reader('parent.child.grandchild2') with db.reader('parent.child.grandchild1') as reader: for i in range(5, 10): reader.recursive_ack(reader[i]) with db.reader('parent.child.grandchild1') as reader: assert list(reader) == entries[:5] with db.reader('parent.child.grandchild2') as reader: for i in range(5): reader.recursive_ack(reader[i]) with db.reader('parent.child.grandchild2') as reader: assert list(reader) == entries[5:] with db.reader('parent.child') as reader: assert not list(reader) with db.reader('parent') as reader: assert not list(reader)
def test_purge_with_not_found(acked): from binlog.databases import Entries with TemporaryDirectory() as tmpdir: with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(100)] db.bulk_create(entries) db.register_reader('myreader') with db.reader('myreader') as reader: for pk in acked: reader.ack(reader[pk]) # Delete everything with db.data(write=True) as res: with Entries.cursor(res) as cursor: for pk in range(100): cursor.pop(pk) removed, not_found = db.purge(chunk_size=10) assert removed == 0 # Because purge now uses ANDS the Entries cursor with # `common_acked`, "not_found" always will be 0. assert not_found == 0
def test_clone_reader_inherits_progress(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(10)] db.bulk_create(entries) db.register_reader('reader1') with db.reader('reader1') as reader: for i in range(5): assert reader.ack(i) with db.reader('reader1') as reader: for a, b in zip(reader, range(5, 10)): assert a["idx"] == b db.clone_reader('reader1', 'reader2') with db.reader('reader2') as reader: for a, b in zip(reader, range(5, 10)): assert a["idx"] == b with db.reader('reader1') as reader: for i in reader: assert reader.ack(i) with db.reader('reader2') as reader: for a, b in zip(reader, range(5, 10)): assert a["idx"] == b
def test_reader_filter_exact(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i, even=(i % 2 == 0)) for i in range(100)] db.bulk_create(entries) with db.reader() as r: for a, b in zip_longest(r.filter(even=True), range(0, 100, 2)): assert a.pk == b
def test_slice_step_cannot_be_zero(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(4)] db.bulk_create(entries) with db.reader() as reader: with pytest.raises(ValueError): reader[::0]
def test_slice_negative_start_stop_step(tmpdir, start, stop, step): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(4)] db.bulk_create(entries) with db.reader() as reader: expected = list(entries[start:stop:step]) current = list(reader[start:stop:step]) assert expected == current
def test_reader_index(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(10)] db.bulk_create(entries) db.register_reader('myreader') with db.reader('myreader') as reader: for e in entries: assert e == reader[e['idx']]
def test_reader_reads_bulk_create(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(10)] db.bulk_create(entries) db.register_reader('myreader') with db.reader('myreader') as reader: for current, expected in zip_longest(reader, entries): assert current == expected
def test_bulk_create(tmpdir): with Model.open(tmpdir) as db: entries = [Model(data=i) for i in range(10)] db.bulk_create(entries) for entry in entries: assert entry.saved assert entry.pk == entry['data']
def test_reader_filter_exact_multiple(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i, fizz=(i % 3 == 0), buzz=(i % 5 == 0)) for i in range(100)] db.bulk_create(entries) with db.reader() as r: for a, b in zip_longest(r.filter(fizz=True, buzz=True), [x for x in range(0, 100) if x % 3 == x % 5 == 0]): assert a.pk == b
def test_model_have_meta_with_defaults(): from binlog.model import Model b = Model() assert b._meta['config_db_name'] == 'Config' assert b._meta['entries_db_name'] == 'Entries' assert b._meta['checkpoints_db_name'] == 'Checkpoints' assert b._meta['index_db_format'] == ('{model._meta[entries_db_name]}' '__idx__' '{index_name}') assert b._meta['readers_env_directory'] == 'readers' assert b._meta['data_env_directory'] == 'data'
def test_reader_negative_index(tmpdir): with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(10)] db.bulk_create(entries) db.register_reader('myreader') with db.reader('myreader') as reader: positive_indexes = range(len(entries)) negative_indexes = [(1 + i) * -1 for i in reversed(positive_indexes)] for pos, neg in zip(positive_indexes, negative_indexes): assert reader[pos] == reader[neg]
def test_save_store_data(tmpdir): from binlog.model import Model env = lmdb.open(str(tmpdir), max_dbs=1) with env.begin(write=True) as txn: db = env.open_db(b'test', txn=txn) m = Model(data=0) assert m.save(0, db, txn) with env.begin(write=False) as txn: db = env.open_db(b'test', txn=txn) with txn.cursor(db) as cursor: assert cursor.get(struct.pack("!Q", 0))
def test_reader_reads_except_acked(acks): with TemporaryDirectory() as tmpdir: with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(100)] db.bulk_create(entries) db.register_reader('myreader') with db.reader('myreader') as reader: for pk in acks: reader.ack(reader[pk]) expected = set(range(100)) - set(acks) current = {e.pk for e in reader} assert expected == current
def test_bulk_create_index_collision(tmpdir): from binlog.exceptions import IntegrityError env = lmdb.open(os.path.join(str(tmpdir), Model._meta["data_env_directory"]), max_dbs=1) with env.begin(write=True) as txn: entries_db = env.open_db(Model._meta["entries_db_name"].encode("utf-8"), txn=txn) with txn.cursor(entries_db) as cursor: raw = cursor.put(struct.pack("!Q", 5), b'I am not supposed to be here!') with Model.open(tmpdir) as db: entries = [Model(data=i) for i in range(10)] with pytest.raises(IntegrityError): db.bulk_create(entries)
def test_purge_with_one_reader(acked): with TemporaryDirectory() as tmpdir: with Model.open(tmpdir) as db: entries = [Model(idx=i) for i in range(100)] db.bulk_create(entries) db.register_reader('myreader') with db.reader('myreader') as reader: for pk in acked: reader.ack(reader[pk]) removed, not_found = db.purge() assert removed == len(acked) assert not_found == 0 with db.reader() as reader: for pk in range(100): if pk in acked: with pytest.raises(IndexError): reader[pk] else: assert reader[pk]
def test_reader_filter_exact_index_and_nonindex(tmpdir): from binlog.index import NumericIndex class MyModel(Model): fizz = NumericIndex(mandatory=True) with MyModel.open(tmpdir) as db: entries = [Model(idx=i, fizz=int(i % 3 == 0), buzz=int(i % 5 == 0)) for i in range(100)] assert db.bulk_create(entries) == len(entries) with db.reader() as r: current = r.filter(fizz=1, buzz=1) expected = [x for x in range(0, 100) if x % 3 == x % 5 == 0] for a, b in zip_longest(current, expected): if a is None or b is None: assert False, (a, b) else: assert a.pk == b
def test_ack_on_unsaved_event(tmpdir): with Model.open(tmpdir) as db: db.register_reader('myreader') with db.reader('myreader') as reader: with pytest.raises(ValueError): reader.ack(Model(test='data'))