def test_truncate_table_with_index(testset1):
"""
Fill a table with records that has indexes, truncate the table and check that
all index records have been deleted as well.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
with db.begin(write=True) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
stats = zlmdb.TransactionStats()
with zlmdb.Database(dbpath) as db:
with db.begin(write=True, stats=stats) as txn:
records = schema.users.truncate(txn)
assert records == len(testset1) * (
len(schema.users.indexes()) + 1)
assert stats.dels == records
assert stats.puts == 0
def test_delete_nonindexes2(testset1):
"""
WARNING: quadratic run-time (in testset size)
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
with db.begin(write=True) as txn:
for j in range(10):
fullset = set(range(j * 100, (j + 1) * 100))
for i in range(100):
user_oid = j * 100 + i
del schema.users[txn, user_oid]
fullset.discard(user_oid)
user_oids = set(
schema.idx_users_by_realm.select(txn,
return_keys=False,
from_key=(j, 0),
to_key=(j + 1,
0)))
assert fullset == user_oids
def test_truncate_table():
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = Schema1()
stats = zlmdb.TransactionStats()
tabs = [
schema.tab_oid_json,
schema.tab_str_json,
schema.tab_uuid_json,
schema.tab_oid_cbor,
schema.tab_str_cbor,
schema.tab_uuid_cbor,
schema.tab_oid_pickle,
schema.tab_str_pickle,
schema.tab_uuid_pickle,
]
with zlmdb.Database(dbpath) as db:
with db.begin(write=True, stats=stats) as txn:
for tab in tabs:
tab.truncate(txn)
print(stats.puts)
print(stats.dels)
def test_delete_nonunique_indexes(testset1):
"""
Insert records into a table with a non-unique index, delete data records and check
that index records have been deleted as a consequence too.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
with db.begin(write=True) as txn:
for user in testset1:
del schema.users[txn, user.oid]
with db.begin() as txn:
for j in range(10):
user_oids = list(
schema.idx_users_by_realm.select(txn,
return_keys=False,
from_key=(j, 0),
to_key=(j + 1, 0)))
assert [] == user_oids
def test_fill_non_unique_indexes(testset1):
"""
Insert records into a table with a non-unique, non-nullable indexed column.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
# check non-unique indexes
with db.begin() as txn:
for j in range(10):
user_oids = list(
schema.idx_users_by_realm.select(txn,
return_keys=False,
from_key=(j, 0),
to_key=(j + 1, 0)))
assert list(range(j * 100, (j + 1) * 100)) == user_oids
def main2(reactor):
dbpath = '/tmp/zlmdb1'
print('Using database directory {}'.format(dbpath))
schema = UsersSchema()
with zlmdb.Database(dbpath) as db:
N = 1000
with db.begin() as txn:
cnt_begin = schema.tab_oid_fbs.count(txn)
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for i in range(N):
user = UserFbs.create_test_user()
schema.tab_oid_fbs[txn, user.oid] = user
assert stats.puts == N
assert stats.dels == 0
stats.reset()
with db.begin() as txn:
cnt_end = schema.tab_oid_fbs.count(txn)
cnt = cnt_end - cnt_begin
assert cnt == N
print('{} records written, {} records total'.format(cnt, cnt_end))
yield util.sleep(1)
def test_pmap_flatbuffers_count():
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = UsersSchema()
# max DB size is 100 MB
with zlmdb.Database(dbpath, maxsize=100 * (2**20)) as db:
oids = set()
oid_to_referred_by = {}
stats = zlmdb.TransactionStats()
# number of transactions
M = 5
# number of insert rows per transaction
N = 10000
for j in range(M):
with db.begin(write=True, stats=stats) as txn:
for i in range(N):
user = User.create_test_user()
schema.tab_oid_fbs[txn, user.oid] = user
oids.add(user.oid)
oid_to_referred_by[user.oid] = user.referred_by
assert stats.puts == N
assert stats.dels == 0
duration_ns = stats.duration
duration_ms = int(duration_ns / 1000000.)
rows_per_sec = int(round(float(stats.puts + stats.dels) * 1000. / float(duration_ms)))
print('Transaction ended: puts={} / dels={} rows in {} ms, {} rows/sec'.format(
stats.puts, stats.dels, duration_ms, rows_per_sec))
stats.reset()
# count all rows
with db.begin() as txn:
cnt = schema.tab_oid_fbs.count(txn)
assert cnt == N * M
# retrieve
with db.begin() as txn:
for j in range(5):
started = zlmdb.walltime()
M = 100
for i in range(M):
for oid in random.sample(oids, N):
user = schema.tab_oid_fbs[txn, oid]
assert user
assert user.referred_by == oid_to_referred_by.get(oid, None)
duration_ns = zlmdb.walltime() - started
duration_ms = int(duration_ns / 1000000.)
rows_per_sec = int(round(float(M * N) * 1000. / float(duration_ms)))
print('Transaction ended: {} rows read in {} ms, {} rows/sec'.format(
M * N, duration_ms, rows_per_sec))
def test_fill_indexes_nullable(testset1):
"""
Test filling a table with multiple indexes, some of which are on NULLable columns, and fill
with records that have those column values actually NULL.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
_user = deepcopy(user)
# "user.email" is an indexed column that is nullable
_user.email = None
# "user.mrealm" is an indexed (composite) column that is nullable
_user.mrealm = None
schema.users[txn, _user.oid] = _user
# check indexes has been written to (in addition to the table itself)
num_indexes = len(schema.users.indexes())
# because we have set 2 indexed columns to NULL, we need to subtract those 2
# from the total number of indexes
assert stats.puts == len(testset1) * (1 + num_indexes - 2)
# check saved objects
with db.begin() as txn:
for user in testset1:
_user = deepcopy(user)
_user.email = None
_user.mrealm = None
obj = schema.users[txn, _user.oid]
assert _user == obj
# check unique indexes
with db.begin() as txn:
for user in testset1:
# check one of the indexes that was indeed filled
user_oid = schema.idx_users_by_authid[txn, user.authid]
assert user.oid == user_oid
# check indexes that have NOT been filled
user_oid = schema.idx_users_by_email[txn, user.email]
assert user_oid is None
user_oid = schema.idx_users_by_mrealm_authid[txn,
(user.mrealm,
user.authid)]
assert user_oid is None
def test_delete_indexes(testset1):
"""
Insert records into a table with indexes, delete data records and check that index
records have been deleted as a consequence too.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
# insert data records
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
# check that all index records have been deleted as well
with db.begin(write=True) as txn:
for user in testset1:
del schema.users[txn, user.oid]
user_oid = schema.idx_users_by_authid[txn, user.authid]
assert user_oid is None
user_oid = schema.idx_users_by_email[txn, user.email]
assert user_oid is None
user_oid = schema.idx_users_by_realm[txn, (user.realm_oid,
user.oid)]
assert user_oid is None
user_oid = schema.idx_users_by_icecream[txn,
(user.icecream,
user.oid)]
assert user_oid is None
user_oid = schema.idx_users_by_mrealm_authid[txn,
(user.mrealm,
user.authid)]
assert user_oid is None
user_oid = schema.idx_users_by_mrealm_notnull_authid[txn, (
user.mrealm_notnull, user.authid)]
assert user_oid is None
def test_fill_with_indexes(testset1):
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
# check indexes has been written to (in addition to the table itself)
num_indexes = len(schema.users.indexes())
assert stats.puts == len(testset1) * (1 + num_indexes)
def test_set_notnull_indexes_nullable(testset1):
"""
Fill table with indexed column (unique-nullable) with indexed column
values NON-NULL, then (in a 2nd transaction) set the indexed column
to NULL value and check that index records are created.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
# fill table with NULLs in indexed column
with db.begin(write=True) as txn:
for user in testset1:
_user = deepcopy(user)
_user.email = None
schema.users[txn, _user.oid] = _user
# now update table with NON-NULLs in indexed column
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
_user = schema.users[txn, user.oid]
_user.email = user.email
schema.users[txn, _user.oid] = _user
# check that the table records have their indexed
# column values updated to NON-NULLs
with db.begin() as txn:
for user in testset1:
obj = schema.users[txn, user.oid]
assert user == obj
# check that the index records that previously not existed
# have been created (as the indexed column values have been
# set to NON-NULLs)
with db.begin() as txn:
for user in testset1:
user_oid = schema.idx_users_by_authid[txn, user.authid]
assert user.oid == user_oid
user_oid = schema.idx_users_by_email[txn, user.email]
assert user.oid == user_oid
def test_pmap_value_types():
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = Schema1()
n = 100
stats = zlmdb.TransactionStats()
tabs = [
(schema.tab_oid_json, schema.tab_str_json, schema.tab_uuid_json),
(schema.tab_oid_cbor, schema.tab_str_cbor, schema.tab_uuid_cbor),
(schema.tab_oid_pickle, schema.tab_str_pickle,
schema.tab_uuid_pickle),
]
with zlmdb.Database(dbpath) as db:
for tab_oid, tab_str, tab_uuid in tabs:
with db.begin(write=True, stats=stats) as txn:
for i in range(n):
user = User.create_test_user(i)
tab_oid[txn, user.oid] = user
tab_str[txn, user.authid] = user
tab_uuid[txn, user.uuid] = user
print('transaction committed')
assert stats.puts == n * 3
assert stats.dels == 0
stats.reset()
with db.begin() as txn:
cnt = tab_oid.count(txn)
assert cnt == n
cnt = tab_str.count(txn)
assert cnt == n
cnt = tab_uuid.count(txn)
assert cnt == n
print('database closed')
def test_truncate_table_with_index(testset1):
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = Schema4()
with zlmdb.Database(dbpath) as db:
with db.begin(write=True) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
stats = zlmdb.TransactionStats()
with zlmdb.Database(dbpath) as db:
with db.begin(write=True, stats=stats) as txn:
records = schema.users.truncate(txn)
print('table truncated:', records)
print(stats.puts)
print(stats.dels)
def test_pmap_flatbuffers_values():
with TemporaryDirectory() as dbpath:
print('Using temporary directory {} for database'.format(dbpath))
schema = UsersSchema()
with zlmdb.Database(dbpath) as db:
N = 100
stats = zlmdb.TransactionStats()
with db.begin(write=True, stats=stats) as txn:
for i in range(N):
user = User.create_test_user()
schema.tab_oid_fbs[txn, user.oid] = user
assert stats.puts == N
assert stats.dels == 0
stats.reset()
with db.begin() as txn:
cnt = schema.tab_oid_fbs.count(txn)
assert cnt == N
def test_fill_indexes(testset1):
"""
Fill a table with multiple indexes with data records that have all columns filled
with NON-NULL values.
"""
with TemporaryDirectory() as dbpath:
schema = Schema4()
with zlmdb.Database(dbpath) as db:
stats = zlmdb.TransactionStats()
# fill table, which also triggers inserts into the index pmaps
with db.begin(write=True, stats=stats) as txn:
for user in testset1:
schema.users[txn, user.oid] = user
# check indexes has been written to (in addition to the table itself)
num_indexes = len(schema.users.indexes())
assert stats.puts == len(testset1) * (1 + num_indexes)
# check saved objects
with db.begin() as txn:
for user in testset1:
obj = schema.users[txn, user.oid]
assert user == obj
# check unique indexes
with db.begin() as txn:
for user in testset1:
user_oid = schema.idx_users_by_authid[txn, user.authid]
assert user.oid == user_oid
user_oid = schema.idx_users_by_email[txn, user.email]
assert user.oid == user_oid
user_oid = schema.idx_users_by_icecream[txn,
(user.icecream,
user.oid)]
assert user.oid == user_oid
user_oid = schema.idx_users_by_mrealm_authid[txn,
(user.mrealm,
user.authid)]
assert user.oid == user_oid
user_oid = schema.idx_users_by_mrealm_notnull_authid[txn, (
user.mrealm_notnull, user.authid)]
assert user.oid == user_oid
# check non-unique index
users_by_icecream = {}
for user in testset1:
if user.icecream not in users_by_icecream:
users_by_icecream[user.icecream] = set()
users_by_icecream[user.icecream].add(user.oid)
MAX_OID = 9007199254740992
with db.begin() as txn:
for icecream in users_by_icecream:
for _, user_oid in schema.idx_users_by_icecream.select(
txn,
from_key=(icecream, 0),
to_key=(icecream, MAX_OID + 1),
return_values=False):
assert user_oid in users_by_icecream[icecream]
