def test_truncate_table_with_index(testset1): """ Fill a table with records that has indexes, truncate the table and check that all index records have been deleted as well. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: with db.begin(write=True) as txn: for user in testset1: schema.users[txn, user.oid] = user stats = zlmdb.TransactionStats() with zlmdb.Database(dbpath) as db: with db.begin(write=True, stats=stats) as txn: records = schema.users.truncate(txn) assert records == len(testset1) * ( len(schema.users.indexes()) + 1) assert stats.dels == records assert stats.puts == 0
def test_delete_nonindexes2(testset1): """ WARNING: quadratic run-time (in testset size) """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for user in testset1: schema.users[txn, user.oid] = user with db.begin(write=True) as txn: for j in range(10): fullset = set(range(j * 100, (j + 1) * 100)) for i in range(100): user_oid = j * 100 + i del schema.users[txn, user_oid] fullset.discard(user_oid) user_oids = set( schema.idx_users_by_realm.select(txn, return_keys=False, from_key=(j, 0), to_key=(j + 1, 0))) assert fullset == user_oids
def test_truncate_table(): with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = Schema1() stats = zlmdb.TransactionStats() tabs = [ schema.tab_oid_json, schema.tab_str_json, schema.tab_uuid_json, schema.tab_oid_cbor, schema.tab_str_cbor, schema.tab_uuid_cbor, schema.tab_oid_pickle, schema.tab_str_pickle, schema.tab_uuid_pickle, ] with zlmdb.Database(dbpath) as db: with db.begin(write=True, stats=stats) as txn: for tab in tabs: tab.truncate(txn) print(stats.puts) print(stats.dels)
def test_delete_nonunique_indexes(testset1): """ Insert records into a table with a non-unique index, delete data records and check that index records have been deleted as a consequence too. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for user in testset1: schema.users[txn, user.oid] = user with db.begin(write=True) as txn: for user in testset1: del schema.users[txn, user.oid] with db.begin() as txn: for j in range(10): user_oids = list( schema.idx_users_by_realm.select(txn, return_keys=False, from_key=(j, 0), to_key=(j + 1, 0))) assert [] == user_oids
def test_fill_non_unique_indexes(testset1): """ Insert records into a table with a non-unique, non-nullable indexed column. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for user in testset1: schema.users[txn, user.oid] = user # check non-unique indexes with db.begin() as txn: for j in range(10): user_oids = list( schema.idx_users_by_realm.select(txn, return_keys=False, from_key=(j, 0), to_key=(j + 1, 0))) assert list(range(j * 100, (j + 1) * 100)) == user_oids
def main2(reactor): dbpath = '/tmp/zlmdb1' print('Using database directory {}'.format(dbpath)) schema = UsersSchema() with zlmdb.Database(dbpath) as db: N = 1000 with db.begin() as txn: cnt_begin = schema.tab_oid_fbs.count(txn) stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for i in range(N): user = UserFbs.create_test_user() schema.tab_oid_fbs[txn, user.oid] = user assert stats.puts == N assert stats.dels == 0 stats.reset() with db.begin() as txn: cnt_end = schema.tab_oid_fbs.count(txn) cnt = cnt_end - cnt_begin assert cnt == N print('{} records written, {} records total'.format(cnt, cnt_end)) yield util.sleep(1)
def test_pmap_flatbuffers_count(): with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = UsersSchema() # max DB size is 100 MB with zlmdb.Database(dbpath, maxsize=100 * (2**20)) as db: oids = set() oid_to_referred_by = {} stats = zlmdb.TransactionStats() # number of transactions M = 5 # number of insert rows per transaction N = 10000 for j in range(M): with db.begin(write=True, stats=stats) as txn: for i in range(N): user = User.create_test_user() schema.tab_oid_fbs[txn, user.oid] = user oids.add(user.oid) oid_to_referred_by[user.oid] = user.referred_by assert stats.puts == N assert stats.dels == 0 duration_ns = stats.duration duration_ms = int(duration_ns / 1000000.) rows_per_sec = int(round(float(stats.puts + stats.dels) * 1000. / float(duration_ms))) print('Transaction ended: puts={} / dels={} rows in {} ms, {} rows/sec'.format( stats.puts, stats.dels, duration_ms, rows_per_sec)) stats.reset() # count all rows with db.begin() as txn: cnt = schema.tab_oid_fbs.count(txn) assert cnt == N * M # retrieve with db.begin() as txn: for j in range(5): started = zlmdb.walltime() M = 100 for i in range(M): for oid in random.sample(oids, N): user = schema.tab_oid_fbs[txn, oid] assert user assert user.referred_by == oid_to_referred_by.get(oid, None) duration_ns = zlmdb.walltime() - started duration_ms = int(duration_ns / 1000000.) rows_per_sec = int(round(float(M * N) * 1000. / float(duration_ms))) print('Transaction ended: {} rows read in {} ms, {} rows/sec'.format( M * N, duration_ms, rows_per_sec))
def test_fill_indexes_nullable(testset1): """ Test filling a table with multiple indexes, some of which are on NULLable columns, and fill with records that have those column values actually NULL. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for user in testset1: _user = deepcopy(user) # "user.email" is an indexed column that is nullable _user.email = None # "user.mrealm" is an indexed (composite) column that is nullable _user.mrealm = None schema.users[txn, _user.oid] = _user # check indexes has been written to (in addition to the table itself) num_indexes = len(schema.users.indexes()) # because we have set 2 indexed columns to NULL, we need to subtract those 2 # from the total number of indexes assert stats.puts == len(testset1) * (1 + num_indexes - 2) # check saved objects with db.begin() as txn: for user in testset1: _user = deepcopy(user) _user.email = None _user.mrealm = None obj = schema.users[txn, _user.oid] assert _user == obj # check unique indexes with db.begin() as txn: for user in testset1: # check one of the indexes that was indeed filled user_oid = schema.idx_users_by_authid[txn, user.authid] assert user.oid == user_oid # check indexes that have NOT been filled user_oid = schema.idx_users_by_email[txn, user.email] assert user_oid is None user_oid = schema.idx_users_by_mrealm_authid[txn, (user.mrealm, user.authid)] assert user_oid is None
def test_delete_indexes(testset1): """ Insert records into a table with indexes, delete data records and check that index records have been deleted as a consequence too. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() # insert data records with db.begin(write=True, stats=stats) as txn: for user in testset1: schema.users[txn, user.oid] = user # check that all index records have been deleted as well with db.begin(write=True) as txn: for user in testset1: del schema.users[txn, user.oid] user_oid = schema.idx_users_by_authid[txn, user.authid] assert user_oid is None user_oid = schema.idx_users_by_email[txn, user.email] assert user_oid is None user_oid = schema.idx_users_by_realm[txn, (user.realm_oid, user.oid)] assert user_oid is None user_oid = schema.idx_users_by_icecream[txn, (user.icecream, user.oid)] assert user_oid is None user_oid = schema.idx_users_by_mrealm_authid[txn, (user.mrealm, user.authid)] assert user_oid is None user_oid = schema.idx_users_by_mrealm_notnull_authid[txn, ( user.mrealm_notnull, user.authid)] assert user_oid is None
def test_fill_with_indexes(testset1): with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for user in testset1: schema.users[txn, user.oid] = user # check indexes has been written to (in addition to the table itself) num_indexes = len(schema.users.indexes()) assert stats.puts == len(testset1) * (1 + num_indexes)
def test_set_notnull_indexes_nullable(testset1): """ Fill table with indexed column (unique-nullable) with indexed column values NON-NULL, then (in a 2nd transaction) set the indexed column to NULL value and check that index records are created. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() # fill table with NULLs in indexed column with db.begin(write=True) as txn: for user in testset1: _user = deepcopy(user) _user.email = None schema.users[txn, _user.oid] = _user # now update table with NON-NULLs in indexed column with db.begin(write=True, stats=stats) as txn: for user in testset1: _user = schema.users[txn, user.oid] _user.email = user.email schema.users[txn, _user.oid] = _user # check that the table records have their indexed # column values updated to NON-NULLs with db.begin() as txn: for user in testset1: obj = schema.users[txn, user.oid] assert user == obj # check that the index records that previously not existed # have been created (as the indexed column values have been # set to NON-NULLs) with db.begin() as txn: for user in testset1: user_oid = schema.idx_users_by_authid[txn, user.authid] assert user.oid == user_oid user_oid = schema.idx_users_by_email[txn, user.email] assert user.oid == user_oid
def test_pmap_value_types(): with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = Schema1() n = 100 stats = zlmdb.TransactionStats() tabs = [ (schema.tab_oid_json, schema.tab_str_json, schema.tab_uuid_json), (schema.tab_oid_cbor, schema.tab_str_cbor, schema.tab_uuid_cbor), (schema.tab_oid_pickle, schema.tab_str_pickle, schema.tab_uuid_pickle), ] with zlmdb.Database(dbpath) as db: for tab_oid, tab_str, tab_uuid in tabs: with db.begin(write=True, stats=stats) as txn: for i in range(n): user = User.create_test_user(i) tab_oid[txn, user.oid] = user tab_str[txn, user.authid] = user tab_uuid[txn, user.uuid] = user print('transaction committed') assert stats.puts == n * 3 assert stats.dels == 0 stats.reset() with db.begin() as txn: cnt = tab_oid.count(txn) assert cnt == n cnt = tab_str.count(txn) assert cnt == n cnt = tab_uuid.count(txn) assert cnt == n print('database closed')
def test_truncate_table_with_index(testset1): with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = Schema4() with zlmdb.Database(dbpath) as db: with db.begin(write=True) as txn: for user in testset1: schema.users[txn, user.oid] = user stats = zlmdb.TransactionStats() with zlmdb.Database(dbpath) as db: with db.begin(write=True, stats=stats) as txn: records = schema.users.truncate(txn) print('table truncated:', records) print(stats.puts) print(stats.dels)
def test_pmap_flatbuffers_values(): with TemporaryDirectory() as dbpath: print('Using temporary directory {} for database'.format(dbpath)) schema = UsersSchema() with zlmdb.Database(dbpath) as db: N = 100 stats = zlmdb.TransactionStats() with db.begin(write=True, stats=stats) as txn: for i in range(N): user = User.create_test_user() schema.tab_oid_fbs[txn, user.oid] = user assert stats.puts == N assert stats.dels == 0 stats.reset() with db.begin() as txn: cnt = schema.tab_oid_fbs.count(txn) assert cnt == N
def test_fill_indexes(testset1): """ Fill a table with multiple indexes with data records that have all columns filled with NON-NULL values. """ with TemporaryDirectory() as dbpath: schema = Schema4() with zlmdb.Database(dbpath) as db: stats = zlmdb.TransactionStats() # fill table, which also triggers inserts into the index pmaps with db.begin(write=True, stats=stats) as txn: for user in testset1: schema.users[txn, user.oid] = user # check indexes has been written to (in addition to the table itself) num_indexes = len(schema.users.indexes()) assert stats.puts == len(testset1) * (1 + num_indexes) # check saved objects with db.begin() as txn: for user in testset1: obj = schema.users[txn, user.oid] assert user == obj # check unique indexes with db.begin() as txn: for user in testset1: user_oid = schema.idx_users_by_authid[txn, user.authid] assert user.oid == user_oid user_oid = schema.idx_users_by_email[txn, user.email] assert user.oid == user_oid user_oid = schema.idx_users_by_icecream[txn, (user.icecream, user.oid)] assert user.oid == user_oid user_oid = schema.idx_users_by_mrealm_authid[txn, (user.mrealm, user.authid)] assert user.oid == user_oid user_oid = schema.idx_users_by_mrealm_notnull_authid[txn, ( user.mrealm_notnull, user.authid)] assert user.oid == user_oid # check non-unique index users_by_icecream = {} for user in testset1: if user.icecream not in users_by_icecream: users_by_icecream[user.icecream] = set() users_by_icecream[user.icecream].add(user.oid) MAX_OID = 9007199254740992 with db.begin() as txn: for icecream in users_by_icecream: for _, user_oid in schema.idx_users_by_icecream.select( txn, from_key=(icecream, 0), to_key=(icecream, MAX_OID + 1), return_values=False): assert user_oid in users_by_icecream[icecream]