def verify(kvs: hse.Kvs, pfx: str, cnt: int):
# [HSE_REVISIT] Getting all keys is too slow/freezes
get_cnt = 100
if cnt < get_cnt:
get_cnt = cnt
for i in range(get_cnt):
k = "{}-{:028}".format(pfx, i)
val = kvs.get(k.encode())
assert val is not None
assert val.decode() == k
with kvs.cursor(pfx.encode()) as c:
assert sum(1 for _ in c.items()) == cnt
with kvs.cursor(pfx.encode(), flags=hse.CursorFlag.REVERSE) as rc:
assert sum(1 for _ in rc.items()) == cnt
# create, seek, reads
with kvs.cursor(pfx.encode()) as c:
c.seek(pfx.encode())
assert sum(1 for _ in c.items()) == cnt
with kvs.cursor(pfx.encode(), flags=hse.CursorFlag.REVERSE) as rc:
# Bump up the last character so prefix is larger than all keys
ch = pfx[-1]
i = ord(ch[0])
i += 1
seek_key = pfx[:-1] + chr(i)
rc.seek(seek_key.encode())
assert sum(1 for _ in rc.items()) == cnt
def verify_keys(kvs: hse.Kvs, pfx: str, start: int, end: int):
with kvs.cursor(filt=pfx.encode()) as cur:
assert sum(1 for _ in cur.items()) == end - start
with kvs.cursor(filt=pfx.encode()) as cur:
k_id = start
for (k, _) in cur.items():
expected = f"{pfx}-{k_id:0>10}".encode()
assert k == expected
assert k_id < end
k_id = k_id + 1
def run_test_2(kvdb: hse.Kvdb, kvs: hse.Kvs):
t1 = kvdb.transaction()
t1.begin()
kvs.prefix_delete(b"def", txn=t1) # LC
kvdb.sync()
kvs.prefix_delete(b"def", txn=t1) # LC
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"def01", b"val1", txn=t) # C0
kvs.put(b"def02", b"val1", txn=t) # C0
kvs.put(b"def03", b"val1", txn=t) # C0
kvs.put(b"def04", b"val1", txn=t) # C0
t1.commit() # ptomb has higher seqno
with kvs.cursor(filt=b"def") as c:
c.read()
assert c.eof is True
##c.update()
c.read()
assert c.eof is True
c.seek(b"def03")
c.read()
assert c.eof is True
c.seek(b"def02")
##c.update()
c.read()
assert c.eof is True
def run_test_1(kvdb: hse.Kvdb, kvs: hse.Kvs):
with kvdb.transaction() as t:
kvs.put(b"abc01", b"val0", txn=t) # LC
kvs.put(b"abc02", b"val0", txn=t) # LC
kvs.prefix_delete(b"abc", txn=t) # LC
kvdb.sync()
kvs.put(b"abc01", b"val1", txn=t) # C0
kvs.put(b"abc02", b"val1", txn=t) # C0
kvs.put(b"abc03", b"val1", txn=t) # C0
kvs.put(b"abc04", b"val1", txn=t) # C0
with kvs.cursor(filt=b"abc", txn=t) as c:
kv = c.read()
assert kv == (b"abc01", b"val1")
kv = c.read()
assert kv == (b"abc02", b"val1")
##c.update()
kv = c.read()
assert kv == (b"abc03", b"val1")
c.seek(b"abc02")
##c.update()
kv = c.read()
assert kv == (b"abc02", b"val1")
def separate_keys(kvdb: hse.Kvdb, kvs: hse.Kvs):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"ab03", b"val-c0", txn=t)
with kvdb.transaction() as t:
# Point Get
assert kvs.get(b"ab01", txn=t) == b"val-cn"
assert kvs.get(b"ab02", txn=t) == b"val-lc"
assert kvs.get(b"ab03", txn=t) == b"val-c0"
# Probe
cnt, *kv = kvs.prefix_probe(b"ab0", txn=t)
assert cnt == hse.KvsPfxProbeCnt.MUL
assert kv == [b"ab03", b"val-c0"]
# Cursor
with kvs.cursor(filt=b"ab", txn=t) as c:
# Read all keys
assert c.read() == (b"ab01", b"val-cn")
assert c.read() == (b"ab02", b"val-lc")
assert c.read() == (b"ab03", b"val-c0")
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab01")
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab02")
assert c.read() == (b"ab02", b"val-lc")
c.seek(b"ab03")
assert c.read() == (b"ab03", b"val-c0")
# Read, update, read
c.seek(b"ab01")
##c.update()
assert c.read() == (b"ab01", b"val-cn")
##c.update()
assert c.read() == (b"ab02", b"val-lc")
##c.update()
assert c.read() == (b"ab03", b"val-c0")
##c.update()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab01")
##c.update()
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab02")
##c.update()
assert c.read() == (b"ab02", b"val-lc")
c.seek(b"ab03")
##c.update()
assert c.read() == (b"ab03", b"val-c0")
def duplicate_c0_lc(kvdb: hse.Kvdb, kvs: hse.Kvs, cursor_sync: bool = False):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-c0", txn=t)
with kvdb.transaction() as t:
with kvs.cursor(filt=b"ab", txn=t) as c:
if cursor_sync:
kvdb.sync()
# Point Get
assert kvs.get(b"ab01", txn=t) == b"val-cn"
assert kvs.get(b"ab02", txn=t) == b"val-c0"
# Probe
cnt, *kv = kvs.prefix_probe(b"ab0", txn=t)
assert cnt == hse.KvsPfxProbeCnt.MUL
assert kv == [b"ab02", b"val-c0"]
# Read all keys
assert c.read() == (b"ab01", b"val-cn")
assert c.read() == (b"ab02", b"val-c0")
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab00")
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab01")
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab02")
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab03")
assert c.read() == (None, None) and c.eof == True
# Read, update, read
c.seek(b"ab01")
##c.update()
assert c.read() == (b"ab01", b"val-cn")
##c.update()
assert c.read() == (b"ab02", b"val-c0")
##c.update()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab01")
##c.update()
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab02")
##c.update()
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab03")
##c.update()
assert c.read() == (None, None) and c.eof == True
def verify_pfx_data(kvdb: hse.Kvdb, kvs: hse.Kvs, pfx: int):
pfx_bytes = pfx.to_bytes(4, byteorder="big")
with kvdb.transaction() as txn, kvs.cursor(filt=pfx_bytes,
txn=txn) as cursor:
for sfx in range(NUM_KEYS):
key_exp = pfx_bytes + sfx.to_bytes(1, byteorder="big")
key, val = cursor.read()
assert key == key_exp
assert val == key_exp
cursor.read()
assert cursor.eof
def tombs_c0_lc(kvdb: hse.Kvdb, kvs: hse.Kvs, cursor_sync: bool = False):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.delete(b"ab02", txn=t)
# Cursor
with kvs.cursor(filt=b"ab") as c:
if cursor_sync:
kvdb.sync()
# Point Get
assert kvs.get(b"ab01")[0] == b"val-cn"
assert kvs.get(b"ab02")[0] == None
# Probe
cnt, k, _, v, _ = kvs.prefix_probe(b"ab0")
assert cnt == hse.KvsPfxProbeCnt.ONE
assert (k, v) == (b"ab01", b"val-cn")
# Read all keys
assert c.read() == (b"ab01", b"val-cn")
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab00")
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab01")
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab02")
assert c.read() == (None, None) and c.eof == True
# Read, update, read
c.seek(b"ab01")
c.update_view()
assert c.read() == (b"ab01", b"val-cn")
c.update_view()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab01")
c.update_view()
assert c.read() == (b"ab01", b"val-cn")
c.seek(b"ab02")
c.update_view()
assert c.read() == (None, None) and c.eof == True
def duplicate_lc_cn(kvdb: hse.Kvdb, kvs: hse.Kvs, cursor_sync: bool = False):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-c0", txn=t)
# Cursor
with kvs.cursor(filt=b"ab", flags=hse.CursorFlag.REVERSE) as c:
if cursor_sync:
kvdb.sync()
# Read all keys
assert c.read() == (b"ab02", b"val-c0")
assert c.read() == (b"ab01", b"val-lc")
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab03")
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab02")
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab01")
assert c.read() == (b"ab01", b"val-lc")
c.seek(b"ab00")
assert c.read() == (None, None) and c.eof == True
# Read, update, read
c.seek(b"ab02")
c.update_view()
assert c.read() == (b"ab02", b"val-c0")
c.update_view()
assert c.read() == (b"ab01", b"val-lc")
c.update_view()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab02")
c.update_view()
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab01")
c.update_view()
assert c.read() == (b"ab01", b"val-lc")
c.seek(b"ab00")
c.update_view()
assert c.read() == (None, None) and c.eof == True
def ptombs_c0_lc(kvdb: hse.Kvdb, kvs: hse.Kvs, cursor_sync: bool = False):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.prefix_delete(b"ab", txn=t)
# Cursor
with kvdb.transaction() as t:
with kvs.cursor(filt=b"ab", txn=t) as c:
if cursor_sync:
kvdb.sync()
# Point Get
assert kvs.get(b"ab01", txn=t) == None
assert kvs.get(b"ab02", txn=t) == None
# Probe
cnt, *_ = kvs.prefix_probe(b"ab0", txn=t)
assert cnt == hse.KvsPfxProbeCnt.ZERO
# Read all keys
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab00")
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab01")
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab02")
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab03")
assert c.read() == (None, None) and c.eof == True
# Read, update, read
c.seek(b"ab01")
##c.update()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab01")
##c.update()
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab02")
##c.update()
assert c.read() == (None, None) and c.eof == True
def separate_keys(kvdb: hse.Kvdb, kvs: hse.Kvs):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"ab03", b"val-c0", txn=t)
# Cursor
with kvs.cursor(filt=b"ab", flags=hse.CursorFlag.REVERSE) as c:
# Read all keys
assert c.read() == (b"ab03", b"val-c0")
assert c.read() == (b"ab02", b"val-lc")
assert c.read() == (b"ab01", b"val-cn")
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab03")
assert c.read() == (b"ab03", b"val-c0")
c.seek(b"ab02")
assert c.read() == (b"ab02", b"val-lc")
c.seek(b"ab01")
assert c.read() == (b"ab01", b"val-cn")
# Read, update, read
c.seek(b"ab03")
assert c.read() == (b"ab03", b"val-c0")
c.update_view()
assert c.read() == (b"ab02", b"val-lc")
c.update_view()
assert c.read() == (b"ab01", b"val-cn")
c.update_view()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab03")
c.update_view()
assert c.read() == (b"ab03", b"val-c0")
c.seek(b"ab02")
c.update_view()
assert c.read() == (b"ab02", b"val-lc")
c.seek(b"ab01")
c.update_view()
assert c.read() == (b"ab01", b"val-cn")
def run_test_3(kvdb: hse.Kvdb, kvs: hse.Kvs):
with kvdb.transaction() as t:
kvs.put(b"ghi01", b"val1", txn=t) # LC
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"ghi01", b"val2", txn=t) # C0
with kvs.cursor(filt=b"ghi") as c:
kvdb.sync(
) # Moves val2 to cn while val1 stays in LC until it's garbage collected
for (k, v) in c.items():
getval = kvs.get(k)
assert v
assert v.decode() == "val2"
assert v == getval
pass
def ptombs_c0_lc(kvdb: hse.Kvdb, kvs: hse.Kvs, cursor_sync: bool = False):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.prefix_delete(b"ab", txn=t)
# Cursor
with kvs.cursor(filt=b"ab", flags=hse.CursorCreateFlag.REV) as c:
if cursor_sync:
kvdb.sync()
# Read all keys
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab00")
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab01")
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab02")
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab03")
assert c.read() == (None, None) and c.eof == True
# Read, update, read
c.seek(b"ab01")
c.update_view()
assert c.read() == (None, None) and c.eof == True
# Seek, update, read
c.seek(b"ab01")
c.update_view()
assert c.read() == (None, None) and c.eof == True
c.seek(b"ab02")
c.update_view()
assert c.read() == (None, None) and c.eof == True
def run_test(kvdb: hse.Kvdb, kvs: hse.Kvs):
with kvdb.transaction() as t5:
kvs.put(b"ab1", b"val1", txn=t5)
kvs.put(b"ab2", b"val2", txn=t5)
kvs.put(b"ab3", b"val3", txn=t5)
t0 = kvdb.transaction()
t0.begin()
t1 = kvdb.transaction()
t1.begin()
kvs.put(b"aa1", b"uncommitted-aa1", txn=t0) # commit
kvs.put(b"aa2", b"uncommitted-aa2", txn=t0) # commit
kvs.put(b"aa3", b"uncommitted-aa3", txn=t1) # abort
val = kvs.get(b"aa1", txn=t0)[0]
assert val == b"uncommitted-aa1"
with kvs.cursor(txn=t0) as c:
assert sum(1 for _ in c.items()) == 5
c.seek(b"aa2")
kv = c.read()
assert kv == (b"aa2", b"uncommitted-aa2")
kv = c.read()
assert kv == (b"ab1", b"val1")
kv = c.read()
assert kv == (b"ab2", b"val2")
kv = c.read()
assert kv == (b"ab3", b"val3")
c.read()
assert c.eof
c.seek(b"ab2")
kv = c.read()
assert kv == (b"ab2", b"val2")
kv = c.read()
assert kv == (b"ab3", b"val3")
c.read()
assert c.eof
with kvs.cursor(
txn=t0, flags=hse.CursorCreateFlag.REV
) as c:
assert sum(1 for _ in c.items()) == 5
c.seek(b"aa2")
kv = c.read()
assert kv == (b"aa2", b"uncommitted-aa2")
kv = c.read()
assert kv == (b"aa1", b"uncommitted-aa1")
c.read()
assert c.eof
c.seek(b"ab2")
kv = c.read()
assert kv == (b"ab2", b"val2")
kv = c.read()
assert kv == (b"ab1", b"val1")
kv = c.read()
assert kv == (b"aa2", b"uncommitted-aa2")
kv = c.read()
assert kv == (b"aa1", b"uncommitted-aa1")
c.read()
assert c.eof
kvdb.sync()
# Get from C0
with kvdb.transaction() as t5:
val = kvs.get(b"ab3", txn=t5)[0]
assert val == b"val3"
# Get from LC
val = kvs.get(b"aa1", txn=t0)[0]
assert val == b"uncommitted-aa1" # uncommitted data from current txn
val = kvs.get(b"aa3", txn=t0)[0]
assert val == None # uncommitted data from some other txn
val = kvs.get(b"aa3", txn=t1)[0]
assert val == b"uncommitted-aa3" # uncommitted data from current txn
t0.commit()
t1.abort()
kvdb.sync()
# Get from CN. Keys were previously in LC.
with kvdb.transaction() as t5:
# Committed. Should be visible
val = kvs.get(b"aa1", txn=t5)[0]
assert val == b"uncommitted-aa1"
# Aborted. Should not see this key.
val = kvs.get(b"aa3", txn=t5)[0]
assert val == None
pass
def duplicate_lc_cn(kvdb: hse.Kvdb, kvs: hse.Kvs, cursor_sync: bool = False):
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-cn", txn=t)
with kvdb.transaction() as t:
kvs.put(b"ab01", b"val-lc", txn=t)
kvdb.sync()
with kvdb.transaction() as t:
kvs.put(b"ab02", b"val-c0", txn=t)
# Cursor
with kvs.cursor(filt=b"ab") as c:
probe_kv = (b"ab02", b"val-c0")
if cursor_sync:
probe_kv = (b"ab01", b"val-lc")
kvdb.sync()
# Point Get
assert kvs.get(b"ab01")[0] == b"val-lc"
assert kvs.get(b"ab02")[0] == b"val-c0"
# Probe
cnt, k, _, v, _ = kvs.prefix_probe(b"ab0")
assert cnt == hse.KvsPfxProbeCnt.MUL
assert (k, v) == probe_kv
# Read all keys
assert c.read() == (b"ab01", b"val-lc")
assert c.read() == (b"ab02", b"val-c0")
assert c.read() == (None, None) and c.eof == True
# Seek, read
c.seek(b"ab00")
assert c.read() == (b"ab01", b"val-lc")
c.seek(b"ab01")
assert c.read() == (b"ab01", b"val-lc")
c.seek(b"ab02")
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab03")
assert c.read() == (None, None) and c.eof == True
# Read, update, read
c.seek(b"ab01")
c.update_view()
assert c.read() == (b"ab01", b"val-lc")
c.update_view()
assert c.read() == (b"ab02", b"val-c0")
c.update_view()
assert c.read() == (None, None) and c.eof == True
# Seek, update_view, read
c.seek(b"ab01")
c.update_view()
assert c.read() == (b"ab01", b"val-lc")
c.seek(b"ab02")
c.update_view()
assert c.read() == (b"ab02", b"val-c0")
c.seek(b"ab03")
c.update_view()
assert c.read() == (None, None) and c.eof == True