def _pack_maybe_with_versionstamp(t, prefix=None):
if not isinstance(t, tuple):
raise Exception("fdbtuple pack() expects a tuple, got a " + str(type(t)))
bytes_list = [prefix] if prefix is not None else []
child_bytes, version_pos = _reduce_children(map(_encode, t))
if version_pos >= 0:
version_pos += len(prefix) if prefix is not None else 0
bytes_list.extend(child_bytes)
if fdb.is_api_version_selected() and fdb.get_api_version() < 520:
bytes_list.append(struct.pack('<H', version_pos))
else:
bytes_list.append(struct.pack('<L', version_pos))
else:
bytes_list.extend(child_bytes)
return b''.join(bytes_list), version_pos
def _decode(v, pos):
code = six.indexbytes(v, pos)
if code == NULL_CODE:
return None, pos + 1
elif code == BYTES_CODE:
end = _find_terminator(v, pos + 1)
return v[pos + 1:end].replace(b"\x00\xFF", b"\x00"), end + 1
elif code == STRING_CODE:
end = _find_terminator(v, pos + 1)
return v[pos + 1:end].replace(b"\x00\xFF",
b"\x00").decode("utf-8"), end + 1
elif code >= INT_ZERO_CODE and code < POS_INT_END:
n = code - 20
end = pos + 1 + n
return struct.unpack(">Q", b'\x00' * (8 - n) + v[pos + 1:end])[0], end
elif code > NEG_INT_START and code < INT_ZERO_CODE:
n = 20 - code
end = pos + 1 + n
return struct.unpack(
">Q", b'\x00' * (8 - n) + v[pos + 1:end])[0] - _size_limits[n], end
elif code == POS_INT_END: # 0x1d; Positive 9-255 byte integer
length = six.indexbytes(v, pos + 1)
val = 0
for i in _range(length):
val = val << 8
val += six.indexbytes(v, pos + 2 + i)
return val, pos + 2 + length
elif code == NEG_INT_START: # 0x0b; Negative 9-255 byte integer
length = six.indexbytes(v, pos + 1) ^ 0xff
val = 0
for i in _range(length):
val = val << 8
val += six.indexbytes(v, pos + 2 + i)
return val - (1 << (length * 8)) + 1, pos + 2 + length
elif code == FLOAT_CODE:
return SingleFloat(
struct.unpack(">f", _float_adjust(v[pos + 1:pos + 5],
False))[0]), pos + 5
elif code == DOUBLE_CODE:
return struct.unpack(">d", _float_adjust(v[pos + 1:pos + 9],
False))[0], pos + 9
elif code == UUID_CODE:
return uuid.UUID(bytes=v[pos + 1:pos + 17]), pos + 17
elif code == FALSE_CODE:
if fdb.is_api_version_selected() and fdb.get_api_version() < 500:
raise ValueError("Invalid API version " + str(fdb._version) +
" for boolean types")
return False, pos + 1
elif code == TRUE_CODE:
if fdb.is_api_version_selected() and fdb.get_api_version() < 500:
raise ValueError("Invalid API version " + str(fdb._version) +
" for boolean types")
return True, pos + 1
elif code == VERSIONSTAMP_CODE:
return Versionstamp.from_bytes(v,
pos + 1), pos + 1 + Versionstamp.LENGTH
elif code == NESTED_CODE:
ret = []
end_pos = pos + 1
while end_pos < len(v):
if six.indexbytes(v, end_pos) == 0x00:
if end_pos + 1 < len(v) and six.indexbytes(
v, end_pos + 1) == 0xff:
ret.append(None)
end_pos += 2
else:
break
else:
val, end_pos = _decode(v, end_pos)
ret.append(val)
return tuple(ret), end_pos + 1
else:
raise ValueError("Unknown data type in DB: " + repr(v))
def run(self):
for idx, i in enumerate(self.instructions):
op_tuple = fdb.tuple.unpack(i.value)
op = op_tuple[0]
# print("Stack is %r" % self.stack)
# if op != "PUSH" and op != "SWAP":
# print("%d. Instruction is %s" % (idx, op))
isDatabase = op.endswith(six.u('_DATABASE'))
isSnapshot = op.endswith(six.u('_SNAPSHOT'))
if isDatabase:
op = op[:-9]
obj = self.db
elif isSnapshot:
op = op[:-9]
obj = self.current_transaction().snapshot
else:
obj = self.current_transaction()
inst = Instruction(obj, self.stack, op, idx, isDatabase,
isSnapshot)
try:
if inst.op == six.u("PUSH"):
inst.push(op_tuple[1])
elif inst.op == six.u("DUP"):
inst.stack.push(*self.stack[0])
elif inst.op == six.u("EMPTY_STACK"):
self.stack = Stack()
elif inst.op == six.u("SWAP"):
idx = inst.pop()
self.stack[0], self.stack[idx] = self.stack[
idx], self.stack[0]
elif inst.op == six.u("POP"):
inst.pop()
elif inst.op == six.u("SUB"):
a, b = inst.pop(2)
inst.push(a - b)
elif inst.op == six.u("CONCAT"):
a, b = inst.pop(2)
inst.push(a + b)
elif inst.op == six.u("WAIT_FUTURE"):
old_idx, item = inst.pop(with_idx=True)
inst.stack.push(old_idx, item)
elif inst.op == six.u("NEW_TRANSACTION"):
self.new_transaction()
elif inst.op == six.u("USE_TRANSACTION"):
self.switch_transaction(inst.pop())
elif inst.op == six.u("ON_ERROR"):
inst.push(inst.tr.on_error(inst.pop()))
elif inst.op == six.u("GET"):
key = inst.pop()
num = random.randint(0, 2)
if num == 0:
f = obj[key]
elif num == 1:
f = obj.get(key)
else:
f = obj.__getitem__(key)
if f == None:
inst.push(b'RESULT_NOT_PRESENT')
else:
inst.push(f)
elif inst.op == six.u("GET_KEY"):
key, or_equal, offset, prefix = inst.pop(4)
result = obj.get_key(fdb.KeySelector(
key, or_equal, offset))
if result.startswith(prefix):
inst.push(result)
elif result < prefix:
inst.push(prefix)
else:
inst.push(strinc(prefix))
elif inst.op == six.u("GET_RANGE"):
begin, end, limit, reverse, mode = inst.pop(5)
if limit == 0 and mode == -1 and random.random() < 0.5:
if reverse:
r = obj[begin:end:-1]
else:
r = obj[begin:end]
else:
r = obj.get_range(begin, end, limit, reverse, mode)
self.push_range(inst, r)
elif inst.op == six.u("GET_RANGE_STARTS_WITH"):
prefix, limit, reverse, mode = inst.pop(4)
self.push_range(
inst,
obj.get_range_startswith(prefix, limit, reverse, mode))
elif inst.op == six.u("GET_RANGE_SELECTOR"):
begin_key, begin_or_equal, begin_offset, end_key, end_or_equal, end_offset, limit, reverse, mode, prefix = inst.pop(
10)
beginSel = fdb.KeySelector(begin_key, begin_or_equal,
begin_offset)
endSel = fdb.KeySelector(end_key, end_or_equal, end_offset)
if limit == 0 and mode == -1 and random.random() < 0.5:
if reverse:
r = obj[beginSel:endSel:-1]
else:
r = obj[beginSel:endSel]
else:
r = obj.get_range(beginSel, endSel, limit, reverse,
mode)
self.push_range(inst, r, prefix_filter=prefix)
elif inst.op == six.u("GET_READ_VERSION"):
self.last_version = obj.get_read_version().wait()
inst.push(b"GOT_READ_VERSION")
elif inst.op == six.u("SET"):
key, value = inst.pop(2)
if random.random() < 0.5:
obj[key] = value
else:
obj.set(key, value)
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("LOG_STACK"):
prefix = inst.pop()
entries = {}
while len(self.stack) > 0:
stack_index = len(self.stack) - 1
entries[stack_index] = inst.pop(with_idx=True)
if len(entries) == 100:
self.log_stack(self.db, prefix, entries)
entries = {}
self.log_stack(self.db, prefix, entries)
elif inst.op == six.u("ATOMIC_OP"):
opType, key, value = inst.pop(3)
getattr(obj, opType.lower())(key, value)
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("SET_READ_VERSION"):
inst.tr.set_read_version(self.last_version)
elif inst.op == six.u("CLEAR"):
if random.random() < 0.5:
del obj[inst.pop()]
else:
obj.clear(inst.pop())
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("CLEAR_RANGE"):
begin, end = inst.pop(2)
num = random.randint(0, 2)
if num == 0:
del obj[begin:end]
elif num == 1:
obj.clear_range(begin, end)
else:
obj.__delitem__(slice(begin, end))
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("CLEAR_RANGE_STARTS_WITH"):
obj.clear_range_startswith(inst.pop())
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("READ_CONFLICT_RANGE"):
inst.tr.add_read_conflict_range(inst.pop(), inst.pop())
inst.push(b"SET_CONFLICT_RANGE")
elif inst.op == six.u("WRITE_CONFLICT_RANGE"):
inst.tr.add_write_conflict_range(inst.pop(), inst.pop())
inst.push(b"SET_CONFLICT_RANGE")
elif inst.op == six.u("READ_CONFLICT_KEY"):
inst.tr.add_read_conflict_key(inst.pop())
inst.push(b"SET_CONFLICT_KEY")
elif inst.op == six.u("WRITE_CONFLICT_KEY"):
inst.tr.add_write_conflict_key(inst.pop())
inst.push(b"SET_CONFLICT_KEY")
elif inst.op == six.u("DISABLE_WRITE_CONFLICT"):
inst.tr.options.set_next_write_no_write_conflict_range()
elif inst.op == six.u("COMMIT"):
inst.push(inst.tr.commit())
elif inst.op == six.u("RESET"):
inst.tr.reset()
elif inst.op == six.u("CANCEL"):
inst.tr.cancel()
elif inst.op == six.u("GET_COMMITTED_VERSION"):
self.last_version = inst.tr.get_committed_version()
inst.push(b"GOT_COMMITTED_VERSION")
elif inst.op == six.u("GET_VERSIONSTAMP"):
inst.push(inst.tr.get_versionstamp())
elif inst.op == six.u("TUPLE_PACK"):
count = inst.pop()
items = inst.pop(count)
inst.push(fdb.tuple.pack(tuple(items)))
elif inst.op == six.u("TUPLE_PACK_WITH_VERSIONSTAMP"):
prefix = inst.pop()
count = inst.pop()
items = inst.pop(count)
if not fdb.tuple.has_incomplete_versionstamp(
items) and random.random() < 0.5:
inst.push(b"ERROR: NONE")
else:
try:
packed = fdb.tuple.pack_with_versionstamp(
tuple(items), prefix=prefix)
inst.push(b"OK")
inst.push(packed)
except ValueError as e:
if str(e).startswith("No incomplete"):
inst.push(b"ERROR: NONE")
else:
inst.push(b"ERROR: MULTIPLE")
elif inst.op == six.u("TUPLE_UNPACK"):
for i in fdb.tuple.unpack(inst.pop()):
inst.push(fdb.tuple.pack((i, )))
elif inst.op == six.u("TUPLE_SORT"):
count = inst.pop()
items = inst.pop(count)
unpacked = map(fdb.tuple.unpack, items)
if six.PY3:
sorted_items = sorted(unpacked, key=fdb.tuple.pack)
else:
sorted_items = sorted(unpacked, cmp=fdb.tuple.compare)
for item in sorted_items:
inst.push(fdb.tuple.pack(item))
elif inst.op == six.u("TUPLE_RANGE"):
count = inst.pop()
items = inst.pop(count)
r = fdb.tuple.range(tuple(items))
inst.push(r.start)
inst.push(r.stop)
elif inst.op == six.u("ENCODE_FLOAT"):
f_bytes = inst.pop()
f = struct.unpack(">f", f_bytes)[0]
inst.push(fdb.tuple.SingleFloat(f))
elif inst.op == six.u("ENCODE_DOUBLE"):
d_bytes = inst.pop()
d = struct.unpack(">d", d_bytes)[0]
inst.push(d)
elif inst.op == six.u("DECODE_FLOAT"):
f = inst.pop()
f_bytes = struct.pack(">f", f.value)
inst.push(f_bytes)
elif inst.op == six.u("DECODE_DOUBLE"):
d = inst.pop()
d_bytes = struct.pack(">d", d)
inst.push(d_bytes)
elif inst.op == six.u("START_THREAD"):
t = Tester(self.db, inst.pop())
thr = threading.Thread(target=t.run)
thr.start()
self.threads.append(thr)
elif inst.op == six.u("WAIT_EMPTY"):
prefix = inst.pop()
Tester.wait_empty(self.db, prefix)
inst.push(b"WAITED_FOR_EMPTY")
elif inst.op == six.u("UNIT_TESTS"):
assert fdb.is_api_version_selected()
api_version = fdb.get_api_version()
try:
fdb.api_version(api_version + 1)
raise RuntimeError(
'Was not stopped from selecting two API versions')
except RuntimeError as e:
assert str(
e
) == 'FDB API already loaded at version {}'.format(
api_version)
try:
fdb.api_version(api_version - 1)
raise RuntimeError(
'Was not stopped from selecting two API versions')
except RuntimeError as e:
assert str(
e
) == 'FDB API already loaded at version {}'.format(
api_version)
fdb.api_version(api_version)
try:
db.options.set_location_cache_size(100001)
test_options(db)
test_watches(db)
test_cancellation(db)
test_retry_limits(db)
test_timeouts(db)
test_combinations(db)
test_locality(db)
test_predicates()
except fdb.FDBError as e:
print("Unit tests failed: %s" % e.description)
traceback.print_exc()
raise Exception("Unit tests failed: %s" %
e.description)
elif inst.op.startswith(six.u('DIRECTORY_')):
self.directory_extension.process_instruction(inst)
else:
raise Exception("Unknown op %s" % inst.op)
except fdb.FDBError as e:
# print('ERROR: %s' % repr(e))
inst.stack.push(
idx, fdb.tuple.pack(
(b"ERROR", str(e.code).encode('ascii'))))
# print(" to %s" % self.stack)
# print()
[thr.join() for thr in self.threads]
#
import ctypes
import sys
import os
import struct
import threading
import time
import random
import time
import traceback
sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..')]
import fdb
assert not fdb.is_api_version_selected()
try:
fdb.get_api_version()
except RuntimeError as e:
assert str(e) == 'API version is not set'
fdb.api_version(int(sys.argv[2]))
assert int(sys.argv[2]) == fdb.get_api_version()
from fdb import six
from fdb.impl import strinc
import fdb.tuple
from directory_extension import DirectoryExtension
from cancellation_timeout_tests import test_timeouts
from cancellation_timeout_tests import test_cancellation