def run(self):
for idx, i in enumerate(self.instructions):
op_tuple = fdb.tuple.unpack(i.value)
op = op_tuple[0]
# print("Stack is %r" % self.stack)
# if op != "PUSH" and op != "SWAP":
# print("%d. Instruction is %s" % (idx, op))
isDatabase = op.endswith(six.u('_DATABASE'))
isSnapshot = op.endswith(six.u('_SNAPSHOT'))
if isDatabase:
op = op[:-9]
obj = self.db
elif isSnapshot:
op = op[:-9]
obj = self.current_transaction().snapshot
else:
obj = self.current_transaction()
inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot)
try:
if inst.op == six.u("PUSH"):
inst.push(op_tuple[1])
elif inst.op == six.u("DUP"):
inst.stack.push(*self.stack[0])
elif inst.op == six.u("EMPTY_STACK"):
self.stack = Stack()
elif inst.op == six.u("SWAP"):
idx = inst.pop()
self.stack[0], self.stack[idx] = self.stack[idx], self.stack[0]
elif inst.op == six.u("POP"):
inst.pop()
elif inst.op == six.u("SUB"):
a, b = inst.pop(2)
inst.push(a - b)
elif inst.op == six.u("CONCAT"):
a, b = inst.pop(2)
inst.push(a + b)
elif inst.op == six.u("WAIT_FUTURE"):
old_idx, item = inst.pop(with_idx=True)
inst.stack.push(old_idx, item)
elif inst.op == six.u("NEW_TRANSACTION"):
self.new_transaction()
elif inst.op == six.u("USE_TRANSACTION"):
self.switch_transaction(inst.pop())
elif inst.op == six.u("ON_ERROR"):
inst.push(inst.tr.on_error(inst.pop()))
elif inst.op == six.u("GET"):
key = inst.pop()
num = random.randint(0, 2)
if num == 0:
f = obj[key]
elif num == 1:
f = obj.get(key)
else:
f = obj.__getitem__(key)
if f == None:
inst.push(b'RESULT_NOT_PRESENT')
else:
inst.push(f)
elif inst.op == six.u("GET_ESTIMATED_RANGE_SIZE"):
begin, end = inst.pop(2)
estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait()
inst.push(b"GOT_ESTIMATED_RANGE_SIZE")
elif inst.op == six.u("GET_KEY"):
key, or_equal, offset, prefix = inst.pop(4)
result = obj.get_key(fdb.KeySelector(key, or_equal, offset))
if result.startswith(prefix):
inst.push(result)
elif result < prefix:
inst.push(prefix)
else:
inst.push(strinc(prefix))
elif inst.op == six.u("GET_RANGE"):
begin, end, limit, reverse, mode = inst.pop(5)
if limit == 0 and mode == -1 and random.random() < 0.5:
if reverse:
r = obj[begin:end:-1]
else:
r = obj[begin:end]
else:
r = obj.get_range(begin, end, limit, reverse, mode)
self.push_range(inst, r)
elif inst.op == six.u("GET_RANGE_STARTS_WITH"):
prefix, limit, reverse, mode = inst.pop(4)
self.push_range(inst, obj.get_range_startswith(prefix, limit, reverse, mode))
elif inst.op == six.u("GET_RANGE_SELECTOR"):
begin_key, begin_or_equal, begin_offset, end_key, end_or_equal, end_offset, limit, reverse, mode, prefix = inst.pop(10)
beginSel = fdb.KeySelector(begin_key, begin_or_equal, begin_offset)
endSel = fdb.KeySelector(end_key, end_or_equal, end_offset)
if limit == 0 and mode == -1 and random.random() < 0.5:
if reverse:
r = obj[beginSel:endSel:-1]
else:
r = obj[beginSel:endSel]
else:
r = obj.get_range(beginSel, endSel, limit, reverse, mode)
self.push_range(inst, r, prefix_filter=prefix)
elif inst.op == six.u("GET_READ_VERSION"):
self.last_version = obj.get_read_version().wait()
inst.push(b"GOT_READ_VERSION")
elif inst.op == six.u("SET"):
key, value = inst.pop(2)
if random.random() < 0.5:
obj[key] = value
else:
obj.set(key, value)
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("LOG_STACK"):
prefix = inst.pop()
entries = {}
while len(self.stack) > 0:
stack_index = len(self.stack) - 1
entries[stack_index] = inst.pop(with_idx=True)
if len(entries) == 100:
self.log_stack(self.db, prefix, entries)
entries = {}
self.log_stack(self.db, prefix, entries)
elif inst.op == six.u("ATOMIC_OP"):
opType, key, value = inst.pop(3)
getattr(obj, opType.lower())(key, value)
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("SET_READ_VERSION"):
inst.tr.set_read_version(self.last_version)
elif inst.op == six.u("CLEAR"):
if random.random() < 0.5:
del obj[inst.pop()]
else:
obj.clear(inst.pop())
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("CLEAR_RANGE"):
begin, end = inst.pop(2)
num = random.randint(0, 2)
if num == 0:
del obj[begin:end]
elif num == 1:
obj.clear_range(begin, end)
else:
obj.__delitem__(slice(begin, end))
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("CLEAR_RANGE_STARTS_WITH"):
obj.clear_range_startswith(inst.pop())
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("READ_CONFLICT_RANGE"):
inst.tr.add_read_conflict_range(inst.pop(), inst.pop())
inst.push(b"SET_CONFLICT_RANGE")
elif inst.op == six.u("WRITE_CONFLICT_RANGE"):
inst.tr.add_write_conflict_range(inst.pop(), inst.pop())
inst.push(b"SET_CONFLICT_RANGE")
elif inst.op == six.u("READ_CONFLICT_KEY"):
inst.tr.add_read_conflict_key(inst.pop())
inst.push(b"SET_CONFLICT_KEY")
elif inst.op == six.u("WRITE_CONFLICT_KEY"):
inst.tr.add_write_conflict_key(inst.pop())
inst.push(b"SET_CONFLICT_KEY")
elif inst.op == six.u("DISABLE_WRITE_CONFLICT"):
inst.tr.options.set_next_write_no_write_conflict_range()
elif inst.op == six.u("COMMIT"):
inst.push(inst.tr.commit())
elif inst.op == six.u("RESET"):
inst.tr.reset()
elif inst.op == six.u("CANCEL"):
inst.tr.cancel()
elif inst.op == six.u("GET_COMMITTED_VERSION"):
self.last_version = inst.tr.get_committed_version()
inst.push(b"GOT_COMMITTED_VERSION")
elif inst.op == six.u("GET_APPROXIMATE_SIZE"):
approximate_size = inst.tr.get_approximate_size().wait()
inst.push(b"GOT_APPROXIMATE_SIZE")
elif inst.op == six.u("GET_VERSIONSTAMP"):
inst.push(inst.tr.get_versionstamp())
elif inst.op == six.u("TUPLE_PACK"):
count = inst.pop()
items = inst.pop(count)
inst.push(fdb.tuple.pack(tuple(items)))
elif inst.op == six.u("TUPLE_PACK_WITH_VERSIONSTAMP"):
prefix = inst.pop()
count = inst.pop()
items = inst.pop(count)
if not fdb.tuple.has_incomplete_versionstamp(items) and random.random() < 0.5:
inst.push(b"ERROR: NONE")
else:
try:
packed = fdb.tuple.pack_with_versionstamp(tuple(items), prefix=prefix)
inst.push(b"OK")
inst.push(packed)
except ValueError as e:
if str(e).startswith("No incomplete"):
inst.push(b"ERROR: NONE")
else:
inst.push(b"ERROR: MULTIPLE")
elif inst.op == six.u("TUPLE_UNPACK"):
for i in fdb.tuple.unpack(inst.pop()):
inst.push(fdb.tuple.pack((i,)))
elif inst.op == six.u("TUPLE_SORT"):
count = inst.pop()
items = inst.pop(count)
unpacked = map(fdb.tuple.unpack, items)
if six.PY3:
sorted_items = sorted(unpacked, key=fdb.tuple.pack)
else:
sorted_items = sorted(unpacked, cmp=fdb.tuple.compare)
for item in sorted_items:
inst.push(fdb.tuple.pack(item))
elif inst.op == six.u("TUPLE_RANGE"):
count = inst.pop()
items = inst.pop(count)
r = fdb.tuple.range(tuple(items))
inst.push(r.start)
inst.push(r.stop)
elif inst.op == six.u("ENCODE_FLOAT"):
f_bytes = inst.pop()
f = struct.unpack(">f", f_bytes)[0]
if not math.isnan(f) and not math.isinf(f) and not f == -0.0 and f == int(f):
f = int(f)
inst.push(fdb.tuple.SingleFloat(f))
elif inst.op == six.u("ENCODE_DOUBLE"):
d_bytes = inst.pop()
d = struct.unpack(">d", d_bytes)[0]
inst.push(d)
elif inst.op == six.u("DECODE_FLOAT"):
f = inst.pop()
f_bytes = struct.pack(">f", f.value)
inst.push(f_bytes)
elif inst.op == six.u("DECODE_DOUBLE"):
d = inst.pop()
d_bytes = struct.pack(">d", d)
inst.push(d_bytes)
elif inst.op == six.u("START_THREAD"):
t = Tester(self.db, inst.pop())
thr = threading.Thread(target=t.run)
thr.start()
self.threads.append(thr)
elif inst.op == six.u("WAIT_EMPTY"):
prefix = inst.pop()
Tester.wait_empty(self.db, prefix)
inst.push(b"WAITED_FOR_EMPTY")
elif inst.op == six.u("UNIT_TESTS"):
try:
test_db_options(db)
test_options(db)
test_watches(db)
test_cancellation(db)
test_retry_limits(db)
test_db_retry_limits(db)
test_timeouts(db)
test_db_timeouts(db)
test_combinations(db)
test_locality(db)
test_predicates()
test_size_limit_option(db)
test_get_approximate_size(db)
except fdb.FDBError as e:
print("Unit tests failed: %s" % e.description)
traceback.print_exc()
raise Exception("Unit tests failed: %s" % e.description)
elif inst.op.startswith(six.u('DIRECTORY_')):
self.directory_extension.process_instruction(inst)
else:
raise Exception("Unknown op %s" % inst.op)
except fdb.FDBError as e:
# print('ERROR: %r' % e)
inst.stack.push(idx, fdb.tuple.pack((b"ERROR", str(e.code).encode('ascii'))))
# print(" to %s" % self.stack)
# print()
[thr.join() for thr in self.threads]
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import traceback
import sys
import fdb
import fdb.directory_impl
from fdb import six
ops_that_create_dirs = [
six.u('DIRECTORY_CREATE_SUBSPACE'),
six.u('DIRECTORY_CREATE_LAYER'),
six.u('DIRECTORY_CREATE_OR_OPEN'),
six.u('DIRECTORY_CREATE'),
six.u('DIRECTORY_OPEN'),
six.u('DIRECTORY_MOVE'),
six.u('DIRECTORY_MOVE_TO'),
six.u('DIRECTORY_OPEN_SUBSPACE'),
]
log_all = False
log_instructions = False
log_ops = False
log_dirs = False
log_errors = False
def process_instruction(self, inst):
try:
if log_all or log_instructions:
print("%d. %s" % (inst.index, inst.op))
directory = self.dir_list[self.dir_index]
if inst.op == six.u('DIRECTORY_CREATE_SUBSPACE'):
path = self.pop_tuples(inst.stack)
raw_prefix = inst.pop()
log_op('created subspace at %r: %r' % (path, raw_prefix))
self.append_dir(inst, fdb.Subspace(path, raw_prefix))
elif inst.op == six.u('DIRECTORY_CREATE_LAYER'):
index1, index2, allow_manual_prefixes = inst.pop(3)
if self.dir_list[index1] is None or self.dir_list[
index2] is None:
log_op('create directory layer: None')
self.append_dir(inst, None)
else:
log_op(
'create directory layer: node_subspace (%d) = %r, content_subspace (%d) = %r, allow_manual_prefixes = %d'
% (index1, self.dir_list[index1].rawPrefix, index2,
self.dir_list[index2].rawPrefix,
allow_manual_prefixes))
self.append_dir(
inst,
fdb.DirectoryLayer(self.dir_list[index1],
self.dir_list[index2],
allow_manual_prefixes == 1))
elif inst.op == six.u('DIRECTORY_CHANGE'):
self.dir_index = inst.pop()
if not self.dir_list[self.dir_index]:
self.dir_index = self.error_index
if log_dirs or log_all:
new_dir = self.dir_list[self.dir_index]
clazz = new_dir.__class__.__name__
new_path = repr(new_dir._path) if hasattr(
new_dir, '_path') else "<na>"
print('changed directory to %d (%s @%r)' %
(self.dir_index, clazz, new_path))
elif inst.op == six.u('DIRECTORY_SET_ERROR_INDEX'):
self.error_index = inst.pop()
elif inst.op == six.u('DIRECTORY_CREATE_OR_OPEN'):
path = self.pop_tuples(inst.stack)
layer = inst.pop()
log_op('create_or_open %r: layer=%r' %
(directory.get_path() + path, layer))
d = directory.create_or_open(inst.tr, path, layer or b'')
self.append_dir(inst, d)
elif inst.op == six.u('DIRECTORY_CREATE'):
path = self.pop_tuples(inst.stack)
layer, prefix = inst.pop(2)
log_op('create %r: layer=%r, prefix=%r' %
(directory.get_path() + path, layer, prefix))
self.append_dir(
inst, directory.create(inst.tr, path, layer or b'',
prefix))
elif inst.op == six.u('DIRECTORY_OPEN'):
path = self.pop_tuples(inst.stack)
layer = inst.pop()
log_op('open %r: layer=%r' %
(directory.get_path() + path, layer))
self.append_dir(inst,
directory.open(inst.tr, path, layer or b''))
elif inst.op == six.u('DIRECTORY_MOVE'):
old_path, new_path = self.pop_tuples(inst.stack, 2)
log_op('move %r to %r' % (directory.get_path() + old_path,
directory.get_path() + new_path))
self.append_dir(inst,
directory.move(inst.tr, old_path, new_path))
elif inst.op == six.u('DIRECTORY_MOVE_TO'):
new_absolute_path = self.pop_tuples(inst.stack)
log_op('move %r to %r' %
(directory.get_path(), new_absolute_path))
self.append_dir(inst,
directory.move_to(inst.tr, new_absolute_path))
elif inst.op == six.u('DIRECTORY_REMOVE'):
count = inst.pop()
if count == 0:
log_op('remove %r' % (directory.get_path(), ))
directory.remove(inst.tr)
else:
path = self.pop_tuples(inst.stack)
log_op('remove %r' % (directory.get_path() + path, ))
directory.remove(inst.tr, path)
elif inst.op == six.u('DIRECTORY_REMOVE_IF_EXISTS'):
count = inst.pop()
if count == 0:
log_op('remove_if_exists %r' % (directory.get_path(), ))
directory.remove_if_exists(inst.tr)
else:
path = self.pop_tuples(inst.stack)
log_op('remove_if_exists %r' %
(directory.get_path() + path, ))
directory.remove_if_exists(inst.tr, path)
elif inst.op == six.u('DIRECTORY_LIST'):
count = inst.pop()
if count == 0:
result = directory.list(inst.tr)
log_op('list %r' % (directory.get_path(), ))
else:
path = self.pop_tuples(inst.stack)
result = directory.list(inst.tr, path)
log_op('list %r' % (directory.get_path() + path, ))
inst.push(fdb.tuple.pack(tuple(result)))
elif inst.op == six.u('DIRECTORY_EXISTS'):
count = inst.pop()
if count == 0:
result = directory.exists(inst.tr)
log_op('exists %r: %d' % (directory.get_path(), result))
else:
path = self.pop_tuples(inst.stack)
result = directory.exists(inst.tr, path)
log_op('exists %r: %d' %
(directory.get_path() + path, result))
if result:
inst.push(1)
else:
inst.push(0)
elif inst.op == six.u('DIRECTORY_PACK_KEY'):
key_tuple = self.pop_tuples(inst.stack)
inst.push(directory.pack(key_tuple))
elif inst.op == six.u('DIRECTORY_UNPACK_KEY'):
key = inst.pop()
log_op('unpack %r in subspace with prefix %r' %
(key, directory.rawPrefix))
tup = directory.unpack(key)
for t in tup:
inst.push(t)
elif inst.op == six.u('DIRECTORY_RANGE'):
tup = self.pop_tuples(inst.stack)
rng = directory.range(tup)
inst.push(rng.start)
inst.push(rng.stop)
elif inst.op == six.u('DIRECTORY_CONTAINS'):
key = inst.pop()
result = directory.contains(key)
if result:
inst.push(1)
else:
inst.push(0)
elif inst.op == six.u('DIRECTORY_OPEN_SUBSPACE'):
path = self.pop_tuples(inst.stack)
log_op('open_subspace %r (at %r)' % (path, directory.key()))
self.append_dir(inst, directory.subspace(path))
elif inst.op == six.u('DIRECTORY_LOG_SUBSPACE'):
prefix = inst.pop()
inst.tr[prefix +
fdb.tuple.pack((self.dir_index, ))] = directory.key()
elif inst.op == six.u('DIRECTORY_LOG_DIRECTORY'):
prefix = inst.pop()
exists = directory.exists(inst.tr)
if exists:
children = tuple(directory.list(inst.tr))
else:
children = ()
logSubspace = fdb.Subspace((self.dir_index, ), prefix)
inst.tr[logSubspace[six.u('path')]] = fdb.tuple.pack(
directory.get_path())
inst.tr[logSubspace[six.u('layer')]] = fdb.tuple.pack(
(directory.get_layer(), ))
inst.tr[logSubspace[six.u('exists')]] = fdb.tuple.pack(
(int(exists), ))
inst.tr[logSubspace[six.u('children')]] = fdb.tuple.pack(
children)
elif inst.op == six.u('DIRECTORY_STRIP_PREFIX'):
s = inst.pop()
if not s.startswith(directory.key()):
raise Exception(
'String %r does not start with raw prefix %r' %
(s, directory.key()))
inst.push(s[len(directory.key()):])
else:
raise Exception('Unknown op: %s' % inst.op)
except Exception as e:
if log_all or log_errors:
print(e)
# traceback.print_exc(file=sys.stdout)
if inst.op in ops_that_create_dirs:
self.append_dir(inst, None)
inst.push(b'DIRECTORY_ERROR')
def randomElement():
r = random.randint(0, 9)
if r == 0:
if random.random() < 0.5:
chars = [b'\x00', b'\x01', b'a', b'7', b'\xfe', b'\ff']
return b''.join(
[random.choice(chars) for c in _range(random.randint(0, 5))])
else:
return b''.join([
six.int2byte(random.randint(0, 255))
for _ in _range(random.randint(0, 10))
])
elif r == 1:
if random.random() < 0.5:
chars = [
u('\x00'),
u('\x01'),
u('a'),
u('7'),
u('\xfe'),
u('\ff'),
u('\u0000'),
u('\u0001'),
u('\uffff'),
u('\uff00'),
u('\U0001f4a9')
]
return u('').join(
[random.choice(chars) for c in _range(random.randint(0, 10))])
else:
return u('').join(
[randomUnicode() for _ in _range(random.randint(0, 10))])
elif r == 2:
return random.choice([-1, 1]) * min(
2**random.randint(0, 2040) + random.randint(-10, 10), 2**2040 - 1)
elif r == 3:
return random.choice([-1, 1]) * 2**random.randint(
0, 64) + random.randint(-10, 10)
elif r == 4:
return None
elif r == 5:
ret = random.choice([
float('-nan'),
float('-inf'), -0.0, 0.0,
float('inf'),
float('nan')
])
if random.random() < 0.5:
return SingleFloat(ret)
else:
return ret
elif r == 6:
is_double = random.random() < 0.5
byte_str = b''.join([
six.int2byte(random.randint(0, 255))
for _ in _range(8 if is_double else 4)
])
if is_double:
return struct.unpack(">d", byte_str)[0]
else:
return SingleFloat(struct.unpack(">f", byte_str)[0])
elif r == 7:
return random.random() < 0.5
elif r == 8:
return uuid.uuid4()
elif r == 9:
return [randomElement() for _ in _range(random.randint(0, 5))]
