def run(self): for idx, i in enumerate(self.instructions): op_tuple = fdb.tuple.unpack(i.value) op = op_tuple[0] # print("Stack is %r" % self.stack) # if op != "PUSH" and op != "SWAP": # print("%d. Instruction is %s" % (idx, op)) isDatabase = op.endswith(six.u('_DATABASE')) isSnapshot = op.endswith(six.u('_SNAPSHOT')) if isDatabase: op = op[:-9] obj = self.db elif isSnapshot: op = op[:-9] obj = self.current_transaction().snapshot else: obj = self.current_transaction() inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot) try: if inst.op == six.u("PUSH"): inst.push(op_tuple[1]) elif inst.op == six.u("DUP"): inst.stack.push(*self.stack[0]) elif inst.op == six.u("EMPTY_STACK"): self.stack = Stack() elif inst.op == six.u("SWAP"): idx = inst.pop() self.stack[0], self.stack[idx] = self.stack[idx], self.stack[0] elif inst.op == six.u("POP"): inst.pop() elif inst.op == six.u("SUB"): a, b = inst.pop(2) inst.push(a - b) elif inst.op == six.u("CONCAT"): a, b = inst.pop(2) inst.push(a + b) elif inst.op == six.u("WAIT_FUTURE"): old_idx, item = inst.pop(with_idx=True) inst.stack.push(old_idx, item) elif inst.op == six.u("NEW_TRANSACTION"): self.new_transaction() elif inst.op == six.u("USE_TRANSACTION"): self.switch_transaction(inst.pop()) elif inst.op == six.u("ON_ERROR"): inst.push(inst.tr.on_error(inst.pop())) elif inst.op == six.u("GET"): key = inst.pop() num = random.randint(0, 2) if num == 0: f = obj[key] elif num == 1: f = obj.get(key) else: f = obj.__getitem__(key) if f == None: inst.push(b'RESULT_NOT_PRESENT') else: inst.push(f) elif inst.op == six.u("GET_ESTIMATED_RANGE_SIZE"): begin, end = inst.pop(2) estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait() inst.push(b"GOT_ESTIMATED_RANGE_SIZE") elif inst.op == six.u("GET_KEY"): key, or_equal, offset, prefix = inst.pop(4) result = obj.get_key(fdb.KeySelector(key, or_equal, offset)) if result.startswith(prefix): inst.push(result) elif result < prefix: inst.push(prefix) else: inst.push(strinc(prefix)) elif inst.op == six.u("GET_RANGE"): begin, end, limit, reverse, mode = inst.pop(5) if limit == 0 and mode == -1 and random.random() < 0.5: if reverse: r = obj[begin:end:-1] else: r = obj[begin:end] else: r = obj.get_range(begin, end, limit, reverse, mode) self.push_range(inst, r) elif inst.op == six.u("GET_RANGE_STARTS_WITH"): prefix, limit, reverse, mode = inst.pop(4) self.push_range(inst, obj.get_range_startswith(prefix, limit, reverse, mode)) elif inst.op == six.u("GET_RANGE_SELECTOR"): begin_key, begin_or_equal, begin_offset, end_key, end_or_equal, end_offset, limit, reverse, mode, prefix = inst.pop(10) beginSel = fdb.KeySelector(begin_key, begin_or_equal, begin_offset) endSel = fdb.KeySelector(end_key, end_or_equal, end_offset) if limit == 0 and mode == -1 and random.random() < 0.5: if reverse: r = obj[beginSel:endSel:-1] else: r = obj[beginSel:endSel] else: r = obj.get_range(beginSel, endSel, limit, reverse, mode) self.push_range(inst, r, prefix_filter=prefix) elif inst.op == six.u("GET_READ_VERSION"): self.last_version = obj.get_read_version().wait() inst.push(b"GOT_READ_VERSION") elif inst.op == six.u("SET"): key, value = inst.pop(2) if random.random() < 0.5: obj[key] = value else: obj.set(key, value) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("LOG_STACK"): prefix = inst.pop() entries = {} while len(self.stack) > 0: stack_index = len(self.stack) - 1 entries[stack_index] = inst.pop(with_idx=True) if len(entries) == 100: self.log_stack(self.db, prefix, entries) entries = {} self.log_stack(self.db, prefix, entries) elif inst.op == six.u("ATOMIC_OP"): opType, key, value = inst.pop(3) getattr(obj, opType.lower())(key, value) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("SET_READ_VERSION"): inst.tr.set_read_version(self.last_version) elif inst.op == six.u("CLEAR"): if random.random() < 0.5: del obj[inst.pop()] else: obj.clear(inst.pop()) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("CLEAR_RANGE"): begin, end = inst.pop(2) num = random.randint(0, 2) if num == 0: del obj[begin:end] elif num == 1: obj.clear_range(begin, end) else: obj.__delitem__(slice(begin, end)) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("CLEAR_RANGE_STARTS_WITH"): obj.clear_range_startswith(inst.pop()) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("READ_CONFLICT_RANGE"): inst.tr.add_read_conflict_range(inst.pop(), inst.pop()) inst.push(b"SET_CONFLICT_RANGE") elif inst.op == six.u("WRITE_CONFLICT_RANGE"): inst.tr.add_write_conflict_range(inst.pop(), inst.pop()) inst.push(b"SET_CONFLICT_RANGE") elif inst.op == six.u("READ_CONFLICT_KEY"): inst.tr.add_read_conflict_key(inst.pop()) inst.push(b"SET_CONFLICT_KEY") elif inst.op == six.u("WRITE_CONFLICT_KEY"): inst.tr.add_write_conflict_key(inst.pop()) inst.push(b"SET_CONFLICT_KEY") elif inst.op == six.u("DISABLE_WRITE_CONFLICT"): inst.tr.options.set_next_write_no_write_conflict_range() elif inst.op == six.u("COMMIT"): inst.push(inst.tr.commit()) elif inst.op == six.u("RESET"): inst.tr.reset() elif inst.op == six.u("CANCEL"): inst.tr.cancel() elif inst.op == six.u("GET_COMMITTED_VERSION"): self.last_version = inst.tr.get_committed_version() inst.push(b"GOT_COMMITTED_VERSION") elif inst.op == six.u("GET_APPROXIMATE_SIZE"): approximate_size = inst.tr.get_approximate_size().wait() inst.push(b"GOT_APPROXIMATE_SIZE") elif inst.op == six.u("GET_VERSIONSTAMP"): inst.push(inst.tr.get_versionstamp()) elif inst.op == six.u("TUPLE_PACK"): count = inst.pop() items = inst.pop(count) inst.push(fdb.tuple.pack(tuple(items))) elif inst.op == six.u("TUPLE_PACK_WITH_VERSIONSTAMP"): prefix = inst.pop() count = inst.pop() items = inst.pop(count) if not fdb.tuple.has_incomplete_versionstamp(items) and random.random() < 0.5: inst.push(b"ERROR: NONE") else: try: packed = fdb.tuple.pack_with_versionstamp(tuple(items), prefix=prefix) inst.push(b"OK") inst.push(packed) except ValueError as e: if str(e).startswith("No incomplete"): inst.push(b"ERROR: NONE") else: inst.push(b"ERROR: MULTIPLE") elif inst.op == six.u("TUPLE_UNPACK"): for i in fdb.tuple.unpack(inst.pop()): inst.push(fdb.tuple.pack((i,))) elif inst.op == six.u("TUPLE_SORT"): count = inst.pop() items = inst.pop(count) unpacked = map(fdb.tuple.unpack, items) if six.PY3: sorted_items = sorted(unpacked, key=fdb.tuple.pack) else: sorted_items = sorted(unpacked, cmp=fdb.tuple.compare) for item in sorted_items: inst.push(fdb.tuple.pack(item)) elif inst.op == six.u("TUPLE_RANGE"): count = inst.pop() items = inst.pop(count) r = fdb.tuple.range(tuple(items)) inst.push(r.start) inst.push(r.stop) elif inst.op == six.u("ENCODE_FLOAT"): f_bytes = inst.pop() f = struct.unpack(">f", f_bytes)[0] if not math.isnan(f) and not math.isinf(f) and not f == -0.0 and f == int(f): f = int(f) inst.push(fdb.tuple.SingleFloat(f)) elif inst.op == six.u("ENCODE_DOUBLE"): d_bytes = inst.pop() d = struct.unpack(">d", d_bytes)[0] inst.push(d) elif inst.op == six.u("DECODE_FLOAT"): f = inst.pop() f_bytes = struct.pack(">f", f.value) inst.push(f_bytes) elif inst.op == six.u("DECODE_DOUBLE"): d = inst.pop() d_bytes = struct.pack(">d", d) inst.push(d_bytes) elif inst.op == six.u("START_THREAD"): t = Tester(self.db, inst.pop()) thr = threading.Thread(target=t.run) thr.start() self.threads.append(thr) elif inst.op == six.u("WAIT_EMPTY"): prefix = inst.pop() Tester.wait_empty(self.db, prefix) inst.push(b"WAITED_FOR_EMPTY") elif inst.op == six.u("UNIT_TESTS"): try: test_db_options(db) test_options(db) test_watches(db) test_cancellation(db) test_retry_limits(db) test_db_retry_limits(db) test_timeouts(db) test_db_timeouts(db) test_combinations(db) test_locality(db) test_predicates() test_size_limit_option(db) test_get_approximate_size(db) except fdb.FDBError as e: print("Unit tests failed: %s" % e.description) traceback.print_exc() raise Exception("Unit tests failed: %s" % e.description) elif inst.op.startswith(six.u('DIRECTORY_')): self.directory_extension.process_instruction(inst) else: raise Exception("Unknown op %s" % inst.op) except fdb.FDBError as e: # print('ERROR: %r' % e) inst.stack.push(idx, fdb.tuple.pack((b"ERROR", str(e.code).encode('ascii')))) # print(" to %s" % self.stack) # print() [thr.join() for thr in self.threads]
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import traceback import sys import fdb import fdb.directory_impl from fdb import six ops_that_create_dirs = [ six.u('DIRECTORY_CREATE_SUBSPACE'), six.u('DIRECTORY_CREATE_LAYER'), six.u('DIRECTORY_CREATE_OR_OPEN'), six.u('DIRECTORY_CREATE'), six.u('DIRECTORY_OPEN'), six.u('DIRECTORY_MOVE'), six.u('DIRECTORY_MOVE_TO'), six.u('DIRECTORY_OPEN_SUBSPACE'), ] log_all = False log_instructions = False log_ops = False log_dirs = False log_errors = False
def process_instruction(self, inst): try: if log_all or log_instructions: print("%d. %s" % (inst.index, inst.op)) directory = self.dir_list[self.dir_index] if inst.op == six.u('DIRECTORY_CREATE_SUBSPACE'): path = self.pop_tuples(inst.stack) raw_prefix = inst.pop() log_op('created subspace at %r: %r' % (path, raw_prefix)) self.append_dir(inst, fdb.Subspace(path, raw_prefix)) elif inst.op == six.u('DIRECTORY_CREATE_LAYER'): index1, index2, allow_manual_prefixes = inst.pop(3) if self.dir_list[index1] is None or self.dir_list[ index2] is None: log_op('create directory layer: None') self.append_dir(inst, None) else: log_op( 'create directory layer: node_subspace (%d) = %r, content_subspace (%d) = %r, allow_manual_prefixes = %d' % (index1, self.dir_list[index1].rawPrefix, index2, self.dir_list[index2].rawPrefix, allow_manual_prefixes)) self.append_dir( inst, fdb.DirectoryLayer(self.dir_list[index1], self.dir_list[index2], allow_manual_prefixes == 1)) elif inst.op == six.u('DIRECTORY_CHANGE'): self.dir_index = inst.pop() if not self.dir_list[self.dir_index]: self.dir_index = self.error_index if log_dirs or log_all: new_dir = self.dir_list[self.dir_index] clazz = new_dir.__class__.__name__ new_path = repr(new_dir._path) if hasattr( new_dir, '_path') else "<na>" print('changed directory to %d (%s @%r)' % (self.dir_index, clazz, new_path)) elif inst.op == six.u('DIRECTORY_SET_ERROR_INDEX'): self.error_index = inst.pop() elif inst.op == six.u('DIRECTORY_CREATE_OR_OPEN'): path = self.pop_tuples(inst.stack) layer = inst.pop() log_op('create_or_open %r: layer=%r' % (directory.get_path() + path, layer)) d = directory.create_or_open(inst.tr, path, layer or b'') self.append_dir(inst, d) elif inst.op == six.u('DIRECTORY_CREATE'): path = self.pop_tuples(inst.stack) layer, prefix = inst.pop(2) log_op('create %r: layer=%r, prefix=%r' % (directory.get_path() + path, layer, prefix)) self.append_dir( inst, directory.create(inst.tr, path, layer or b'', prefix)) elif inst.op == six.u('DIRECTORY_OPEN'): path = self.pop_tuples(inst.stack) layer = inst.pop() log_op('open %r: layer=%r' % (directory.get_path() + path, layer)) self.append_dir(inst, directory.open(inst.tr, path, layer or b'')) elif inst.op == six.u('DIRECTORY_MOVE'): old_path, new_path = self.pop_tuples(inst.stack, 2) log_op('move %r to %r' % (directory.get_path() + old_path, directory.get_path() + new_path)) self.append_dir(inst, directory.move(inst.tr, old_path, new_path)) elif inst.op == six.u('DIRECTORY_MOVE_TO'): new_absolute_path = self.pop_tuples(inst.stack) log_op('move %r to %r' % (directory.get_path(), new_absolute_path)) self.append_dir(inst, directory.move_to(inst.tr, new_absolute_path)) elif inst.op == six.u('DIRECTORY_REMOVE'): count = inst.pop() if count == 0: log_op('remove %r' % (directory.get_path(), )) directory.remove(inst.tr) else: path = self.pop_tuples(inst.stack) log_op('remove %r' % (directory.get_path() + path, )) directory.remove(inst.tr, path) elif inst.op == six.u('DIRECTORY_REMOVE_IF_EXISTS'): count = inst.pop() if count == 0: log_op('remove_if_exists %r' % (directory.get_path(), )) directory.remove_if_exists(inst.tr) else: path = self.pop_tuples(inst.stack) log_op('remove_if_exists %r' % (directory.get_path() + path, )) directory.remove_if_exists(inst.tr, path) elif inst.op == six.u('DIRECTORY_LIST'): count = inst.pop() if count == 0: result = directory.list(inst.tr) log_op('list %r' % (directory.get_path(), )) else: path = self.pop_tuples(inst.stack) result = directory.list(inst.tr, path) log_op('list %r' % (directory.get_path() + path, )) inst.push(fdb.tuple.pack(tuple(result))) elif inst.op == six.u('DIRECTORY_EXISTS'): count = inst.pop() if count == 0: result = directory.exists(inst.tr) log_op('exists %r: %d' % (directory.get_path(), result)) else: path = self.pop_tuples(inst.stack) result = directory.exists(inst.tr, path) log_op('exists %r: %d' % (directory.get_path() + path, result)) if result: inst.push(1) else: inst.push(0) elif inst.op == six.u('DIRECTORY_PACK_KEY'): key_tuple = self.pop_tuples(inst.stack) inst.push(directory.pack(key_tuple)) elif inst.op == six.u('DIRECTORY_UNPACK_KEY'): key = inst.pop() log_op('unpack %r in subspace with prefix %r' % (key, directory.rawPrefix)) tup = directory.unpack(key) for t in tup: inst.push(t) elif inst.op == six.u('DIRECTORY_RANGE'): tup = self.pop_tuples(inst.stack) rng = directory.range(tup) inst.push(rng.start) inst.push(rng.stop) elif inst.op == six.u('DIRECTORY_CONTAINS'): key = inst.pop() result = directory.contains(key) if result: inst.push(1) else: inst.push(0) elif inst.op == six.u('DIRECTORY_OPEN_SUBSPACE'): path = self.pop_tuples(inst.stack) log_op('open_subspace %r (at %r)' % (path, directory.key())) self.append_dir(inst, directory.subspace(path)) elif inst.op == six.u('DIRECTORY_LOG_SUBSPACE'): prefix = inst.pop() inst.tr[prefix + fdb.tuple.pack((self.dir_index, ))] = directory.key() elif inst.op == six.u('DIRECTORY_LOG_DIRECTORY'): prefix = inst.pop() exists = directory.exists(inst.tr) if exists: children = tuple(directory.list(inst.tr)) else: children = () logSubspace = fdb.Subspace((self.dir_index, ), prefix) inst.tr[logSubspace[six.u('path')]] = fdb.tuple.pack( directory.get_path()) inst.tr[logSubspace[six.u('layer')]] = fdb.tuple.pack( (directory.get_layer(), )) inst.tr[logSubspace[six.u('exists')]] = fdb.tuple.pack( (int(exists), )) inst.tr[logSubspace[six.u('children')]] = fdb.tuple.pack( children) elif inst.op == six.u('DIRECTORY_STRIP_PREFIX'): s = inst.pop() if not s.startswith(directory.key()): raise Exception( 'String %r does not start with raw prefix %r' % (s, directory.key())) inst.push(s[len(directory.key()):]) else: raise Exception('Unknown op: %s' % inst.op) except Exception as e: if log_all or log_errors: print(e) # traceback.print_exc(file=sys.stdout) if inst.op in ops_that_create_dirs: self.append_dir(inst, None) inst.push(b'DIRECTORY_ERROR')
def randomElement(): r = random.randint(0, 9) if r == 0: if random.random() < 0.5: chars = [b'\x00', b'\x01', b'a', b'7', b'\xfe', b'\ff'] return b''.join( [random.choice(chars) for c in _range(random.randint(0, 5))]) else: return b''.join([ six.int2byte(random.randint(0, 255)) for _ in _range(random.randint(0, 10)) ]) elif r == 1: if random.random() < 0.5: chars = [ u('\x00'), u('\x01'), u('a'), u('7'), u('\xfe'), u('\ff'), u('\u0000'), u('\u0001'), u('\uffff'), u('\uff00'), u('\U0001f4a9') ] return u('').join( [random.choice(chars) for c in _range(random.randint(0, 10))]) else: return u('').join( [randomUnicode() for _ in _range(random.randint(0, 10))]) elif r == 2: return random.choice([-1, 1]) * min( 2**random.randint(0, 2040) + random.randint(-10, 10), 2**2040 - 1) elif r == 3: return random.choice([-1, 1]) * 2**random.randint( 0, 64) + random.randint(-10, 10) elif r == 4: return None elif r == 5: ret = random.choice([ float('-nan'), float('-inf'), -0.0, 0.0, float('inf'), float('nan') ]) if random.random() < 0.5: return SingleFloat(ret) else: return ret elif r == 6: is_double = random.random() < 0.5 byte_str = b''.join([ six.int2byte(random.randint(0, 255)) for _ in _range(8 if is_double else 4) ]) if is_double: return struct.unpack(">d", byte_str)[0] else: return SingleFloat(struct.unpack(">f", byte_str)[0]) elif r == 7: return random.random() < 0.5 elif r == 8: return uuid.uuid4() elif r == 9: return [randomElement() for _ in _range(random.randint(0, 5))]