def find_version_for_timestamp(self, tr, timestamp, start): """ Uses Timekeeper to find the closest version to a timestamp. If start is True, will find the greatest version at or before timestamp. If start is False, will find the smallest version at or after the timestamp. :param tr: :param timestamp: :param start: :return: """ tr.options.set_read_system_keys() tr.options.set_read_lock_aware() timekeeper_prefix = b'\xff\x02/timeKeeper/map/' timestamp_packed = fdb.tuple.pack((timestamp, )) if start: start_key = timekeeper_prefix end_key = fdb.KeySelector.first_greater_than(timekeeper_prefix + timestamp_packed) reverse = True else: start_key = fdb.KeySelector.first_greater_or_equal( timekeeper_prefix + timestamp_packed) end_key = fdb.KeySelector.first_greater_or_equal( strinc(timekeeper_prefix)) reverse = False for k, v in tr.snapshot.get_range(start_key, end_key, limit=1, reverse=reverse): return fdb.tuple.unpack(v)[0] return 0 if start else 0x8000000000000000 # we didn't find any timekeeper data so find the max range
def __init__(self, db, full_output=True, type_filter=None, min_timestamp=None, max_timestamp=None): self.db = db self.full_output = full_output self.type_filter = type_filter self.min_timestamp = min_timestamp self.max_timestamp = max_timestamp ''' Keys look like this FF - 2 bytes \xff\x02 SSSSSSSSSS - 10 bytes Version Stamp RRRRRRRRRRRRRRRR - 16 bytes Transaction id NNNN - 4 Bytes Chunk number TTTT - 4 Bytes Total number of chunks ''' sample_key = "FF/fdbClientInfo/client_latency/SSSSSSSSSS/RRRRRRRRRRRRRRRR/NNNNTTTT/" self.client_latency_start = b'\xff\x02/fdbClientInfo/client_latency/' self.client_latency_start_key_selector = fdb.KeySelector.first_greater_than( self.client_latency_start) self.client_latency_end_key_selector = fdb.KeySelector.first_greater_or_equal( strinc(self.client_latency_start)) self.version_stamp_start_idx = sample_key.index('S') self.version_stamp_end_idx = sample_key.rindex('S') self.tr_id_start_idx = sample_key.index('R') self.tr_id_end_idx = sample_key.rindex('R') self.chunk_num_start_idx = sample_key.index('N') self.num_chunks_start_idx = sample_key.index('T') self.tr_info_map = {} self.num_chunks_stored = 0 self.num_transactions_discarded = 0
def run(self): for idx, i in enumerate(self.instructions): op_tuple = fdb.tuple.unpack(i.value) op = op_tuple[0] # print("Stack is %r" % self.stack) # if op != "PUSH" and op != "SWAP": # print("%d. Instruction is %s" % (idx, op)) isDatabase = op.endswith(six.u('_DATABASE')) isSnapshot = op.endswith(six.u('_SNAPSHOT')) if isDatabase: op = op[:-9] obj = self.db elif isSnapshot: op = op[:-9] obj = self.current_transaction().snapshot else: obj = self.current_transaction() inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot) try: if inst.op == six.u("PUSH"): inst.push(op_tuple[1]) elif inst.op == six.u("DUP"): inst.stack.push(*self.stack[0]) elif inst.op == six.u("EMPTY_STACK"): self.stack = Stack() elif inst.op == six.u("SWAP"): idx = inst.pop() self.stack[0], self.stack[idx] = self.stack[idx], self.stack[0] elif inst.op == six.u("POP"): inst.pop() elif inst.op == six.u("SUB"): a, b = inst.pop(2) inst.push(a - b) elif inst.op == six.u("CONCAT"): a, b = inst.pop(2) inst.push(a + b) elif inst.op == six.u("WAIT_FUTURE"): old_idx, item = inst.pop(with_idx=True) inst.stack.push(old_idx, item) elif inst.op == six.u("NEW_TRANSACTION"): self.new_transaction() elif inst.op == six.u("USE_TRANSACTION"): self.switch_transaction(inst.pop()) elif inst.op == six.u("ON_ERROR"): inst.push(inst.tr.on_error(inst.pop())) elif inst.op == six.u("GET"): key = inst.pop() num = random.randint(0, 2) if num == 0: f = obj[key] elif num == 1: f = obj.get(key) else: f = obj.__getitem__(key) if f == None: inst.push(b'RESULT_NOT_PRESENT') else: inst.push(f) elif inst.op == six.u("GET_ESTIMATED_RANGE_SIZE"): begin, end = inst.pop(2) estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait() inst.push(b"GOT_ESTIMATED_RANGE_SIZE") elif inst.op == six.u("GET_KEY"): key, or_equal, offset, prefix = inst.pop(4) result = obj.get_key(fdb.KeySelector(key, or_equal, offset)) if result.startswith(prefix): inst.push(result) elif result < prefix: inst.push(prefix) else: inst.push(strinc(prefix)) elif inst.op == six.u("GET_RANGE"): begin, end, limit, reverse, mode = inst.pop(5) if limit == 0 and mode == -1 and random.random() < 0.5: if reverse: r = obj[begin:end:-1] else: r = obj[begin:end] else: r = obj.get_range(begin, end, limit, reverse, mode) self.push_range(inst, r) elif inst.op == six.u("GET_RANGE_STARTS_WITH"): prefix, limit, reverse, mode = inst.pop(4) self.push_range(inst, obj.get_range_startswith(prefix, limit, reverse, mode)) elif inst.op == six.u("GET_RANGE_SELECTOR"): begin_key, begin_or_equal, begin_offset, end_key, end_or_equal, end_offset, limit, reverse, mode, prefix = inst.pop(10) beginSel = fdb.KeySelector(begin_key, begin_or_equal, begin_offset) endSel = fdb.KeySelector(end_key, end_or_equal, end_offset) if limit == 0 and mode == -1 and random.random() < 0.5: if reverse: r = obj[beginSel:endSel:-1] else: r = obj[beginSel:endSel] else: r = obj.get_range(beginSel, endSel, limit, reverse, mode) self.push_range(inst, r, prefix_filter=prefix) elif inst.op == six.u("GET_READ_VERSION"): self.last_version = obj.get_read_version().wait() inst.push(b"GOT_READ_VERSION") elif inst.op == six.u("SET"): key, value = inst.pop(2) if random.random() < 0.5: obj[key] = value else: obj.set(key, value) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("LOG_STACK"): prefix = inst.pop() entries = {} while len(self.stack) > 0: stack_index = len(self.stack) - 1 entries[stack_index] = inst.pop(with_idx=True) if len(entries) == 100: self.log_stack(self.db, prefix, entries) entries = {} self.log_stack(self.db, prefix, entries) elif inst.op == six.u("ATOMIC_OP"): opType, key, value = inst.pop(3) getattr(obj, opType.lower())(key, value) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("SET_READ_VERSION"): inst.tr.set_read_version(self.last_version) elif inst.op == six.u("CLEAR"): if random.random() < 0.5: del obj[inst.pop()] else: obj.clear(inst.pop()) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("CLEAR_RANGE"): begin, end = inst.pop(2) num = random.randint(0, 2) if num == 0: del obj[begin:end] elif num == 1: obj.clear_range(begin, end) else: obj.__delitem__(slice(begin, end)) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("CLEAR_RANGE_STARTS_WITH"): obj.clear_range_startswith(inst.pop()) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("READ_CONFLICT_RANGE"): inst.tr.add_read_conflict_range(inst.pop(), inst.pop()) inst.push(b"SET_CONFLICT_RANGE") elif inst.op == six.u("WRITE_CONFLICT_RANGE"): inst.tr.add_write_conflict_range(inst.pop(), inst.pop()) inst.push(b"SET_CONFLICT_RANGE") elif inst.op == six.u("READ_CONFLICT_KEY"): inst.tr.add_read_conflict_key(inst.pop()) inst.push(b"SET_CONFLICT_KEY") elif inst.op == six.u("WRITE_CONFLICT_KEY"): inst.tr.add_write_conflict_key(inst.pop()) inst.push(b"SET_CONFLICT_KEY") elif inst.op == six.u("DISABLE_WRITE_CONFLICT"): inst.tr.options.set_next_write_no_write_conflict_range() elif inst.op == six.u("COMMIT"): inst.push(inst.tr.commit()) elif inst.op == six.u("RESET"): inst.tr.reset() elif inst.op == six.u("CANCEL"): inst.tr.cancel() elif inst.op == six.u("GET_COMMITTED_VERSION"): self.last_version = inst.tr.get_committed_version() inst.push(b"GOT_COMMITTED_VERSION") elif inst.op == six.u("GET_APPROXIMATE_SIZE"): approximate_size = inst.tr.get_approximate_size().wait() inst.push(b"GOT_APPROXIMATE_SIZE") elif inst.op == six.u("GET_VERSIONSTAMP"): inst.push(inst.tr.get_versionstamp()) elif inst.op == six.u("TUPLE_PACK"): count = inst.pop() items = inst.pop(count) inst.push(fdb.tuple.pack(tuple(items))) elif inst.op == six.u("TUPLE_PACK_WITH_VERSIONSTAMP"): prefix = inst.pop() count = inst.pop() items = inst.pop(count) if not fdb.tuple.has_incomplete_versionstamp(items) and random.random() < 0.5: inst.push(b"ERROR: NONE") else: try: packed = fdb.tuple.pack_with_versionstamp(tuple(items), prefix=prefix) inst.push(b"OK") inst.push(packed) except ValueError as e: if str(e).startswith("No incomplete"): inst.push(b"ERROR: NONE") else: inst.push(b"ERROR: MULTIPLE") elif inst.op == six.u("TUPLE_UNPACK"): for i in fdb.tuple.unpack(inst.pop()): inst.push(fdb.tuple.pack((i,))) elif inst.op == six.u("TUPLE_SORT"): count = inst.pop() items = inst.pop(count) unpacked = map(fdb.tuple.unpack, items) if six.PY3: sorted_items = sorted(unpacked, key=fdb.tuple.pack) else: sorted_items = sorted(unpacked, cmp=fdb.tuple.compare) for item in sorted_items: inst.push(fdb.tuple.pack(item)) elif inst.op == six.u("TUPLE_RANGE"): count = inst.pop() items = inst.pop(count) r = fdb.tuple.range(tuple(items)) inst.push(r.start) inst.push(r.stop) elif inst.op == six.u("ENCODE_FLOAT"): f_bytes = inst.pop() f = struct.unpack(">f", f_bytes)[0] if not math.isnan(f) and not math.isinf(f) and not f == -0.0 and f == int(f): f = int(f) inst.push(fdb.tuple.SingleFloat(f)) elif inst.op == six.u("ENCODE_DOUBLE"): d_bytes = inst.pop() d = struct.unpack(">d", d_bytes)[0] inst.push(d) elif inst.op == six.u("DECODE_FLOAT"): f = inst.pop() f_bytes = struct.pack(">f", f.value) inst.push(f_bytes) elif inst.op == six.u("DECODE_DOUBLE"): d = inst.pop() d_bytes = struct.pack(">d", d) inst.push(d_bytes) elif inst.op == six.u("START_THREAD"): t = Tester(self.db, inst.pop()) thr = threading.Thread(target=t.run) thr.start() self.threads.append(thr) elif inst.op == six.u("WAIT_EMPTY"): prefix = inst.pop() Tester.wait_empty(self.db, prefix) inst.push(b"WAITED_FOR_EMPTY") elif inst.op == six.u("UNIT_TESTS"): try: test_db_options(db) test_options(db) test_watches(db) test_cancellation(db) test_retry_limits(db) test_db_retry_limits(db) test_timeouts(db) test_db_timeouts(db) test_combinations(db) test_locality(db) test_predicates() test_size_limit_option(db) test_get_approximate_size(db) except fdb.FDBError as e: print("Unit tests failed: %s" % e.description) traceback.print_exc() raise Exception("Unit tests failed: %s" % e.description) elif inst.op.startswith(six.u('DIRECTORY_')): self.directory_extension.process_instruction(inst) else: raise Exception("Unknown op %s" % inst.op) except fdb.FDBError as e: # print('ERROR: %r' % e) inst.stack.push(idx, fdb.tuple.pack((b"ERROR", str(e.code).encode('ascii')))) # print(" to %s" % self.stack) # print() [thr.join() for thr in self.threads]
def _is_prefix_empty(self, tr, prefix): return len(list(tr.get_range(prefix, _impl.strinc(prefix), limit=1))) == 0
def _is_prefix_free(self, tr, prefix): # Returns true if the given prefix does not "intersect" any currently # allocated prefix (including the root node). This means that it neither # contains any other prefix nor is contained by any other prefix. return prefix and not self._node_containing_key(tr, prefix) \ and not len(list(tr.get_range(self._node_subspace.pack((prefix,)), self._node_subspace.pack((_impl.strinc(prefix),)), limit=1)))