def correctness_get_key( self, db, store, data, num_get_keys, max_keys_per_transaction ): selectors = [] for i in range(0, num_get_keys): index = random.randint(0, len(data) - 1) or_equal = random.randint(0, 1) offset = random.randint( max(-index + (1 - or_equal), -10), min(len(data) - index - 1 + or_equal, 10), ) key = sorted(data)[index][0] selector = fdb.KeySelector(key, or_equal, offset) selectors.append(selector) keys_retrieved = 0 while keys_retrieved < len(selectors): sub_selectors = selectors[ keys_retrieved : keys_retrieved + max_keys_per_transaction ] db_keys = self.correctness_get_key_transactional(db, sub_selectors) for i in range(0, num_get_keys): if db_keys[i] != store.get_key(sub_selectors[i]): return False keys_retrieved += max_keys_per_transaction return True
def run_get_key(self, tr, count=2000): tr.options.set_retry_limit(5) s = time.time() for i in range(count): tr.get_key( fdb.KeySelector(self.random_key(), True, random.randint(-10, 10))).wait() return count / (time.time() - s)
def run(self): for idx, i in enumerate(self.instructions): op_tuple = fdb.tuple.unpack(i.value) op = op_tuple[0] # print("Stack is %r" % self.stack) # if op != "PUSH" and op != "SWAP": # print("%d. Instruction is %s" % (idx, op)) isDatabase = op.endswith(six.u('_DATABASE')) isSnapshot = op.endswith(six.u('_SNAPSHOT')) if isDatabase: op = op[:-9] obj = self.db elif isSnapshot: op = op[:-9] obj = self.current_transaction().snapshot else: obj = self.current_transaction() inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot) try: if inst.op == six.u("PUSH"): inst.push(op_tuple[1]) elif inst.op == six.u("DUP"): inst.stack.push(*self.stack[0]) elif inst.op == six.u("EMPTY_STACK"): self.stack = Stack() elif inst.op == six.u("SWAP"): idx = inst.pop() self.stack[0], self.stack[idx] = self.stack[idx], self.stack[0] elif inst.op == six.u("POP"): inst.pop() elif inst.op == six.u("SUB"): a, b = inst.pop(2) inst.push(a - b) elif inst.op == six.u("CONCAT"): a, b = inst.pop(2) inst.push(a + b) elif inst.op == six.u("WAIT_FUTURE"): old_idx, item = inst.pop(with_idx=True) inst.stack.push(old_idx, item) elif inst.op == six.u("NEW_TRANSACTION"): self.new_transaction() elif inst.op == six.u("USE_TRANSACTION"): self.switch_transaction(inst.pop()) elif inst.op == six.u("ON_ERROR"): inst.push(inst.tr.on_error(inst.pop())) elif inst.op == six.u("GET"): key = inst.pop() num = random.randint(0, 2) if num == 0: f = obj[key] elif num == 1: f = obj.get(key) else: f = obj.__getitem__(key) if f == None: inst.push(b'RESULT_NOT_PRESENT') else: inst.push(f) elif inst.op == six.u("GET_ESTIMATED_RANGE_SIZE"): begin, end = inst.pop(2) estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait() inst.push(b"GOT_ESTIMATED_RANGE_SIZE") elif inst.op == six.u("GET_KEY"): key, or_equal, offset, prefix = inst.pop(4) result = obj.get_key(fdb.KeySelector(key, or_equal, offset)) if result.startswith(prefix): inst.push(result) elif result < prefix: inst.push(prefix) else: inst.push(strinc(prefix)) elif inst.op == six.u("GET_RANGE"): begin, end, limit, reverse, mode = inst.pop(5) if limit == 0 and mode == -1 and random.random() < 0.5: if reverse: r = obj[begin:end:-1] else: r = obj[begin:end] else: r = obj.get_range(begin, end, limit, reverse, mode) self.push_range(inst, r) elif inst.op == six.u("GET_RANGE_STARTS_WITH"): prefix, limit, reverse, mode = inst.pop(4) self.push_range(inst, obj.get_range_startswith(prefix, limit, reverse, mode)) elif inst.op == six.u("GET_RANGE_SELECTOR"): begin_key, begin_or_equal, begin_offset, end_key, end_or_equal, end_offset, limit, reverse, mode, prefix = inst.pop(10) beginSel = fdb.KeySelector(begin_key, begin_or_equal, begin_offset) endSel = fdb.KeySelector(end_key, end_or_equal, end_offset) if limit == 0 and mode == -1 and random.random() < 0.5: if reverse: r = obj[beginSel:endSel:-1] else: r = obj[beginSel:endSel] else: r = obj.get_range(beginSel, endSel, limit, reverse, mode) self.push_range(inst, r, prefix_filter=prefix) elif inst.op == six.u("GET_READ_VERSION"): self.last_version = obj.get_read_version().wait() inst.push(b"GOT_READ_VERSION") elif inst.op == six.u("SET"): key, value = inst.pop(2) if random.random() < 0.5: obj[key] = value else: obj.set(key, value) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("LOG_STACK"): prefix = inst.pop() entries = {} while len(self.stack) > 0: stack_index = len(self.stack) - 1 entries[stack_index] = inst.pop(with_idx=True) if len(entries) == 100: self.log_stack(self.db, prefix, entries) entries = {} self.log_stack(self.db, prefix, entries) elif inst.op == six.u("ATOMIC_OP"): opType, key, value = inst.pop(3) getattr(obj, opType.lower())(key, value) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("SET_READ_VERSION"): inst.tr.set_read_version(self.last_version) elif inst.op == six.u("CLEAR"): if random.random() < 0.5: del obj[inst.pop()] else: obj.clear(inst.pop()) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("CLEAR_RANGE"): begin, end = inst.pop(2) num = random.randint(0, 2) if num == 0: del obj[begin:end] elif num == 1: obj.clear_range(begin, end) else: obj.__delitem__(slice(begin, end)) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("CLEAR_RANGE_STARTS_WITH"): obj.clear_range_startswith(inst.pop()) if obj == self.db: inst.push(b"RESULT_NOT_PRESENT") elif inst.op == six.u("READ_CONFLICT_RANGE"): inst.tr.add_read_conflict_range(inst.pop(), inst.pop()) inst.push(b"SET_CONFLICT_RANGE") elif inst.op == six.u("WRITE_CONFLICT_RANGE"): inst.tr.add_write_conflict_range(inst.pop(), inst.pop()) inst.push(b"SET_CONFLICT_RANGE") elif inst.op == six.u("READ_CONFLICT_KEY"): inst.tr.add_read_conflict_key(inst.pop()) inst.push(b"SET_CONFLICT_KEY") elif inst.op == six.u("WRITE_CONFLICT_KEY"): inst.tr.add_write_conflict_key(inst.pop()) inst.push(b"SET_CONFLICT_KEY") elif inst.op == six.u("DISABLE_WRITE_CONFLICT"): inst.tr.options.set_next_write_no_write_conflict_range() elif inst.op == six.u("COMMIT"): inst.push(inst.tr.commit()) elif inst.op == six.u("RESET"): inst.tr.reset() elif inst.op == six.u("CANCEL"): inst.tr.cancel() elif inst.op == six.u("GET_COMMITTED_VERSION"): self.last_version = inst.tr.get_committed_version() inst.push(b"GOT_COMMITTED_VERSION") elif inst.op == six.u("GET_APPROXIMATE_SIZE"): approximate_size = inst.tr.get_approximate_size().wait() inst.push(b"GOT_APPROXIMATE_SIZE") elif inst.op == six.u("GET_VERSIONSTAMP"): inst.push(inst.tr.get_versionstamp()) elif inst.op == six.u("TUPLE_PACK"): count = inst.pop() items = inst.pop(count) inst.push(fdb.tuple.pack(tuple(items))) elif inst.op == six.u("TUPLE_PACK_WITH_VERSIONSTAMP"): prefix = inst.pop() count = inst.pop() items = inst.pop(count) if not fdb.tuple.has_incomplete_versionstamp(items) and random.random() < 0.5: inst.push(b"ERROR: NONE") else: try: packed = fdb.tuple.pack_with_versionstamp(tuple(items), prefix=prefix) inst.push(b"OK") inst.push(packed) except ValueError as e: if str(e).startswith("No incomplete"): inst.push(b"ERROR: NONE") else: inst.push(b"ERROR: MULTIPLE") elif inst.op == six.u("TUPLE_UNPACK"): for i in fdb.tuple.unpack(inst.pop()): inst.push(fdb.tuple.pack((i,))) elif inst.op == six.u("TUPLE_SORT"): count = inst.pop() items = inst.pop(count) unpacked = map(fdb.tuple.unpack, items) if six.PY3: sorted_items = sorted(unpacked, key=fdb.tuple.pack) else: sorted_items = sorted(unpacked, cmp=fdb.tuple.compare) for item in sorted_items: inst.push(fdb.tuple.pack(item)) elif inst.op == six.u("TUPLE_RANGE"): count = inst.pop() items = inst.pop(count) r = fdb.tuple.range(tuple(items)) inst.push(r.start) inst.push(r.stop) elif inst.op == six.u("ENCODE_FLOAT"): f_bytes = inst.pop() f = struct.unpack(">f", f_bytes)[0] if not math.isnan(f) and not math.isinf(f) and not f == -0.0 and f == int(f): f = int(f) inst.push(fdb.tuple.SingleFloat(f)) elif inst.op == six.u("ENCODE_DOUBLE"): d_bytes = inst.pop() d = struct.unpack(">d", d_bytes)[0] inst.push(d) elif inst.op == six.u("DECODE_FLOAT"): f = inst.pop() f_bytes = struct.pack(">f", f.value) inst.push(f_bytes) elif inst.op == six.u("DECODE_DOUBLE"): d = inst.pop() d_bytes = struct.pack(">d", d) inst.push(d_bytes) elif inst.op == six.u("START_THREAD"): t = Tester(self.db, inst.pop()) thr = threading.Thread(target=t.run) thr.start() self.threads.append(thr) elif inst.op == six.u("WAIT_EMPTY"): prefix = inst.pop() Tester.wait_empty(self.db, prefix) inst.push(b"WAITED_FOR_EMPTY") elif inst.op == six.u("UNIT_TESTS"): try: test_db_options(db) test_options(db) test_watches(db) test_cancellation(db) test_retry_limits(db) test_db_retry_limits(db) test_timeouts(db) test_db_timeouts(db) test_combinations(db) test_locality(db) test_predicates() test_size_limit_option(db) test_get_approximate_size(db) except fdb.FDBError as e: print("Unit tests failed: %s" % e.description) traceback.print_exc() raise Exception("Unit tests failed: %s" % e.description) elif inst.op.startswith(six.u('DIRECTORY_')): self.directory_extension.process_instruction(inst) else: raise Exception("Unknown op %s" % inst.op) except fdb.FDBError as e: # print('ERROR: %r' % e) inst.stack.push(idx, fdb.tuple.pack((b"ERROR", str(e.code).encode('ascii')))) # print(" to %s" % self.stack) # print() [thr.join() for thr in self.threads]
def test_functions(self, db): self.callback = False self.callbackError = "" try: tr = db.create_transaction() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("db.create_transaction failed")) return try: tr["testkey"] = "testvalue" value = tr["testkey"] value.is_ready() value.block_until_ready() value.wait() except KeyboardInterrupt: raise except Exception: self.result.add_error( self.get_error("Set/Get value failed (block until ready)") ) try: value = tr["testkey"] value.wait() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get value failed")) try: tr["testkey"] = "newtestvalue" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Replace value failed")) try: value = tr["fakekey"] # The following line would generate a segfault # value.capi.fdb_future_block_until_ready(0) value.wait() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get non-existent key failed")) try: tr.commit().wait() tr.get_committed_version() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Commit failed")) try: tr.reset() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Reset failed")) try: version = tr.get_read_version() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("tr.get_read_version failed")) try: value = tr["testkey"] value.wait() tr.reset() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get and reset failed")) try: tr.set_read_version(version.wait()) except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Set read version failed")) try: value = tr["testkey"] callback_time = time.time() value.on_ready(self.test_callback) except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get future and set callback failed")) try: del tr["testkey"] except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Delete key failed")) try: del tr["fakekey"] except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Delete non-existent key failed")) try: tr.set("testkey", "testvalue") value = tr.get("testkey") value.wait() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Future.get failed")) try: tr.clear("testkey") except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Clear key failed")) try: tr["testkey1"] = "testvalue1" tr["testkey2"] = "testvalue2" tr["testkey3"] = "testvalue3" for k, v in tr.get_range("testkey1", "testkey3"): v += "" for k, v in tr.get_range("testkey1", "testkey2", 2): v += "" for k, v in tr["testkey1":"testkey3"]: v += "" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get range failed")) try: tr["otherkey1"] = "othervalue1" tr["otherkey2"] = "othervalue2" for k, v in tr.get_range_startswith("testkey"): v += "" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get range starts with failed")) try: tr.clear_range_startswith("otherkey") except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Clear range starts with failed")) try: tr.clear_range("testkey1", "testkey3") except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Clear range failed")) try: tr["testkey1"] = "testvalue1" tr["testkey2"] = "testvalue2" tr["testkey3"] = "testvalue3" begin = fdb.KeySelector("testkey2", 0, 0) end = fdb.KeySelector("testkey2", 0, 1) except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Create key selector failed")) try: for k, v in tr.get_range(begin, end): v += "" for k, v in tr.get_range(begin, end, 2): v += "" for k, v in tr[begin:end]: v += "" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Get range (key selectors) failed")) try: tr.clear_range(begin, end) tr["testkey1"] = "testvalue1" tr["testkey2"] = "testvalue2" tr["testkey3"] = "testvalue3" del tr[begin:end] tr["testkey1"] = "testvalue1" tr["testkey2"] = "testvalue2" tr["testkey3"] = "testvalue3" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Clear range (key selectors) failed")) try: begin = fdb.KeySelector.last_less_than("testkey2") end = fdb.KeySelector.first_greater_or_equal("testkey2") for k, v in tr.get_range(begin, end): v += "" begin = fdb.KeySelector.last_less_or_equal("testkey2") end = fdb.KeySelector.first_greater_than("testkey2") for k, v in tr.get_range(begin, end): v += "" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Builtin key selectors failed")) try: del tr["testkey1":"testkey3"] except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Delete key range failed")) try: tr.commit().wait() tr.get_committed_version() except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Commit failed")) try: key = fdb.tuple.pack(("k1", "k2", "k3")) k_tuple = fdb.tuple.unpack(key) if k_tuple[0] != "k1" and k_tuple[1] != "k2" and k_tuple[2] != "k3": self.result.add_error( "Tuple <-> key conversion yielded incorrect results" ) except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Tuple <-> key conversion failed")) try: tr[fdb.tuple.pack(("k1", "k2"))] = "v" tr[fdb.tuple.pack(("k1", "k2", "k3"))] = "v1" tr[fdb.tuple.pack(("k1", "k2", "k3", "k4"))] = "v2" for k, v in tr[fdb.tuple.range(("k1", "k2"))]: v += "" except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Tuple get range failed")) try: tr["testint"] = "10" y = int(tr["testint"]) + 1 if y != 11: self.result.add_error("Value retrieval yielded incorrect results") except KeyboardInterrupt: raise except Exception: self.result.add_error(self.get_error("Future value retrieval failed")) if not self.callback: time.sleep(5) if not self.callback: self.result.add_error( "Warning: Future callback not called after %f seconds" % (time.time() - callback_time) ) if len(self.callbackError) > 0: self.result.add_error(self.callbackError)
def testFunctions(self, db): self.callback = False self.callbackError = '' try: tr = db.create_transaction() except KeyboardInterrupt: raise except: self.result.add_error( self.getError('db.create_transaction failed')) return try: tr['testkey'] = 'testvalue' value = tr['testkey'] value.is_ready() value.block_until_ready() value.wait() except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Set/Get value failed (block until ready)')) try: value = tr['testkey'] value.wait() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Get value failed')) try: tr['testkey'] = 'newtestvalue' except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Replace value failed')) try: value = tr['fakekey'] # The following line would generate a segfault # value.capi.fdb_future_block_until_ready(0) value.wait() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Get non-existent key failed')) try: tr.commit().wait() tr.get_committed_version() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Commit failed')) try: tr.reset() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Reset failed')) try: version = tr.get_read_version() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('tr.get_read_version failed')) try: value = tr['testkey'] value.wait() tr.reset() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Get and reset failed')) try: tr.set_read_version(version.wait()) except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Set read version failed')) try: value = tr['testkey'] callbackTime = time.time() value.on_ready(self.testCallback) except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Get future and set callback failed')) try: del tr['testkey'] except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Delete key failed')) try: del tr['fakekey'] except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Delete non-existent key failed')) try: tr.set('testkey', 'testvalue') value = tr.get('testkey') value.wait() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Future.get failed')) try: tr.clear('testkey') except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Clear key failed')) try: tr['testkey1'] = 'testvalue1' tr['testkey2'] = 'testvalue2' tr['testkey3'] = 'testvalue3' for k, v in tr.get_range('testkey1', 'testkey3'): v += '' for k, v in tr.get_range('testkey1', 'testkey2', 2): v += '' for k, v in tr['testkey1':'testkey3']: v += '' except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Get range failed')) try: tr['otherkey1'] = 'othervalue1' tr['otherkey2'] = 'othervalue2' for k, v in tr.get_range_startswith('testkey'): v += '' except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Get range starts with failed')) try: tr.clear_range_startswith('otherkey') except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Clear range starts with failed')) try: tr.clear_range('testkey1', 'testkey3') except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Clear range failed')) try: tr['testkey1'] = 'testvalue1' tr['testkey2'] = 'testvalue2' tr['testkey3'] = 'testvalue3' begin = fdb.KeySelector('testkey2', 0, 0) end = fdb.KeySelector('testkey2', 0, 1) except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Create key selector failed')) try: for k, v in tr.get_range(begin, end): v += '' for k, v in tr.get_range(begin, end, 2): v += '' for k, v in tr[begin:end]: v += '' except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Get range (key selectors) failed')) try: tr.clear_range(begin, end) tr['testkey1'] = 'testvalue1' tr['testkey2'] = 'testvalue2' tr['testkey3'] = 'testvalue3' del tr[begin:end] tr['testkey1'] = 'testvalue1' tr['testkey2'] = 'testvalue2' tr['testkey3'] = 'testvalue3' except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Clear range (key selectors) failed')) try: begin = fdb.KeySelector.last_less_than('testkey2') end = fdb.KeySelector.first_greater_or_equal('testkey2') for k, v in tr.get_range(begin, end): v += '' begin = fdb.KeySelector.last_less_or_equal('testkey2') end = fdb.KeySelector.first_greater_than('testkey2') for k, v in tr.get_range(begin, end): v += '' except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Builtin key selectors failed')) try: del tr['testkey1':'testkey3'] except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Delete key range failed')) try: tr.commit().wait() tr.get_committed_version() except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Commit failed')) try: key = fdb.tuple.pack(('k1', 'k2', 'k3')) kTuple = fdb.tuple.unpack(key) if kTuple[0] != 'k1' and kTuple[1] != 'k2' and kTuple[2] != 'k3': self.result.add_error( 'Tuple <-> key conversion yielded incorrect results') except KeyboardInterrupt: raise except: self.result.add_error( self.getError('Tuple <-> key conversion failed')) try: tr[fdb.tuple.pack(('k1', 'k2'))] = 'v' tr[fdb.tuple.pack(('k1', 'k2', 'k3'))] = 'v1' tr[fdb.tuple.pack(('k1', 'k2', 'k3', 'k4'))] = 'v2' for k, v in tr[fdb.tuple.range(('k1', 'k2'))]: v += '' except KeyboardInterrupt: raise except: self.result.add_error(self.getError('Tuple get range failed')) try: tr['testint'] = '10' y = int(tr['testint']) + 1 if y != 11: self.result.add_error( 'Value retrieval yielded incorrect results') except KeyboardInterrupt: raise except: self.result.add_error(getError('Future value retrieval failed')) if not self.callback: time.sleep(5) if not self.callback: self.result.add_error( 'Warning: Future callback not called after %f seconds' % (time.time() - callbackTime)) if len(self.callbackError) > 0: self.result.add_error(self.callbackError)