def FROM(self, tr, *pattern, seed=Map()): # seed is immutable
"""Yields bindings that match PATTERN"""
assert len(pattern) == len(self._items), "invalid item count"
variable = tuple(isinstance(x, Variable) for x in pattern)
# find the first index suitable for the query
combination = tuple(x for x in range(len(self._items))
if not variable[x])
for subspace, index in enumerate(self._indices):
if is_permutation_prefix(combination, index):
break
else:
raise NStoreException("Oops!")
# `index` variable holds the permutation suitable for the
# query. `subspace` is the "prefix" of that index.
prefix = tuple(pattern[i] for i in index
if not isinstance(pattern[i], Variable))
prefix = (self._prefix, subspace, prefix)
for key, _ in tr.get_range_startswith(fdb.tuple.pack(prefix)[:-1]):
key = fdb.tuple.unpack(key)
items = key[2]
# re-order the items
items = tuple(items[index.index(i)]
for i in range(len(self._items)))
# seed is immutable
bindings = seed
for i, item in enumerate(pattern):
if isinstance(item, Variable):
bindings = bindings.set(item.name, items[i])
yield bindings
def __init__(self, docPath, keyPath):
docPath = ["d"] + docPath.split(".")
if keyPath:
keyPath = keyPath.split(".")
else:
keyPath = []
self.docPath = tuple(docPath)
self.keyPath = tuple(keyPath)
self.dkPath = tuple(docPath + keyPath)
self.index_keys = tuple(i for i, k in enumerate(docPath) if k == '?')
self.key_keys = tuple(i+len(docPath) for i, k in enumerate(keyPath) if k == '?')
self.register(self.dkPath)
self.index_doc = root.index[str(self.plugin_id)]
def __init__(self, docPath, keyPath):
docPath = ["d"] + docPath.split(".")
if keyPath:
keyPath = keyPath.split(".")
else:
keyPath = []
self.docPath = tuple(docPath)
self.keyPath = tuple(keyPath)
self.dkPath = tuple(docPath + keyPath)
self.index_keys = tuple(i for i, k in enumerate(docPath) if k == '?')
self.key_keys = tuple(i + len(docPath) for i, k in enumerate(keyPath)
if k == '?')
self.register(self.dkPath)
self.index_doc = root.index[str(self.plugin_id)]
def push_range(self, inst, iter, prefix_filter=None):
kvs = []
for k, v in iter:
if prefix_filter is None or k.startswith(prefix_filter):
kvs += [k, v]
inst.push(fdb.tuple.pack(tuple(kvs)))
def func(*args):
assert len(args) == arity, "arity mismatch"
leftmost_consts = [arg.id for arg in
itertools.takewhile(lambda x: x.is_const(), args[:-1])]
prefix_tuple = tuple(leftmost_consts)
partial_key = len(leftmost_consts) < arity - 1
for k, v in self._get_generic_predicate(db, prefix_tuple, partial_key):
yield self.subspace.unpack(k)+(v,)
def func(*args):
assert len(args) == arity, "arity mismatch"
leftmost_consts = [arg.id for arg in
itertools.takewhile(lambda x: x.is_const(), args)]
prefix_tuple = (prefix,) + tuple(leftmost_consts)
partial_key = len(leftmost_consts) < arity
for t in self._get_custom_predicate(db, prefix_tuple, partial_key):
yield self.subspace.unpack(t)[1:]
def z_to_xy(z):
m = [0,0]
l = 0
while z:
for i in [0, 1]:
m[i] += (z & 1) << l
z >>= 1
l += 1
return tuple(m)
def reader(self):
for fully_pathed in glob.iglob(os.path.join(self._dir, self._filename)):
if not os.path.isfile(fully_pathed): continue
with open(fully_pathed, 'rb') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=self._delimiter)
first_line = True
for line in csv_reader:
if self._header and first_line:
first_line = False
continue
if self._skip_empty:
line = [v for v in line if v != '']
yield tuple(line)
def reader(self):
for fully_pathed in glob.iglob(os.path.join(self._dir, self._filename)):
if not os.path.isfile(fully_pathed): continue
with open(fully_pathed, 'rb') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=self._delimiter)
first_line = True
for line in csv_reader:
if self._header and first_line:
first_line = False
continue
if self._skip_empty:
line = [v for v in line if v != '']
yield tuple(line)
def _to_unicode_path(self, path):
if isinstance(path, str):
path = unicode(path)
if isinstance(path, unicode):
return (path,)
if isinstance(path, tuple):
path = list(path)
for i, name in enumerate(path):
if isinstance(name, str):
path[i] = unicode(path[i])
elif not isinstance(name, unicode):
raise ValueError('Invalid path: must be a unicode string or a tuple of unicode strings')
return tuple(path)
raise ValueError('Invalid path: must be a unicode string or a tuple of unicode strings')
def random_tuple(self, max_size, incomplete_versionstamps=False):
size = random.randint(1, max_size)
tup = []
for i in range(size):
choice = random.choice(self.types)
if choice == 'int':
tup.append(self.random_int())
elif choice == 'null':
tup.append(None)
elif choice == 'bytes':
tup.append(self.random_string(random.randint(0, 100)))
elif choice == 'string':
tup.append(self.random_unicode_str(random.randint(0, 100)))
elif choice == 'uuid':
tup.append(uuid.uuid4())
elif choice == 'bool':
b = random.random() < 0.5
if self.api_version < 500:
tup.append(int(b))
else:
tup.append(b)
elif choice == 'float':
tup.append(fdb.tuple.SingleFloat(self.random_float(8)))
elif choice == 'double':
tup.append(self.random_float(11))
elif choice == 'tuple':
length = random.randint(0, max_size - size)
if length == 0:
tup.append(())
else:
tup.append(self.random_tuple(length))
elif choice == 'versionstamp':
if incomplete_versionstamps and random.random() < 0.5:
tr_version = fdb.tuple.Versionstamp._UNSET_TR_VERSION
else:
tr_version = self.random_string(10)
user_version = random.randint(0, 0xffff)
tup.append(fdb.tuple.Versionstamp(tr_version, user_version))
else:
assert false
return tuple(tup)
def _to_unicode_path(path):
if isinstance(path, bytes):
path = six.text_type(path)
if isinstance(path, six.text_type):
return (path, )
if isinstance(path, tuple):
path = list(path)
for i, name in enumerate(path):
if isinstance(name, bytes):
path[i] = six.text_type(path[i])
elif not isinstance(name, six.text_type):
raise ValueError(
'Invalid path: must be a unicode string or a tuple of unicode strings'
)
return tuple(path)
raise ValueError(
'Invalid path: must be a unicode string or a tuple of unicode strings')
def run(self):
for idx, i in enumerate(self.instructions):
op_tuple = fdb.tuple.unpack(i.value)
op = op_tuple[0]
# print("Stack is %r" % self.stack)
# if op != "PUSH" and op != "SWAP":
# print("%d. Instruction is %s" % (idx, op))
isDatabase = op.endswith(six.u('_DATABASE'))
isSnapshot = op.endswith(six.u('_SNAPSHOT'))
if isDatabase:
op = op[:-9]
obj = self.db
elif isSnapshot:
op = op[:-9]
obj = self.current_transaction().snapshot
else:
obj = self.current_transaction()
inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot)
try:
if inst.op == six.u("PUSH"):
inst.push(op_tuple[1])
elif inst.op == six.u("DUP"):
inst.stack.push(*self.stack[0])
elif inst.op == six.u("EMPTY_STACK"):
self.stack = Stack()
elif inst.op == six.u("SWAP"):
idx = inst.pop()
self.stack[0], self.stack[idx] = self.stack[idx], self.stack[0]
elif inst.op == six.u("POP"):
inst.pop()
elif inst.op == six.u("SUB"):
a, b = inst.pop(2)
inst.push(a - b)
elif inst.op == six.u("CONCAT"):
a, b = inst.pop(2)
inst.push(a + b)
elif inst.op == six.u("WAIT_FUTURE"):
old_idx, item = inst.pop(with_idx=True)
inst.stack.push(old_idx, item)
elif inst.op == six.u("NEW_TRANSACTION"):
self.new_transaction()
elif inst.op == six.u("USE_TRANSACTION"):
self.switch_transaction(inst.pop())
elif inst.op == six.u("ON_ERROR"):
inst.push(inst.tr.on_error(inst.pop()))
elif inst.op == six.u("GET"):
key = inst.pop()
num = random.randint(0, 2)
if num == 0:
f = obj[key]
elif num == 1:
f = obj.get(key)
else:
f = obj.__getitem__(key)
if f == None:
inst.push(b'RESULT_NOT_PRESENT')
else:
inst.push(f)
elif inst.op == six.u("GET_ESTIMATED_RANGE_SIZE"):
begin, end = inst.pop(2)
estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait()
inst.push(b"GOT_ESTIMATED_RANGE_SIZE")
elif inst.op == six.u("GET_KEY"):
key, or_equal, offset, prefix = inst.pop(4)
result = obj.get_key(fdb.KeySelector(key, or_equal, offset))
if result.startswith(prefix):
inst.push(result)
elif result < prefix:
inst.push(prefix)
else:
inst.push(strinc(prefix))
elif inst.op == six.u("GET_RANGE"):
begin, end, limit, reverse, mode = inst.pop(5)
if limit == 0 and mode == -1 and random.random() < 0.5:
if reverse:
r = obj[begin:end:-1]
else:
r = obj[begin:end]
else:
r = obj.get_range(begin, end, limit, reverse, mode)
self.push_range(inst, r)
elif inst.op == six.u("GET_RANGE_STARTS_WITH"):
prefix, limit, reverse, mode = inst.pop(4)
self.push_range(inst, obj.get_range_startswith(prefix, limit, reverse, mode))
elif inst.op == six.u("GET_RANGE_SELECTOR"):
begin_key, begin_or_equal, begin_offset, end_key, end_or_equal, end_offset, limit, reverse, mode, prefix = inst.pop(10)
beginSel = fdb.KeySelector(begin_key, begin_or_equal, begin_offset)
endSel = fdb.KeySelector(end_key, end_or_equal, end_offset)
if limit == 0 and mode == -1 and random.random() < 0.5:
if reverse:
r = obj[beginSel:endSel:-1]
else:
r = obj[beginSel:endSel]
else:
r = obj.get_range(beginSel, endSel, limit, reverse, mode)
self.push_range(inst, r, prefix_filter=prefix)
elif inst.op == six.u("GET_READ_VERSION"):
self.last_version = obj.get_read_version().wait()
inst.push(b"GOT_READ_VERSION")
elif inst.op == six.u("SET"):
key, value = inst.pop(2)
if random.random() < 0.5:
obj[key] = value
else:
obj.set(key, value)
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("LOG_STACK"):
prefix = inst.pop()
entries = {}
while len(self.stack) > 0:
stack_index = len(self.stack) - 1
entries[stack_index] = inst.pop(with_idx=True)
if len(entries) == 100:
self.log_stack(self.db, prefix, entries)
entries = {}
self.log_stack(self.db, prefix, entries)
elif inst.op == six.u("ATOMIC_OP"):
opType, key, value = inst.pop(3)
getattr(obj, opType.lower())(key, value)
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("SET_READ_VERSION"):
inst.tr.set_read_version(self.last_version)
elif inst.op == six.u("CLEAR"):
if random.random() < 0.5:
del obj[inst.pop()]
else:
obj.clear(inst.pop())
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("CLEAR_RANGE"):
begin, end = inst.pop(2)
num = random.randint(0, 2)
if num == 0:
del obj[begin:end]
elif num == 1:
obj.clear_range(begin, end)
else:
obj.__delitem__(slice(begin, end))
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("CLEAR_RANGE_STARTS_WITH"):
obj.clear_range_startswith(inst.pop())
if obj == self.db:
inst.push(b"RESULT_NOT_PRESENT")
elif inst.op == six.u("READ_CONFLICT_RANGE"):
inst.tr.add_read_conflict_range(inst.pop(), inst.pop())
inst.push(b"SET_CONFLICT_RANGE")
elif inst.op == six.u("WRITE_CONFLICT_RANGE"):
inst.tr.add_write_conflict_range(inst.pop(), inst.pop())
inst.push(b"SET_CONFLICT_RANGE")
elif inst.op == six.u("READ_CONFLICT_KEY"):
inst.tr.add_read_conflict_key(inst.pop())
inst.push(b"SET_CONFLICT_KEY")
elif inst.op == six.u("WRITE_CONFLICT_KEY"):
inst.tr.add_write_conflict_key(inst.pop())
inst.push(b"SET_CONFLICT_KEY")
elif inst.op == six.u("DISABLE_WRITE_CONFLICT"):
inst.tr.options.set_next_write_no_write_conflict_range()
elif inst.op == six.u("COMMIT"):
inst.push(inst.tr.commit())
elif inst.op == six.u("RESET"):
inst.tr.reset()
elif inst.op == six.u("CANCEL"):
inst.tr.cancel()
elif inst.op == six.u("GET_COMMITTED_VERSION"):
self.last_version = inst.tr.get_committed_version()
inst.push(b"GOT_COMMITTED_VERSION")
elif inst.op == six.u("GET_APPROXIMATE_SIZE"):
approximate_size = inst.tr.get_approximate_size().wait()
inst.push(b"GOT_APPROXIMATE_SIZE")
elif inst.op == six.u("GET_VERSIONSTAMP"):
inst.push(inst.tr.get_versionstamp())
elif inst.op == six.u("TUPLE_PACK"):
count = inst.pop()
items = inst.pop(count)
inst.push(fdb.tuple.pack(tuple(items)))
elif inst.op == six.u("TUPLE_PACK_WITH_VERSIONSTAMP"):
prefix = inst.pop()
count = inst.pop()
items = inst.pop(count)
if not fdb.tuple.has_incomplete_versionstamp(items) and random.random() < 0.5:
inst.push(b"ERROR: NONE")
else:
try:
packed = fdb.tuple.pack_with_versionstamp(tuple(items), prefix=prefix)
inst.push(b"OK")
inst.push(packed)
except ValueError as e:
if str(e).startswith("No incomplete"):
inst.push(b"ERROR: NONE")
else:
inst.push(b"ERROR: MULTIPLE")
elif inst.op == six.u("TUPLE_UNPACK"):
for i in fdb.tuple.unpack(inst.pop()):
inst.push(fdb.tuple.pack((i,)))
elif inst.op == six.u("TUPLE_SORT"):
count = inst.pop()
items = inst.pop(count)
unpacked = map(fdb.tuple.unpack, items)
if six.PY3:
sorted_items = sorted(unpacked, key=fdb.tuple.pack)
else:
sorted_items = sorted(unpacked, cmp=fdb.tuple.compare)
for item in sorted_items:
inst.push(fdb.tuple.pack(item))
elif inst.op == six.u("TUPLE_RANGE"):
count = inst.pop()
items = inst.pop(count)
r = fdb.tuple.range(tuple(items))
inst.push(r.start)
inst.push(r.stop)
elif inst.op == six.u("ENCODE_FLOAT"):
f_bytes = inst.pop()
f = struct.unpack(">f", f_bytes)[0]
if not math.isnan(f) and not math.isinf(f) and not f == -0.0 and f == int(f):
f = int(f)
inst.push(fdb.tuple.SingleFloat(f))
elif inst.op == six.u("ENCODE_DOUBLE"):
d_bytes = inst.pop()
d = struct.unpack(">d", d_bytes)[0]
inst.push(d)
elif inst.op == six.u("DECODE_FLOAT"):
f = inst.pop()
f_bytes = struct.pack(">f", f.value)
inst.push(f_bytes)
elif inst.op == six.u("DECODE_DOUBLE"):
d = inst.pop()
d_bytes = struct.pack(">d", d)
inst.push(d_bytes)
elif inst.op == six.u("START_THREAD"):
t = Tester(self.db, inst.pop())
thr = threading.Thread(target=t.run)
thr.start()
self.threads.append(thr)
elif inst.op == six.u("WAIT_EMPTY"):
prefix = inst.pop()
Tester.wait_empty(self.db, prefix)
inst.push(b"WAITED_FOR_EMPTY")
elif inst.op == six.u("UNIT_TESTS"):
try:
test_db_options(db)
test_options(db)
test_watches(db)
test_cancellation(db)
test_retry_limits(db)
test_db_retry_limits(db)
test_timeouts(db)
test_db_timeouts(db)
test_combinations(db)
test_locality(db)
test_predicates()
test_size_limit_option(db)
test_get_approximate_size(db)
except fdb.FDBError as e:
print("Unit tests failed: %s" % e.description)
traceback.print_exc()
raise Exception("Unit tests failed: %s" % e.description)
elif inst.op.startswith(six.u('DIRECTORY_')):
self.directory_extension.process_instruction(inst)
else:
raise Exception("Unknown op %s" % inst.op)
except fdb.FDBError as e:
# print('ERROR: %r' % e)
inst.stack.push(idx, fdb.tuple.pack((b"ERROR", str(e.code).encode('ascii'))))
# print(" to %s" % self.stack)
# print()
[thr.join() for thr in self.threads]
def pack(self, t=tuple()):
return self.rawPrefix + fdb.tuple.pack(t)
def range(self, t=tuple()):
p = fdb.tuple.range(t)
return slice(self.rawPrefix + p.start, self.rawPrefix + p.stop)
def pack_with_versionstamp(self, t=tuple()):
return fdb.tuple.pack_with_versionstamp(t, prefix=self.rawPrefix)
def pack(self, t=tuple()):
return self.rawPrefix + fdb.tuple.pack(t)
def __init__(self, prefixTuple=tuple(), rawPrefix=b''):
self.rawPrefix = fdb.tuple.pack(prefixTuple, prefix=rawPrefix)
def __init__(self, prefixTuple=tuple(), rawPrefix=""):
self.rawPrefix = rawPrefix + fdb.tuple.pack(prefixTuple)
def __init__(self, prefixTuple=tuple(), rawPrefix=""):
self.rawPrefix = rawPrefix + fdb.tuple.pack(prefixTuple)
def range(self, t=tuple()):
raise Exception(
'Cannot get range for the root of a directory partition.')
def pack(self, t=tuple()):
raise Exception(
'Cannot pack keys using the root of a directory partition.')
def pack(self, t=tuple()):
return fdb.tuple.pack(t, prefix=self.rawPrefix)
def range(self, t=tuple()):
p = fdb.tuple.range(t)
return slice(self.rawPrefix + p.start, self.rawPrefix + p.stop)