def generate(self, args, thread_number):
instructions = InstructionSet()
instructions.append('NEW_TRANSACTION')
default_path = unicode('default%d' % self.next_path)
self.next_path += 1
dir_list = directory_util.setup_directories(instructions, default_path,
self.random)
num_dirs = len(dir_list)
instructions.push_args(directory_util.DEFAULT_DIRECTORY_INDEX)
instructions.append('DIRECTORY_CHANGE')
instructions.setup_complete()
current_op = 0
while current_op < args.num_ops:
if args.concurrency > 1:
self.barrier(instructions, thread_number)
instructions.push_args(random.choice(self.transactions))
instructions.append('USE_TRANSACTION')
if thread_number == 0 and args.concurrency > 1:
num_directories = 1
else:
num_directories = int(
max(
1,
pow(random.random(), 4) *
min(self.max_directories_per_transaction,
args.num_ops - current_op)))
for i in range(num_directories):
path = (self.random.random_unicode_str(16), )
op_args = test_util.with_length(path) + ('', None)
directory_util.push_instruction_and_record_prefix(
instructions, 'DIRECTORY_CREATE', op_args, path, num_dirs,
self.random, self.prefix_log)
num_dirs += 1
current_op += num_directories
if args.concurrency > 1:
self.barrier(instructions,
thread_number,
thread_ending=(current_op >= args.num_ops))
if thread_number == 0:
self.commit_transactions(instructions, args)
return instructions
def generate(self, args, thread_number):
instructions = InstructionSet()
op_choices = ['NEW_TRANSACTION', 'COMMIT']
general = ['DIRECTORY_CREATE_SUBSPACE', 'DIRECTORY_CREATE_LAYER']
op_choices += general
directory_mutations = ['DIRECTORY_CREATE_OR_OPEN', 'DIRECTORY_CREATE', 'DIRECTORY_MOVE', 'DIRECTORY_MOVE_TO',
'DIRECTORY_REMOVE', 'DIRECTORY_REMOVE_IF_EXISTS']
directory_reads = ['DIRECTORY_EXISTS', 'DIRECTORY_OPEN', 'DIRECTORY_LIST']
directory_db_mutations = [x + '_DATABASE' for x in directory_mutations]
directory_db_reads = [x + '_DATABASE' for x in directory_reads]
directory_snapshot_reads = [x + '_SNAPSHOT' for x in directory_reads]
directory = []
directory += directory_mutations
directory += directory_reads
directory += directory_db_mutations
directory += directory_db_reads
if not args.no_directory_snapshot_ops:
directory += directory_snapshot_reads
subspace = ['DIRECTORY_PACK_KEY', 'DIRECTORY_UNPACK_KEY', 'DIRECTORY_RANGE', 'DIRECTORY_CONTAINS', 'DIRECTORY_OPEN_SUBSPACE']
instructions.append('NEW_TRANSACTION')
default_path = unicode('default%d' % self.next_path)
self.next_path += 1
self.dir_list = directory_util.setup_directories(instructions, default_path, self.random)
self.root = self.dir_list[0]
instructions.push_args(0)
instructions.append('DIRECTORY_CHANGE')
# Generate some directories that we are going to create in advance. This tests that other bindings
# are compatible with the Python implementation
self.prepopulated_dirs = [(generate_path(min_length=1), self.generate_layer()) for i in range(5)]
for path, layer in self.prepopulated_dirs:
instructions.push_args(layer)
instructions.push_args(*test_util.with_length(path))
instructions.append('DIRECTORY_OPEN')
self.dir_list.append(self.root.add_child(path, DirectoryStateTreeNode(True, True, has_known_prefix=False, is_partition=(layer=='partition'))))
# print('%d. Selected %s, dir=%s, dir_id=%s, has_known_prefix=%s, dir_list_len=%d' \
# % (len(instructions), 'DIRECTORY_OPEN', repr(self.dir_index), self.dir_list[-1].dir_id, False, len(self.dir_list)-1))
instructions.setup_complete()
for i in range(args.num_ops):
if random.random() < 0.5:
while True:
self.dir_index = random.randrange(0, len(self.dir_list))
if not self.dir_list[self.dir_index].state.is_partition or not self.dir_list[self.dir_index].state.deleted:
break
instructions.push_args(self.dir_index)
instructions.append('DIRECTORY_CHANGE')
dir_entry = self.dir_list[self.dir_index]
choices = op_choices[:]
if dir_entry.state.is_directory:
choices += directory
if dir_entry.state.is_subspace:
choices += subspace
op = random.choice(choices)
# print('%d. Selected %s, dir=%d, dir_id=%d, has_known_prefix=%d, dir_list_len=%d' \
# % (len(instructions), op, self.dir_index, dir_entry.dir_id, dir_entry.state.has_known_prefix, len(self.dir_list)))
if op.endswith('_DATABASE') or op.endswith('_SNAPSHOT'):
root_op = op[0:-9]
else:
root_op = op
if root_op == 'NEW_TRANSACTION':
instructions.append(op)
elif root_op == 'COMMIT':
test_util.blocking_commit(instructions)
elif root_op == 'DIRECTORY_CREATE_SUBSPACE':
path = generate_path()
instructions.push_args(generate_prefix(require_unique=False, is_partition=True))
instructions.push_args(*test_util.with_length(path))
instructions.append(op)
self.dir_list.append(DirectoryStateTreeNode(False, True, has_known_prefix=True))
elif root_op == 'DIRECTORY_CREATE_LAYER':
indices = []
prefixes = [generate_prefix(require_unique=args.concurrency==1, is_partition=True) for i in range(2)]
for i in range(2):
instructions.push_args(prefixes[i])
instructions.push_args(*test_util.with_length(generate_path()))
instructions.append('DIRECTORY_CREATE_SUBSPACE')
indices.append(len(self.dir_list))
self.dir_list.append(DirectoryStateTreeNode(False, True, has_known_prefix=True))
instructions.push_args(random.choice([0, 1]))
instructions.push_args(*indices)
instructions.append(op)
self.dir_list.append(DirectoryStateTreeNode.get_layer(prefixes[0]))
elif root_op == 'DIRECTORY_CREATE_OR_OPEN':
# Because allocated prefixes are non-deterministic, we cannot have overlapping
# transactions that allocate/remove these prefixes in a comparison test
if op.endswith('_DATABASE') and args.concurrency == 1:
test_util.blocking_commit(instructions)
path = generate_path()
op_args = test_util.with_length(path) + (self.generate_layer(),)
directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log)
if not op.endswith('_DATABASE') and args.concurrency == 1:
test_util.blocking_commit(instructions)
child_entry = dir_entry.get_descendent(path)
if child_entry is None:
child_entry = DirectoryStateTreeNode(True, True)
child_entry.state.has_known_prefix = False
self.dir_list.append(dir_entry.add_child(path, child_entry))
elif root_op == 'DIRECTORY_CREATE':
layer = self.generate_layer()
is_partition = layer == 'partition'
prefix = generate_prefix(require_unique=is_partition and args.concurrency==1, is_partition=is_partition, min_length=0)
# Because allocated prefixes are non-deterministic, we cannot have overlapping
# transactions that allocate/remove these prefixes in a comparison test
if op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix:
test_util.blocking_commit(instructions)
path = generate_path()
op_args = test_util.with_length(path) + (layer, prefix)
if prefix is None:
directory_util.push_instruction_and_record_prefix(
instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log)
else:
instructions.push_args(*op_args)
instructions.append(op)
if not op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix:
test_util.blocking_commit(instructions)
child_entry = dir_entry.get_descendent(path)
if child_entry is None:
child_entry = DirectoryStateTreeNode(True, True, has_known_prefix=bool(prefix))
elif not bool(prefix):
child_entry.state.has_known_prefix = False
if is_partition:
child_entry.state.is_partition = True
self.dir_list.append(dir_entry.add_child(path, child_entry))
elif root_op == 'DIRECTORY_OPEN':
path = generate_path()
instructions.push_args(self.generate_layer())
instructions.push_args(*test_util.with_length(path))
instructions.append(op)
child_entry = dir_entry.get_descendent(path)
if child_entry is None:
self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False))
else:
self.dir_list.append(dir_entry.add_child(path, child_entry))
elif root_op == 'DIRECTORY_MOVE':
old_path = generate_path()
new_path = generate_path()
instructions.push_args(*(test_util.with_length(old_path) + test_util.with_length(new_path)))
instructions.append(op)
child_entry = dir_entry.get_descendent(old_path)
if child_entry is None:
self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False))
else:
self.dir_list.append(dir_entry.add_child(new_path, child_entry))
# Make sure that the default directory subspace still exists after moving the specified directory
if dir_entry.state.is_directory and not dir_entry.state.is_subspace and old_path == (u'',):
self.ensure_default_directory_subspace(instructions, default_path)
elif root_op == 'DIRECTORY_MOVE_TO':
new_path = generate_path()
instructions.push_args(*test_util.with_length(new_path))
instructions.append(op)
child_entry = dir_entry.get_descendent(())
if child_entry is None:
self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False))
else:
self.dir_list.append(dir_entry.add_child(new_path, child_entry))
# Make sure that the default directory subspace still exists after moving the current directory
self.ensure_default_directory_subspace(instructions, default_path)
elif root_op == 'DIRECTORY_REMOVE' or root_op == 'DIRECTORY_REMOVE_IF_EXISTS':
# Because allocated prefixes are non-deterministic, we cannot have overlapping
# transactions that allocate/remove these prefixes in a comparison test
if op.endswith('_DATABASE') and args.concurrency == 1:
test_util.blocking_commit(instructions)
path = ()
count = random.randint(0, 1)
if count == 1:
path = generate_path()
instructions.push_args(*test_util.with_length(path))
instructions.push_args(count)
instructions.append(op)
dir_entry.delete(path)
# Make sure that the default directory subspace still exists after removing the specified directory
if path == () or (dir_entry.state.is_directory and not dir_entry.state.is_subspace and path == (u'',)):
self.ensure_default_directory_subspace(instructions, default_path)
elif root_op == 'DIRECTORY_LIST' or root_op == 'DIRECTORY_EXISTS':
path = ()
count = random.randint(0, 1)
if count == 1:
path = generate_path()
instructions.push_args(*test_util.with_length(path))
instructions.push_args(count)
instructions.append(op)
elif root_op == 'DIRECTORY_PACK_KEY':
t = self.random.random_tuple(5)
instructions.push_args(*test_util.with_length(t))
instructions.append(op)
instructions.append('DIRECTORY_STRIP_PREFIX')
elif root_op == 'DIRECTORY_UNPACK_KEY' or root_op == 'DIRECTORY_CONTAINS':
if not dir_entry.state.has_known_prefix or random.random() < 0.2 or root_op == 'DIRECTORY_UNPACK_KEY':
t = self.random.random_tuple(5)
instructions.push_args(*test_util.with_length(t))
instructions.append('DIRECTORY_PACK_KEY')
instructions.append(op)
else:
instructions.push_args(fdb.tuple.pack(self.random.random_tuple(5)))
instructions.append(op)
elif root_op == 'DIRECTORY_RANGE' or root_op == 'DIRECTORY_OPEN_SUBSPACE':
t = self.random.random_tuple(5)
instructions.push_args(*test_util.with_length(t))
instructions.append(op)
if root_op == 'DIRECTORY_OPEN_SUBSPACE':
self.dir_list.append(DirectoryStateTreeNode(False, True, dir_entry.state.has_known_prefix))
else:
test_util.to_front(instructions, 1)
instructions.append('DIRECTORY_STRIP_PREFIX')
test_util.to_front(instructions, 1)
instructions.append('DIRECTORY_STRIP_PREFIX')
instructions.begin_finalization()
test_util.blocking_commit(instructions)
instructions.append('NEW_TRANSACTION')
for i, dir_entry in enumerate(self.dir_list):
instructions.push_args(i)
instructions.append('DIRECTORY_CHANGE')
if dir_entry.state.is_directory:
instructions.push_args(self.directory_log.key())
instructions.append('DIRECTORY_LOG_DIRECTORY')
if dir_entry.state.has_known_prefix and dir_entry.state.is_subspace:
# print('%d. Logging subspace: %d' % (i, dir_entry.dir_id))
instructions.push_args(self.subspace_log.key())
instructions.append('DIRECTORY_LOG_SUBSPACE')
if (i + 1) % 100 == 0:
test_util.blocking_commit(instructions)
test_util.blocking_commit(instructions)
instructions.push_args(self.stack_subspace.key())
instructions.append('LOG_STACK')
test_util.blocking_commit(instructions)
return instructions
