def generate(self, args, thread_number): instructions = InstructionSet() instructions.append('NEW_TRANSACTION') default_path = unicode('default%d' % self.next_path) self.next_path += 1 dir_list = directory_util.setup_directories(instructions, default_path, self.random) num_dirs = len(dir_list) instructions.push_args(directory_util.DEFAULT_DIRECTORY_INDEX) instructions.append('DIRECTORY_CHANGE') instructions.setup_complete() current_op = 0 while current_op < args.num_ops: if args.concurrency > 1: self.barrier(instructions, thread_number) instructions.push_args(random.choice(self.transactions)) instructions.append('USE_TRANSACTION') if thread_number == 0 and args.concurrency > 1: num_directories = 1 else: num_directories = int( max( 1, pow(random.random(), 4) * min(self.max_directories_per_transaction, args.num_ops - current_op))) for i in range(num_directories): path = (self.random.random_unicode_str(16), ) op_args = test_util.with_length(path) + ('', None) directory_util.push_instruction_and_record_prefix( instructions, 'DIRECTORY_CREATE', op_args, path, num_dirs, self.random, self.prefix_log) num_dirs += 1 current_op += num_directories if args.concurrency > 1: self.barrier(instructions, thread_number, thread_ending=(current_op >= args.num_ops)) if thread_number == 0: self.commit_transactions(instructions, args) return instructions
def generate(self, args, thread_number): instructions = InstructionSet() op_choices = ['NEW_TRANSACTION', 'COMMIT'] general = ['DIRECTORY_CREATE_SUBSPACE', 'DIRECTORY_CREATE_LAYER'] op_choices += general directory_mutations = ['DIRECTORY_CREATE_OR_OPEN', 'DIRECTORY_CREATE', 'DIRECTORY_MOVE', 'DIRECTORY_MOVE_TO', 'DIRECTORY_REMOVE', 'DIRECTORY_REMOVE_IF_EXISTS'] directory_reads = ['DIRECTORY_EXISTS', 'DIRECTORY_OPEN', 'DIRECTORY_LIST'] directory_db_mutations = [x + '_DATABASE' for x in directory_mutations] directory_db_reads = [x + '_DATABASE' for x in directory_reads] directory_snapshot_reads = [x + '_SNAPSHOT' for x in directory_reads] directory = [] directory += directory_mutations directory += directory_reads directory += directory_db_mutations directory += directory_db_reads if not args.no_directory_snapshot_ops: directory += directory_snapshot_reads subspace = ['DIRECTORY_PACK_KEY', 'DIRECTORY_UNPACK_KEY', 'DIRECTORY_RANGE', 'DIRECTORY_CONTAINS', 'DIRECTORY_OPEN_SUBSPACE'] instructions.append('NEW_TRANSACTION') default_path = unicode('default%d' % self.next_path) self.next_path += 1 self.dir_list = directory_util.setup_directories(instructions, default_path, self.random) self.root = self.dir_list[0] instructions.push_args(0) instructions.append('DIRECTORY_CHANGE') # Generate some directories that we are going to create in advance. This tests that other bindings # are compatible with the Python implementation self.prepopulated_dirs = [(generate_path(min_length=1), self.generate_layer()) for i in range(5)] for path, layer in self.prepopulated_dirs: instructions.push_args(layer) instructions.push_args(*test_util.with_length(path)) instructions.append('DIRECTORY_OPEN') self.dir_list.append(self.root.add_child(path, DirectoryStateTreeNode(True, True, has_known_prefix=False, is_partition=(layer=='partition')))) # print('%d. Selected %s, dir=%s, dir_id=%s, has_known_prefix=%s, dir_list_len=%d' \ # % (len(instructions), 'DIRECTORY_OPEN', repr(self.dir_index), self.dir_list[-1].dir_id, False, len(self.dir_list)-1)) instructions.setup_complete() for i in range(args.num_ops): if random.random() < 0.5: while True: self.dir_index = random.randrange(0, len(self.dir_list)) if not self.dir_list[self.dir_index].state.is_partition or not self.dir_list[self.dir_index].state.deleted: break instructions.push_args(self.dir_index) instructions.append('DIRECTORY_CHANGE') dir_entry = self.dir_list[self.dir_index] choices = op_choices[:] if dir_entry.state.is_directory: choices += directory if dir_entry.state.is_subspace: choices += subspace op = random.choice(choices) # print('%d. Selected %s, dir=%d, dir_id=%d, has_known_prefix=%d, dir_list_len=%d' \ # % (len(instructions), op, self.dir_index, dir_entry.dir_id, dir_entry.state.has_known_prefix, len(self.dir_list))) if op.endswith('_DATABASE') or op.endswith('_SNAPSHOT'): root_op = op[0:-9] else: root_op = op if root_op == 'NEW_TRANSACTION': instructions.append(op) elif root_op == 'COMMIT': test_util.blocking_commit(instructions) elif root_op == 'DIRECTORY_CREATE_SUBSPACE': path = generate_path() instructions.push_args(generate_prefix(require_unique=False, is_partition=True)) instructions.push_args(*test_util.with_length(path)) instructions.append(op) self.dir_list.append(DirectoryStateTreeNode(False, True, has_known_prefix=True)) elif root_op == 'DIRECTORY_CREATE_LAYER': indices = [] prefixes = [generate_prefix(require_unique=args.concurrency==1, is_partition=True) for i in range(2)] for i in range(2): instructions.push_args(prefixes[i]) instructions.push_args(*test_util.with_length(generate_path())) instructions.append('DIRECTORY_CREATE_SUBSPACE') indices.append(len(self.dir_list)) self.dir_list.append(DirectoryStateTreeNode(False, True, has_known_prefix=True)) instructions.push_args(random.choice([0, 1])) instructions.push_args(*indices) instructions.append(op) self.dir_list.append(DirectoryStateTreeNode.get_layer(prefixes[0])) elif root_op == 'DIRECTORY_CREATE_OR_OPEN': # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) path = generate_path() op_args = test_util.with_length(path) + (self.generate_layer(),) directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) if not op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) child_entry = dir_entry.get_descendent(path) if child_entry is None: child_entry = DirectoryStateTreeNode(True, True) child_entry.state.has_known_prefix = False self.dir_list.append(dir_entry.add_child(path, child_entry)) elif root_op == 'DIRECTORY_CREATE': layer = self.generate_layer() is_partition = layer == 'partition' prefix = generate_prefix(require_unique=is_partition and args.concurrency==1, is_partition=is_partition, min_length=0) # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) path = generate_path() op_args = test_util.with_length(path) + (layer, prefix) if prefix is None: directory_util.push_instruction_and_record_prefix( instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) else: instructions.push_args(*op_args) instructions.append(op) if not op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) child_entry = dir_entry.get_descendent(path) if child_entry is None: child_entry = DirectoryStateTreeNode(True, True, has_known_prefix=bool(prefix)) elif not bool(prefix): child_entry.state.has_known_prefix = False if is_partition: child_entry.state.is_partition = True self.dir_list.append(dir_entry.add_child(path, child_entry)) elif root_op == 'DIRECTORY_OPEN': path = generate_path() instructions.push_args(self.generate_layer()) instructions.push_args(*test_util.with_length(path)) instructions.append(op) child_entry = dir_entry.get_descendent(path) if child_entry is None: self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False)) else: self.dir_list.append(dir_entry.add_child(path, child_entry)) elif root_op == 'DIRECTORY_MOVE': old_path = generate_path() new_path = generate_path() instructions.push_args(*(test_util.with_length(old_path) + test_util.with_length(new_path))) instructions.append(op) child_entry = dir_entry.get_descendent(old_path) if child_entry is None: self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False)) else: self.dir_list.append(dir_entry.add_child(new_path, child_entry)) # Make sure that the default directory subspace still exists after moving the specified directory if dir_entry.state.is_directory and not dir_entry.state.is_subspace and old_path == (u'',): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_MOVE_TO': new_path = generate_path() instructions.push_args(*test_util.with_length(new_path)) instructions.append(op) child_entry = dir_entry.get_descendent(()) if child_entry is None: self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False)) else: self.dir_list.append(dir_entry.add_child(new_path, child_entry)) # Make sure that the default directory subspace still exists after moving the current directory self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_REMOVE' or root_op == 'DIRECTORY_REMOVE_IF_EXISTS': # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) dir_entry.delete(path) # Make sure that the default directory subspace still exists after removing the specified directory if path == () or (dir_entry.state.is_directory and not dir_entry.state.is_subspace and path == (u'',)): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_LIST' or root_op == 'DIRECTORY_EXISTS': path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) elif root_op == 'DIRECTORY_PACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append(op) instructions.append('DIRECTORY_STRIP_PREFIX') elif root_op == 'DIRECTORY_UNPACK_KEY' or root_op == 'DIRECTORY_CONTAINS': if not dir_entry.state.has_known_prefix or random.random() < 0.2 or root_op == 'DIRECTORY_UNPACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append('DIRECTORY_PACK_KEY') instructions.append(op) else: instructions.push_args(fdb.tuple.pack(self.random.random_tuple(5))) instructions.append(op) elif root_op == 'DIRECTORY_RANGE' or root_op == 'DIRECTORY_OPEN_SUBSPACE': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append(op) if root_op == 'DIRECTORY_OPEN_SUBSPACE': self.dir_list.append(DirectoryStateTreeNode(False, True, dir_entry.state.has_known_prefix)) else: test_util.to_front(instructions, 1) instructions.append('DIRECTORY_STRIP_PREFIX') test_util.to_front(instructions, 1) instructions.append('DIRECTORY_STRIP_PREFIX') instructions.begin_finalization() test_util.blocking_commit(instructions) instructions.append('NEW_TRANSACTION') for i, dir_entry in enumerate(self.dir_list): instructions.push_args(i) instructions.append('DIRECTORY_CHANGE') if dir_entry.state.is_directory: instructions.push_args(self.directory_log.key()) instructions.append('DIRECTORY_LOG_DIRECTORY') if dir_entry.state.has_known_prefix and dir_entry.state.is_subspace: # print('%d. Logging subspace: %d' % (i, dir_entry.dir_id)) instructions.push_args(self.subspace_log.key()) instructions.append('DIRECTORY_LOG_SUBSPACE') if (i + 1) % 100 == 0: test_util.blocking_commit(instructions) test_util.blocking_commit(instructions) instructions.push_args(self.stack_subspace.key()) instructions.append('LOG_STACK') test_util.blocking_commit(instructions) return instructions