def generate(self, args, thread_number): instructions = InstructionSet() op_choices = ['NEW_TRANSACTION', 'COMMIT'] reads = [ 'GET', 'GET_KEY', 'GET_RANGE', 'GET_RANGE_STARTS_WITH', 'GET_RANGE_SELECTOR' ] mutations = [ 'SET', 'CLEAR', 'CLEAR_RANGE', 'CLEAR_RANGE_STARTS_WITH', 'ATOMIC_OP' ] snapshot_reads = [x + '_SNAPSHOT' for x in reads] database_reads = [x + '_DATABASE' for x in reads] database_mutations = [x + '_DATABASE' for x in mutations] mutations += ['VERSIONSTAMP'] versions = [ 'GET_READ_VERSION', 'SET_READ_VERSION', 'GET_COMMITTED_VERSION' ] snapshot_versions = ['GET_READ_VERSION_SNAPSHOT'] tuples = [ 'TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT' ] if 'versionstamp' in args.types: tuples.append('TUPLE_PACK_WITH_VERSIONSTAMP') resets = ['ON_ERROR', 'RESET', 'CANCEL'] read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = [ 'WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT' ] txn_sizes = ['GET_APPROXIMATE_SIZE'] op_choices += reads op_choices += mutations op_choices += snapshot_reads op_choices += database_reads op_choices += database_mutations op_choices += versions op_choices += snapshot_versions op_choices += tuples op_choices += read_conflicts op_choices += write_conflicts op_choices += resets op_choices += txn_sizes idempotent_atomic_ops = [ 'BIT_AND', 'BIT_OR', 'MAX', 'MIN', 'BYTE_MIN', 'BYTE_MAX' ] atomic_ops = idempotent_atomic_ops + [ 'ADD', 'BIT_XOR', 'APPEND_IF_FITS' ] if args.concurrency > 1: self.max_keys = random.randint(100, 1000) else: self.max_keys = random.randint(100, 10000) instructions.append('NEW_TRANSACTION') instructions.append('GET_READ_VERSION') self.preload_database(instructions, self.max_keys) instructions.setup_complete() for i in range(args.num_ops): op = random.choice(op_choices) index = len(instructions) read_performed = False # print 'Adding instruction %s at %d' % (op, index) if args.concurrency == 1 and (op in database_mutations): self.wait_for_reads(instructions) test_util.blocking_commit(instructions) self.can_get_commit_version = False self.add_stack_items(1) if op in resets or op == 'NEW_TRANSACTION': if args.concurrency == 1: self.wait_for_reads(instructions) self.outstanding_ops = [] if op == 'NEW_TRANSACTION': instructions.append(op) self.can_get_commit_version = True self.can_set_version = True self.can_use_key_selectors = True elif op == 'ON_ERROR': instructions.push_args(random.randint(0, 5000)) instructions.append(op) self.outstanding_ops.append( (self.stack_size, len(instructions) - 1)) if args.concurrency == 1: self.wait_for_reads(instructions) instructions.append('NEW_TRANSACTION') self.can_get_commit_version = True self.can_set_version = True self.can_use_key_selectors = True self.add_strings(1) elif op == 'GET' or op == 'GET_SNAPSHOT' or op == 'GET_DATABASE': self.ensure_key(instructions, 1) instructions.append(op) self.add_strings(1) self.can_set_version = False read_performed = True elif op == 'GET_KEY' or op == 'GET_KEY_SNAPSHOT' or op == 'GET_KEY_DATABASE': if op.endswith('_DATABASE') or self.can_use_key_selectors: self.ensure_key(instructions, 1) instructions.push_args(self.workspace.key()) instructions.push_args( *self.random.random_selector_params()) test_util.to_front(instructions, 3) instructions.append(op) # Don't add key here because we may be outside of our prefix self.add_strings(1) self.can_set_version = False read_performed = True elif op == 'GET_RANGE' or op == 'GET_RANGE_SNAPSHOT' or op == 'GET_RANGE_DATABASE': self.ensure_key(instructions, 2) range_params = self.random.random_range_params() instructions.push_args(*range_params) test_util.to_front(instructions, 4) test_util.to_front(instructions, 4) instructions.append(op) if range_params[0] >= 1 and range_params[ 0] <= 1000: # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) self.can_set_version = False read_performed = True elif op == 'GET_RANGE_STARTS_WITH' or op == 'GET_RANGE_STARTS_WITH_SNAPSHOT' or op == 'GET_RANGE_STARTS_WITH_DATABASE': # TODO: not tested well self.ensure_key(instructions, 1) range_params = self.random.random_range_params() instructions.push_args(*range_params) test_util.to_front(instructions, 3) instructions.append(op) if range_params[0] >= 1 and range_params[ 0] <= 1000: # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) self.can_set_version = False read_performed = True elif op == 'GET_RANGE_SELECTOR' or op == 'GET_RANGE_SELECTOR_SNAPSHOT' or op == 'GET_RANGE_SELECTOR_DATABASE': if op.endswith('_DATABASE') or self.can_use_key_selectors: self.ensure_key(instructions, 2) instructions.push_args(self.workspace.key()) range_params = self.random.random_range_params() instructions.push_args(*range_params) instructions.push_args( *self.random.random_selector_params()) test_util.to_front(instructions, 6) instructions.push_args( *self.random.random_selector_params()) test_util.to_front(instructions, 9) instructions.append(op) if range_params[0] >= 1 and range_params[ 0] <= 1000: # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) self.can_set_version = False read_performed = True elif op == 'GET_READ_VERSION' or op == 'GET_READ_VERSION_SNAPSHOT': instructions.append(op) self.has_version = self.can_set_version self.add_strings(1) elif op == 'SET' or op == 'SET_DATABASE': self.ensure_key_value(instructions) instructions.append(op) if op == 'SET_DATABASE': self.add_stack_items(1) elif op == 'SET_READ_VERSION': if self.has_version and self.can_set_version: instructions.append(op) self.can_set_version = False elif op == 'CLEAR' or op == 'CLEAR_DATABASE': self.ensure_key(instructions, 1) instructions.append(op) if op == 'CLEAR_DATABASE': self.add_stack_items(1) elif op == 'CLEAR_RANGE' or op == 'CLEAR_RANGE_DATABASE': # Protect against inverted range key1 = self.workspace.pack(self.random.random_tuple(5)) key2 = self.workspace.pack(self.random.random_tuple(5)) if key1 > key2: key1, key2 = key2, key1 instructions.push_args(key1, key2) instructions.append(op) if op == 'CLEAR_RANGE_DATABASE': self.add_stack_items(1) elif op == 'CLEAR_RANGE_STARTS_WITH' or op == 'CLEAR_RANGE_STARTS_WITH_DATABASE': self.ensure_key(instructions, 1) instructions.append(op) if op == 'CLEAR_RANGE_STARTS_WITH_DATABASE': self.add_stack_items(1) elif op == 'ATOMIC_OP' or op == 'ATOMIC_OP_DATABASE': self.ensure_key_value(instructions) if op == 'ATOMIC_OP' or args.concurrency > 1: instructions.push_args(random.choice(atomic_ops)) else: instructions.push_args( random.choice(idempotent_atomic_ops)) instructions.append(op) if op == 'ATOMIC_OP_DATABASE': self.add_stack_items(1) elif op == 'VERSIONSTAMP': rand_str1 = self.random.random_string(100) key1 = self.versionstamped_values.pack((rand_str1, )) key2 = self.versionstamped_values_2.pack((rand_str1, )) split = random.randint(0, 70) prefix = self.random.random_string(20 + split) if prefix.endswith(b'\xff'): # Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check # correctly finds where the version is supposed to fit in. prefix += b'\x00' suffix = self.random.random_string(70 - split) rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix key3 = self.versionstamped_keys.pack() + rand_str2 index = len(self.versionstamped_keys.pack()) + len(prefix) key3 = self.versionstamp_key(key3, index) instructions.push_args( 'SET_VERSIONSTAMPED_VALUE', key1, self.versionstamp_value( fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2)) instructions.append('ATOMIC_OP') if args.api_version >= 520: instructions.push_args( 'SET_VERSIONSTAMPED_VALUE', key2, self.versionstamp_value(rand_str2, len(prefix))) instructions.append('ATOMIC_OP') instructions.push_args('SET_VERSIONSTAMPED_KEY', key3, rand_str1) instructions.append('ATOMIC_OP') self.can_use_key_selectors = False elif op == 'READ_CONFLICT_RANGE' or op == 'WRITE_CONFLICT_RANGE': self.ensure_key(instructions, 2) instructions.append(op) self.add_strings(1) elif op == 'READ_CONFLICT_KEY' or op == 'WRITE_CONFLICT_KEY': self.ensure_key(instructions, 1) instructions.append(op) self.add_strings(1) elif op == 'DISABLE_WRITE_CONFLICT': instructions.append(op) elif op == 'COMMIT': if args.concurrency == 1 or i < self.max_keys or random.random( ) < 0.9: if args.concurrency == 1: self.wait_for_reads(instructions) test_util.blocking_commit(instructions) self.can_get_commit_version = False self.add_stack_items(1) self.can_set_version = True self.can_use_key_selectors = True else: instructions.append(op) self.add_strings(1) elif op == 'RESET': instructions.append(op) self.can_get_commit_version = False self.can_set_version = True self.can_use_key_selectors = True elif op == 'CANCEL': instructions.append(op) self.can_set_version = False elif op == 'GET_COMMITTED_VERSION': if self.can_get_commit_version: do_commit = random.random() < 0.5 if do_commit: instructions.append('COMMIT') instructions.append('WAIT_FUTURE') self.add_stack_items(1) instructions.append(op) self.has_version = True self.add_strings(1) if do_commit: instructions.append('RESET') self.can_get_commit_version = False self.can_set_version = True self.can_use_key_selectors = True elif op == 'GET_APPROXIMATE_SIZE': instructions.append(op) self.add_strings(1) elif op == 'TUPLE_PACK' or op == 'TUPLE_RANGE': tup = self.random.random_tuple(10) instructions.push_args(len(tup), *tup) instructions.append(op) if op == 'TUPLE_PACK': self.add_strings(1) else: self.add_strings(2) elif op == 'TUPLE_PACK_WITH_VERSIONSTAMP': tup = (self.random.random_string(20), ) + self.random.random_tuple( 10, incomplete_versionstamps=True) prefix = self.versionstamped_keys.pack() instructions.push_args(prefix, len(tup), *tup) instructions.append(op) self.add_strings(1) versionstamp_param = prefix + fdb.tuple.pack(tup) first_incomplete = versionstamp_param.find( fdb.tuple.Versionstamp._UNSET_TR_VERSION) second_incomplete = -1 if first_incomplete < 0 else \ versionstamp_param.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1) # If there is exactly one incomplete versionstamp, perform the versionstamp operation. if first_incomplete >= 0 and second_incomplete < 0: rand_str = self.random.random_string(100) instructions.push_args(rand_str) test_util.to_front(instructions, 1) instructions.push_args('SET_VERSIONSTAMPED_KEY') instructions.append('ATOMIC_OP') if self.api_version >= 520: version_value_key_2 = self.versionstamped_values_2.pack( (rand_str, )) versionstamped_value = self.versionstamp_value( fdb.tuple.pack(tup), first_incomplete - len(prefix)) instructions.push_args('SET_VERSIONSTAMPED_VALUE', version_value_key_2, versionstamped_value) instructions.append('ATOMIC_OP') version_value_key = self.versionstamped_values.pack( (rand_str, )) instructions.push_args( 'SET_VERSIONSTAMPED_VALUE', version_value_key, self.versionstamp_value( fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup))) instructions.append('ATOMIC_OP') self.can_use_key_selectors = False elif op == 'TUPLE_UNPACK': tup = self.random.random_tuple(10) instructions.push_args(len(tup), *tup) instructions.append('TUPLE_PACK') instructions.append(op) self.add_strings(len(tup)) elif op == 'TUPLE_SORT': tups = self.random.random_tuple_list(10, 30) for tup in tups: instructions.push_args(len(tup), *tup) instructions.append('TUPLE_PACK') instructions.push_args(len(tups)) instructions.append(op) self.add_strings(len(tups)) # Use SUB to test if integers are correctly unpacked elif op == 'SUB': a = self.random.random_int() // 2 b = self.random.random_int() // 2 instructions.push_args(0, a, b) instructions.append(op) instructions.push_args(1) instructions.append('SWAP') instructions.append(op) instructions.push_args(1) instructions.append('TUPLE_PACK') self.add_stack_items(1) elif op == 'ENCODE_FLOAT': f = self.random.random_float(8) f_bytes = struct.pack('>f', f) instructions.push_args(f_bytes) instructions.append(op) self.add_stack_items(1) elif op == 'ENCODE_DOUBLE': d = self.random.random_float(11) d_bytes = struct.pack('>d', d) instructions.push_args(d_bytes) instructions.append(op) self.add_stack_items(1) elif op == 'DECODE_FLOAT': f = self.random.random_float(8) instructions.push_args(fdb.tuple.SingleFloat(f)) instructions.append(op) self.add_strings(1) elif op == 'DECODE_DOUBLE': d = self.random.random_float(11) instructions.push_args(d) instructions.append(op) self.add_strings(1) else: assert False, 'Unknown operation: ' + op if read_performed and op not in database_reads: self.outstanding_ops.append( (self.stack_size, len(instructions) - 1)) if args.concurrency == 1 and (op in database_reads or op in database_mutations): instructions.append('WAIT_FUTURE') instructions.begin_finalization() if args.concurrency == 1: self.wait_for_reads(instructions) test_util.blocking_commit(instructions) self.add_stack_items(1) instructions.append('NEW_TRANSACTION') instructions.push_args(self.stack_subspace.key()) instructions.append('LOG_STACK') test_util.blocking_commit(instructions) return instructions
def generate(self, args, thread_number): instructions = InstructionSet() op_choices = ['NEW_TRANSACTION', 'COMMIT'] general = ['DIRECTORY_CREATE_SUBSPACE', 'DIRECTORY_CREATE_LAYER'] op_choices += general directory_mutations = ['DIRECTORY_CREATE_OR_OPEN', 'DIRECTORY_CREATE', 'DIRECTORY_MOVE', 'DIRECTORY_MOVE_TO', 'DIRECTORY_REMOVE', 'DIRECTORY_REMOVE_IF_EXISTS'] directory_reads = ['DIRECTORY_EXISTS', 'DIRECTORY_OPEN', 'DIRECTORY_LIST'] directory_db_mutations = [x + '_DATABASE' for x in directory_mutations] directory_db_reads = [x + '_DATABASE' for x in directory_reads] directory_snapshot_reads = [x + '_SNAPSHOT' for x in directory_reads] directory = [] directory += directory_mutations directory += directory_reads directory += directory_db_mutations directory += directory_db_reads if not args.no_directory_snapshot_ops: directory += directory_snapshot_reads subspace = ['DIRECTORY_PACK_KEY', 'DIRECTORY_UNPACK_KEY', 'DIRECTORY_RANGE', 'DIRECTORY_CONTAINS', 'DIRECTORY_OPEN_SUBSPACE'] instructions.append('NEW_TRANSACTION') default_path = 'default%d' % self.next_path self.next_path += 1 self.dir_list = directory_util.setup_directories(instructions, default_path, self.random) self.root = self.dir_list[0] instructions.push_args(0) instructions.append('DIRECTORY_CHANGE') # Generate some directories that we are going to create in advance. This tests that other bindings # are compatible with the Python implementation self.prepopulated_dirs = [(generate_path(min_length=1), self.generate_layer()) for i in range(5)] for path, layer in self.prepopulated_dirs: instructions.push_args(layer) instructions.push_args(*test_util.with_length(path)) instructions.append('DIRECTORY_OPEN') self.dir_list.append(self.root.add_child(path, DirectoryStateTreeNode(True, True, has_known_prefix=False, is_partition=(layer==b'partition')))) # print('%d. Selected %s, dir=%s, dir_id=%s, has_known_prefix=%s, dir_list_len=%d' \ # % (len(instructions), 'DIRECTORY_OPEN', repr(self.dir_index), self.dir_list[-1].dir_id, False, len(self.dir_list)-1)) instructions.setup_complete() for i in range(args.num_ops): if random.random() < 0.5: while True: self.dir_index = random.randrange(0, len(self.dir_list)) if not self.dir_list[self.dir_index].state.is_partition or not self.dir_list[self.dir_index].state.deleted: break instructions.push_args(self.dir_index) instructions.append('DIRECTORY_CHANGE') dir_entry = self.dir_list[self.dir_index] choices = op_choices[:] if dir_entry.state.is_directory: choices += directory if dir_entry.state.is_subspace: choices += subspace op = random.choice(choices) # print('%d. Selected %s, dir=%d, dir_id=%d, has_known_prefix=%d, dir_list_len=%d' \ # % (len(instructions), op, self.dir_index, dir_entry.dir_id, dir_entry.state.has_known_prefix, len(self.dir_list))) if op.endswith('_DATABASE') or op.endswith('_SNAPSHOT'): root_op = op[0:-9] else: root_op = op if root_op == 'NEW_TRANSACTION': instructions.append(op) elif root_op == 'COMMIT': test_util.blocking_commit(instructions) elif root_op == 'DIRECTORY_CREATE_SUBSPACE': path = generate_path() instructions.push_args(generate_prefix(require_unique=False, is_partition=True)) instructions.push_args(*test_util.with_length(path)) instructions.append(op) self.dir_list.append(DirectoryStateTreeNode(False, True, has_known_prefix=True)) elif root_op == 'DIRECTORY_CREATE_LAYER': indices = [] prefixes = [generate_prefix(require_unique=args.concurrency==1, is_partition=True) for i in range(2)] for i in range(2): instructions.push_args(prefixes[i]) instructions.push_args(*test_util.with_length(generate_path())) instructions.append('DIRECTORY_CREATE_SUBSPACE') indices.append(len(self.dir_list)) self.dir_list.append(DirectoryStateTreeNode(False, True, has_known_prefix=True)) instructions.push_args(random.choice([0, 1])) instructions.push_args(*indices) instructions.append(op) self.dir_list.append(DirectoryStateTreeNode.get_layer(prefixes[0])) elif root_op == 'DIRECTORY_CREATE_OR_OPEN': # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) path = generate_path() # Partitions that use the high-contention allocator can result in non-determinism if they fail to commit, # so we disallow them in comparison tests op_args = test_util.with_length(path) + (self.generate_layer(allow_partition=args.concurrency>1),) directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) if not op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) child_entry = dir_entry.get_descendent(path) if child_entry is None: child_entry = DirectoryStateTreeNode(True, True) child_entry.state.has_known_prefix = False self.dir_list.append(dir_entry.add_child(path, child_entry)) elif root_op == 'DIRECTORY_CREATE': layer = self.generate_layer() is_partition = layer == b'partition' prefix = generate_prefix(require_unique=is_partition and args.concurrency==1, is_partition=is_partition, min_length=0) # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) path = generate_path() op_args = test_util.with_length(path) + (layer, prefix) if prefix is None: directory_util.push_instruction_and_record_prefix( instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) else: instructions.push_args(*op_args) instructions.append(op) if not op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) child_entry = dir_entry.get_descendent(path) if child_entry is None: child_entry = DirectoryStateTreeNode(True, True, has_known_prefix=bool(prefix)) elif not bool(prefix): child_entry.state.has_known_prefix = False if is_partition: child_entry.state.is_partition = True self.dir_list.append(dir_entry.add_child(path, child_entry)) elif root_op == 'DIRECTORY_OPEN': path = generate_path() instructions.push_args(self.generate_layer()) instructions.push_args(*test_util.with_length(path)) instructions.append(op) child_entry = dir_entry.get_descendent(path) if child_entry is None: self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False)) else: self.dir_list.append(dir_entry.add_child(path, child_entry)) elif root_op == 'DIRECTORY_MOVE': old_path = generate_path() new_path = generate_path() instructions.push_args(*(test_util.with_length(old_path) + test_util.with_length(new_path))) instructions.append(op) child_entry = dir_entry.get_descendent(old_path) if child_entry is None: self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False)) else: self.dir_list.append(dir_entry.add_child(new_path, child_entry)) # Make sure that the default directory subspace still exists after moving the specified directory if dir_entry.state.is_directory and not dir_entry.state.is_subspace and old_path == ('',): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_MOVE_TO': new_path = generate_path() instructions.push_args(*test_util.with_length(new_path)) instructions.append(op) child_entry = dir_entry.get_descendent(()) if child_entry is None: self.dir_list.append(DirectoryStateTreeNode(False, False, has_known_prefix=False)) else: self.dir_list.append(dir_entry.add_child(new_path, child_entry)) # Make sure that the default directory subspace still exists after moving the current directory self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_REMOVE' or root_op == 'DIRECTORY_REMOVE_IF_EXISTS': # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) dir_entry.delete(path) # Make sure that the default directory subspace still exists after removing the specified directory if path == () or (dir_entry.state.is_directory and not dir_entry.state.is_subspace and path == ('',)): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_LIST' or root_op == 'DIRECTORY_EXISTS': path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) elif root_op == 'DIRECTORY_PACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append(op) instructions.append('DIRECTORY_STRIP_PREFIX') elif root_op == 'DIRECTORY_UNPACK_KEY' or root_op == 'DIRECTORY_CONTAINS': if not dir_entry.state.has_known_prefix or random.random() < 0.2 or root_op == 'DIRECTORY_UNPACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append('DIRECTORY_PACK_KEY') instructions.append(op) else: instructions.push_args(fdb.tuple.pack(self.random.random_tuple(5))) instructions.append(op) elif root_op == 'DIRECTORY_RANGE' or root_op == 'DIRECTORY_OPEN_SUBSPACE': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append(op) if root_op == 'DIRECTORY_OPEN_SUBSPACE': self.dir_list.append(DirectoryStateTreeNode(False, True, dir_entry.state.has_known_prefix)) else: test_util.to_front(instructions, 1) instructions.append('DIRECTORY_STRIP_PREFIX') test_util.to_front(instructions, 1) instructions.append('DIRECTORY_STRIP_PREFIX') instructions.begin_finalization() test_util.blocking_commit(instructions) instructions.append('NEW_TRANSACTION') for i, dir_entry in enumerate(self.dir_list): instructions.push_args(i) instructions.append('DIRECTORY_CHANGE') if dir_entry.state.is_directory: instructions.push_args(self.directory_log.key()) instructions.append('DIRECTORY_LOG_DIRECTORY') if dir_entry.state.has_known_prefix and dir_entry.state.is_subspace: # print('%d. Logging subspace: %d' % (i, dir_entry.dir_id)) instructions.push_args(self.subspace_log.key()) instructions.append('DIRECTORY_LOG_SUBSPACE') if (i + 1) % 100 == 0: test_util.blocking_commit(instructions) test_util.blocking_commit(instructions) instructions.push_args(self.stack_subspace.key()) instructions.append('LOG_STACK') test_util.blocking_commit(instructions) return instructions
def generate(self, args, thread_number): instructions = InstructionSet() op_choices = ['NEW_TRANSACTION', 'COMMIT'] general = ['DIRECTORY_CREATE_SUBSPACE', 'DIRECTORY_CREATE_LAYER'] op_choices += general directory_mutations = ['DIRECTORY_CREATE_OR_OPEN', 'DIRECTORY_CREATE', 'DIRECTORY_MOVE', 'DIRECTORY_MOVE_TO', 'DIRECTORY_REMOVE', 'DIRECTORY_REMOVE_IF_EXISTS'] directory_reads = ['DIRECTORY_EXISTS', 'DIRECTORY_OPEN', 'DIRECTORY_LIST'] directory_db_mutations = [x + '_DATABASE' for x in directory_mutations] directory_db_reads = [x + '_DATABASE' for x in directory_reads] directory_snapshot_reads = [x + '_SNAPSHOT' for x in directory_reads] directory = [] directory += directory_mutations directory += directory_reads directory += directory_db_mutations directory += directory_db_reads if not args.no_directory_snapshot_ops: directory += directory_snapshot_reads subspace = ['DIRECTORY_PACK_KEY', 'DIRECTORY_UNPACK_KEY', 'DIRECTORY_RANGE', 'DIRECTORY_CONTAINS', 'DIRECTORY_OPEN_SUBSPACE'] instructions.append('NEW_TRANSACTION') default_path = unicode('default%d' % self.next_path) self.next_path += 1 self.dir_list = directory_util.setup_directories(instructions, default_path, self.random) self.root = self.dir_list[0] instructions.push_args(0) instructions.append('DIRECTORY_CHANGE') # Generate some directories that we are going to create in advance. This tests that other bindings # are compatible with the Python implementation self.prepopulated_dirs = [(generate_path(min_length=1), self.generate_layer()) for i in range(5)] for path, layer in self.prepopulated_dirs: instructions.push_args(layer) instructions.push_args(*test_util.with_length(path)) instructions.append('DIRECTORY_OPEN') # print '%d. Selected %s, dir=%s, has_known_prefix=%s, dir_list_len=%d' \ # % (len(instructions), 'DIRECTORY_OPEN', repr(self.dir_index), False, len(self.dir_list)) self.dir_list.append(self.dir_list[0].add_child(path, default_path, self.root, DirListEntry(True, True, has_known_prefix=False))) instructions.setup_complete() for i in range(args.num_ops): if random.random() < 0.5: self.dir_index = random.randrange(0, len(self.dir_list)) instructions.push_args(self.dir_index) instructions.append('DIRECTORY_CHANGE') choices = op_choices[:] if self.dir_list[self.dir_index].is_directory: choices += directory if self.dir_list[self.dir_index].is_subspace: choices += subspace op = random.choice(choices) dir_entry = self.dir_list[self.dir_index] # print '%d. Selected %s, dir=%s, has_known_prefix=%s, dir_list_len=%d' \ # % (len(instructions), op, repr(self.dir_index), repr(dir_entry.has_known_prefix), len(self.dir_list)) if op.endswith('_DATABASE') or op.endswith('_SNAPSHOT'): root_op = op[0:-9] else: root_op = op if root_op == 'NEW_TRANSACTION': instructions.append(op) elif root_op == 'COMMIT': test_util.blocking_commit(instructions) elif root_op == 'DIRECTORY_CREATE_SUBSPACE': path = generate_path() instructions.push_args(generate_prefix(allow_empty=False, is_partition=True)) instructions.push_args(*test_util.with_length(path)) instructions.append(op) self.dir_list.append(DirListEntry(False, True)) elif root_op == 'DIRECTORY_CREATE_LAYER': indices = [] for i in range(2): instructions.push_args(generate_prefix(allow_empty=False, is_partition=True)) instructions.push_args(*test_util.with_length(generate_path())) instructions.append('DIRECTORY_CREATE_SUBSPACE') indices.append(len(self.dir_list)) self.dir_list.append(DirListEntry(False, True)) instructions.push_args(random.choice([0, 1])) instructions.push_args(*indices) instructions.append(op) self.dir_list.append(DirListEntry(True, False, False)) elif root_op == 'DIRECTORY_CREATE_OR_OPEN': # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) path = generate_path() op_args = test_util.with_length(path) + (self.generate_layer(),) directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) if not op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) self.dir_list.append(dir_entry.add_child(path, default_path, self.root, DirListEntry(True, True, False))) elif root_op == 'DIRECTORY_CREATE': layer = self.generate_layer() is_partition = layer == 'partition' allow_empty_prefix = random.random() < 0.8 prefix = generate_prefix(allow_empty=allow_empty_prefix, is_partition=is_partition) # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) path = generate_path() op_args = test_util.with_length(path) + (layer, prefix) if prefix is None: directory_util.push_instruction_and_record_prefix( instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) else: instructions.push_args(*op_args) instructions.append(op) if not op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) self.dir_list.append(dir_entry.add_child(path, default_path, self.root, DirListEntry(True, True, bool(prefix)))) elif root_op == 'DIRECTORY_OPEN': path = generate_path() instructions.push_args(self.generate_layer()) instructions.push_args(*test_util.with_length(path)) instructions.append(op) self.dir_list.append(dir_entry.add_child(path, default_path, self.root, DirListEntry(True, True))) elif root_op == 'DIRECTORY_MOVE': old_path = generate_path() new_path = generate_path() instructions.push_args(*(test_util.with_length(old_path) + test_util.with_length(new_path))) instructions.append(op) # This could probably be made to sometimes set has_known_prefix to true self.dir_list.append(dir_entry.add_child(new_path, default_path, self.root, DirListEntry(True, True, False))) # Make sure that the default directory subspace still exists after moving the specified directory if dir_entry.is_directory and not dir_entry.is_subspace and old_path == (u'',): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_MOVE_TO': new_path = generate_path() instructions.push_args(*test_util.with_length(new_path)) instructions.append(op) self.dir_list.append(dir_entry.root.add_child(new_path, default_path, self.root, DirListEntry(True, True, dir_entry.has_known_prefix))) # Make sure that the default directory subspace still exists after moving the current directory self.ensure_default_directory_subspace(instructions, default_path) # FIXME: There is currently a problem with removing partitions. In these generated tests, it's possible # for a removed partition to resurrect itself and insert keys into the database using its allocated # prefix. The result is non-deterministic HCA errors. elif root_op == 'DIRECTORY_REMOVE' or root_op == 'DIRECTORY_REMOVE_IF_EXISTS': # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) # Make sure that the default directory subspace still exists after removing the specified directory if path == () or (dir_entry.is_directory and not dir_entry.is_subspace and path == (u'',)): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_LIST' or root_op == 'DIRECTORY_EXISTS': path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) elif root_op == 'DIRECTORY_PACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append(op) instructions.append('DIRECTORY_STRIP_PREFIX') elif root_op == 'DIRECTORY_UNPACK_KEY' or root_op == 'DIRECTORY_CONTAINS': if not dir_entry.has_known_prefix or random.random() < 0.2 or root_op == 'DIRECTORY_UNPACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append('DIRECTORY_PACK_KEY') instructions.append(op) else: instructions.push_args(fdb.tuple.pack(self.random.random_tuple(5))) instructions.append(op) elif root_op == 'DIRECTORY_RANGE' or root_op == 'DIRECTORY_OPEN_SUBSPACE': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) instructions.append(op) if root_op == 'DIRECTORY_OPEN_SUBSPACE': self.dir_list.append(DirListEntry(False, True, dir_entry.has_known_prefix)) else: test_util.to_front(instructions, 1) instructions.append('DIRECTORY_STRIP_PREFIX') test_util.to_front(instructions, 1) instructions.append('DIRECTORY_STRIP_PREFIX') instructions.begin_finalization() test_util.blocking_commit(instructions) instructions.append('NEW_TRANSACTION') for i, dir_entry in enumerate(self.dir_list): instructions.push_args(i) instructions.append('DIRECTORY_CHANGE') if dir_entry.is_directory: instructions.push_args(self.directory_log.key()) instructions.append('DIRECTORY_LOG_DIRECTORY') if dir_entry.has_known_prefix and dir_entry.is_subspace: # print '%d. Logging subspace: %d' % (i, dir_entry.dir_id) instructions.push_args(self.subspace_log.key()) instructions.append('DIRECTORY_LOG_SUBSPACE') if (i + 1) % 100 == 0: test_util.blocking_commit(instructions) instructions.push_args(self.stack_subspace.key()) instructions.append('LOG_STACK') test_util.blocking_commit(instructions) return instructions