def new_conjecture_data(self, prefix, parameter, max_length=BUFFER_SIZE): return ConjectureData( prefix=prefix, parameter=parameter, max_length=max_length, observer=self.tree.new_observer(), )
def generate_corpus(experiment, seed, buffer_size, count): random.seed(seed) try: os.unlink(os.path.join(CORPORA, experiment + '.tar')) except FileNotFoundError: pass shutil.rmtree(os.path.join(CORPORA, experiment), ignore_errors=True) mkdirp(os.path.join(CORPORA, experiment)) experiment = EXPERIMENTS[experiment] completed = 0 while completed < count: try: data = ConjectureData( draw_bytes=lambda data, n: uniform(random, n), max_length=buffer_size, ) gen = experiment.generator(data) info = experiment.calculate_info(gen) error_pred = experiment.calculate_error_predicate(info) except StopTest: continue except Exception: continue print(info) with open(os.path.join(CORPORA, experiment.name, hashlib.sha1(data.buffer).hexdigest()[:16]), "wb") as o: o.write(data.buffer) completed += 1 subprocess.check_call(["apack", experiment.name + ".tar", experiment.name], cwd=CORPORA) shutil.rmtree(os.path.join(CORPORA, experiment.name))
def fuzz(): def try_data(data): with tempfile.TemporaryDirectory() as d: prog = os.path.join(d, 'test.c') try: gen(data, prog) except StopTest: rmf(prog) return name = hashlib.sha1(data.buffer).hexdigest()[:8] print(name) if is_interesting(prog): with open(os.path.join(raw, name), 'wb') as o: o.write(data.buffer) shutil.move(prog, os.path.join(programs, name + '.c')) else: rmf(os.path.join(raw, name)) rmf(os.path.join(programs, name + '.c')) random = Random() while True: try_data(ConjectureData( max_length=BUFFER_SIZE, draw_bytes=lambda self, n: random.getrandbits(n * 8).to_bytes(n, 'big') ))
def new_conjecture_data(self, prefix, max_length=BUFFER_SIZE, observer=None): return ConjectureData( prefix=prefix, max_length=max_length, random=self.random, observer=observer or self.tree.new_observer(), )
def generate_novel_prefix(self, random): """Generate a short random string that (after rewriting) is not a prefix of any buffer previously added to the tree. This is logically equivalent to generating the test case uniformly at random and returning the first point at which we hit unknown territory, but with an optimisation for the only common case where that would be inefficient. """ assert not self.is_exhausted initial = self.find_necessary_prefix_for_novelty() while True: def draw_bytes(data, n): i = data.index if i < len(initial): return initial[i : i + n] else: return uniform(random, n) data = ConjectureData(draw_bytes=draw_bytes, max_length=float("inf")) try: self.simulate_test_function(data) except PreviouslyUnseenBehaviour: return hbytes(data.buffer)
def generate_new_examples(self): def draw_bytes(data, n): return hbytes([255] * n) self.test_function( ConjectureData(draw_bytes=draw_bytes, max_length=self.settings.buffer_size))
def test_partial_buffer(n, rnd): data = ConjectureData( prefix=[n], random=rnd, max_length=2, ) assert data.draw_bytes(2)[0] == n
def new_buffer(self): assert not self.__tree_is_exhausted() self.last_data = ConjectureData( max_length=self.settings.buffer_size, draw_bytes=lambda data, n, distribution: self. __rewrite_for_novelty(data, distribution(self.random, n))) self.test_function(self.last_data) self.last_data.freeze()
def new_buffer(self): self.last_data = ConjectureData( max_length=self.settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(self.random, n) ) self.test_function(self.last_data) self.last_data.freeze()
def test_distribution_is_correctly_translated(inter, rnd): assert inter == sorted(inter) lower, c1, c2, upper = inter d = ConjectureData( draw_bytes=lambda data, n, distribution: distribution(rnd, n), max_length=10**6) assert d.draw(interval(lower, upper, c1, lambda r: c2)) == c2 assert d.draw(interval(lower, upper, c2, lambda r: c1)) == c1
def new_buffer(self): assert not self.__tree_is_exhausted() def draw_bytes(data, n): return self.__rewrite_for_novelty( data, self.__zero_bound(data, uniform(self.random, n)) ) self.last_data = ConjectureData( max_length=self.settings.buffer_size, draw_bytes=draw_bytes ) self.test_function(self.last_data) self.last_data.freeze()
def generate(ctx): random = Random(ctx.obj['seed']) log_data = ctx.obj['log_data'] run_tstl = ctx.obj['run_tstl'] while True: target = ConjectureData( prefix=b'', random=random, max_length=eng.BUFFER_SIZE, ) try: run_tstl(target) except StopTest: pass log_data(target) if target.status == Status.INTERESTING: break ctx.obj['log'].close()
def attempt_to_improve(self, example_index, parameter): """Part of our hill climbing implementation. Attempts to improve a given score by regenerating an example in the data based on a new parameter.""" data = self.current_data self.current_score ex = data.examples[example_index] assert ex.length > 0 prefix_size = ex.start prefix = data.buffer[:prefix_size] dummy = ConjectureData( draw_bytes=draw_bytes_with(prefix, parameter), max_length=BUFFER_SIZE ) try: self.engine.tree.simulate_test_function(dummy) # If this didn't throw an exception then we've already seen this # behaviour before and are trying something too similar to what # we already have. return False except PreviouslyUnseenBehaviour: pass attempt = self.engine.new_conjecture_data( draw_bytes_with(dummy.buffer, parameter) ) self.engine.test_function(attempt) if self.consider_new_test_data(attempt): return True ex_attempt = attempt.examples[example_index] replacement = attempt.buffer[ex_attempt.start : ex_attempt.end] return self.consider_new_buffer(prefix + replacement + data.buffer[ex.end :])
def generate_new_examples(self): if Phase.generate not in self.settings.phases: return zero_data = self.cached_test_function(hbytes( self.settings.buffer_size)) if zero_data.status == Status.OVERRUN or ( zero_data.status == Status.VALID and len(zero_data.buffer) * 2 > self.settings.buffer_size): fail_health_check( self.settings, "The smallest natural example for your test is extremely " "large. This makes it difficult for Hypothesis to generate " "good examples, especially when trying to reduce failing ones " "at the end. Consider reducing the size of your data if it is " "of a fixed size. You could also fix this by improving how " "your data shrinks (see https://hypothesis.readthedocs.io/en/" "latest/data.html#shrinking for details), or by introducing " "default values inside your strategy. e.g. could you replace " "some arguments with their defaults by using " "one_of(none(), some_complex_strategy)?", HealthCheck.large_base_example, ) if zero_data is not Overrun: # If the language starts with writes of length >= cap then there is # only one string in it: Everything after cap is forced to be zero (or # to be whatever value is written there). That means that once we've # tried the zero value, there's nothing left for us to do, so we # exit early here. for i in hrange(self.cap): if i not in zero_data.forced_indices: break else: self.exit_with(ExitReason.finished) self.health_check_state = HealthCheckState() count = 0 while not self.interesting_examples and ( count < 10 or self.health_check_state is not None): prefix = self.generate_novel_prefix() def draw_bytes(data, n): if data.index < len(prefix): result = prefix[data.index:data.index + n] if len(result) < n: result += uniform(self.random, n - len(result)) else: result = uniform(self.random, n) return self.__zero_bound(data, result) targets_found = len(self.covering_examples) last_data = ConjectureData(max_length=self.settings.buffer_size, draw_bytes=draw_bytes) self.test_function(last_data) last_data.freeze() count += 1 mutations = 0 mutator = self._new_mutator() zero_bound_queue = [] while not self.interesting_examples: if zero_bound_queue: # Whenever we generated an example and it hits a bound # which forces zero blocks into it, this creates a weird # distortion effect by making certain parts of the data # stream (especially ones to the right) much more likely # to be zero. We fix this by redistributing the generated # data by shuffling it randomly. This results in the # zero data being spread evenly throughout the buffer. # Hopefully the shrinking this causes will cause us to # naturally fail to hit the bound. # If it doesn't then we will queue the new version up again # (now with more zeros) and try again. overdrawn = zero_bound_queue.pop() buffer = bytearray(overdrawn.buffer) # These will have values written to them that are different # from what's in them anyway, so the value there doesn't # really "count" for distributional purposes, and if we # leave them in then they can cause the fraction of non # zero bytes to increase on redraw instead of decrease. for i in overdrawn.forced_indices: buffer[i] = 0 self.random.shuffle(buffer) buffer = hbytes(buffer) def draw_bytes(data, n): result = buffer[data.index:data.index + n] if len(result) < n: result += hbytes(n - len(result)) return self.__rewrite(data, result) data = ConjectureData(draw_bytes=draw_bytes, max_length=self.settings.buffer_size) self.test_function(data) data.freeze() else: origin = self.target_selector.select() mutations += 1 targets_found = len(self.covering_examples) data = ConjectureData(draw_bytes=mutator(origin), max_length=self.settings.buffer_size) self.test_function(data) data.freeze() if (data.status > origin.status or len(self.covering_examples) > targets_found): mutations = 0 elif data.status < origin.status or mutations >= 10: # Cap the variations of a single example and move on to # an entirely fresh start. Ten is an entirely arbitrary # constant, but it's been working well for years. mutations = 0 mutator = self._new_mutator() if getattr(data, "hit_zero_bound", False): zero_bound_queue.append(data) mutations += 1
def _run(self): self.last_data = None mutations = 0 start_time = time.time() self.reuse_existing_examples() if ( Phase.generate in self.settings.phases and not self.__tree_is_exhausted() ): if ( self.last_data is None or self.last_data.status < Status.INTERESTING ): self.new_buffer() mutator = self._new_mutator() zero_bound_queue = [] while ( self.last_data.status != Status.INTERESTING and not self.__tree_is_exhausted() ): if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max( self.settings.max_iterations, self.settings.max_examples ): self.exit_reason = ExitReason.max_iterations return if ( self.settings.timeout > 0 and time.time() >= start_time + self.settings.timeout ): self.exit_reason = ExitReason.timeout return if zero_bound_queue: # Whenever we generated an example and it hits a bound # which forces zero blocks into it, this creates a weird # distortion effect by making certain parts of the data # stream (especially ones to the right) much more likely # to be zero. We fix this by redistributing the generated # data by shuffling it randomly. This results in the # zero data being spread evenly throughout the buffer. # Hopefully the shrinking this causes will cause us to # naturally fail to hit the bound. # If it doesn't then we will queue the new version up again # (now with more zeros) and try again. overdrawn = zero_bound_queue.pop() buffer = bytearray(overdrawn.buffer) self.random.shuffle(buffer) buffer = hbytes(buffer) if buffer == overdrawn.buffer: continue def draw_bytes(data, n, distribution): result = buffer[data.index:data.index + n] if len(result) < n: result += hbytes(n - len(result)) return self.__rewrite(data, result) data = ConjectureData( draw_bytes=draw_bytes, max_length=self.settings.buffer_size, ) self.test_function(data) data.freeze() elif mutations >= self.settings.max_mutations: mutations = 0 data = self.new_buffer() mutator = self._new_mutator() else: data = ConjectureData( draw_bytes=mutator, max_length=self.settings.buffer_size ) self.test_function(data) data.freeze() prev_data = self.last_data if self.consider_new_test_data(data): self.last_data = data if data.status > prev_data.status: mutations = 0 else: mutator = self._new_mutator() if getattr(data, 'hit_zero_bound', False): zero_bound_queue.append(data) mutations += 1 if self.__tree_is_exhausted(): self.exit_reason = ExitReason.finished return data = self.last_data if data is None: self.exit_reason = ExitReason.finished return assert isinstance(data.output, text_type) if self.settings.max_shrinks <= 0: self.exit_reason = ExitReason.max_shrinks return if Phase.shrink not in self.settings.phases: self.exit_reason = ExitReason.finished return data = ConjectureData.for_buffer(self.last_data.buffer) self.test_function(data) if data.status != Status.INTERESTING: self.exit_reason = ExitReason.flaky return self.shrink()
def new_conjecture_data(self, draw_bytes): return ConjectureData( draw_bytes=draw_bytes, max_length=self.settings.buffer_size, observer=self.tree.new_observer(), )
if __name__ == '__main__': try: os.makedirs(EXAMPLES) except FileExistsError: pass rnd = Random() found = 0 i = 0 while found < LIMIT: i += 1 print("Iter", i) data = ConjectureData(max_length=2**29, draw_bytes=lambda self, n: rnd.getrandbits(8 * n) .to_bytes(n, byteorder='big')) sourcename = "example.c" gen(data, sourcename) print("Generated") if results_for_o1_and_o2_differ(sourcename): found += 1 print("Found", found) result = bytes(data.buffer) name = hashlib.sha1(result).hexdigest()[:16] with open(os.path.join(EXAMPLES, name), 'wb') as outfile: outfile.write(result)
def perform_health_checks(random, settings, test_runner, search_strategy): if not settings.perform_health_check: return if not Settings.default.perform_health_check: return health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = ConjectureData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while (count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20): try: data = ConjectureData(max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n)) with Settings(settings, verbosity=Verbosity.quiet): test_runner( data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: escalate_hypothesis_internal_error() if (HealthCheck.exception_in_generation in settings.suppress_health_check): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( settings, 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or (not count and overruns > 0): fail_health_check( settings, ('Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % (overruns, count), HealthCheck.data_too_large) if filtered_draws >= 50 or (not count and filtered_draws > 0): fail_health_check( settings, ('It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % (filtered_draws, count), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check( settings, ('Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).') % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, )
def _run(self): self.last_data = None mutations = 0 start_time = time.time() if ( self.settings.database is not None and self.database_key is not None ): corpus = sorted( self.settings.database.fetch(self.database_key), key=lambda d: (len(d), d) ) for existing in corpus: if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max( self.settings.max_iterations, self.settings.max_examples ): self.exit_reason = ExitReason.max_iterations return data = ConjectureData.for_buffer(existing) self.test_function(data) data.freeze() self.last_data = data if data.status < Status.VALID: self.settings.database.delete( self.database_key, existing) elif data.status == Status.VALID: # Incremental garbage collection! we store a lot of # examples in the DB as we shrink: Those that stay # interesting get kept, those that become invalid get # dropped, but those that are merely valid gradually go # away over time. if self.random.randint(0, 2) == 0: self.settings.database.delete( self.database_key, existing) else: assert data.status == Status.INTERESTING self.last_data = data break if Phase.generate in self.settings.phases: if ( self.last_data is None or self.last_data.status < Status.INTERESTING ): self.new_buffer() mutator = self._new_mutator() while self.last_data.status != Status.INTERESTING: if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max( self.settings.max_iterations, self.settings.max_examples ): self.exit_reason = ExitReason.max_iterations return if ( self.settings.timeout > 0 and time.time() >= start_time + self.settings.timeout ): self.exit_reason = ExitReason.timeout return if mutations >= self.settings.max_mutations: mutations = 0 self.new_buffer() mutator = self._new_mutator() else: data = ConjectureData( draw_bytes=mutator, max_length=self.settings.buffer_size ) self.test_function(data) data.freeze() prev_data = self.last_data if self.consider_new_test_data(data): self.last_data = data if data.status > prev_data.status: mutations = 0 else: mutator = self._new_mutator() mutations += 1 data = self.last_data if data is None: self.exit_reason = ExitReason.finished return assert isinstance(data.output, text_type) if self.settings.max_shrinks <= 0: self.exit_reason = ExitReason.max_shrinks return if Phase.shrink not in self.settings.phases: self.exit_reason = ExitReason.finished return if not self.last_data.buffer: self.exit_reason = ExitReason.finished return data = ConjectureData.for_buffer(self.last_data.buffer) self.test_function(data) if data.status != Status.INTERESTING: self.exit_reason = ExitReason.flaky return change_counter = -1 while self.changed > change_counter: change_counter = self.changed self.debug('Random interval deletes') failed_deletes = 0 while self.last_data.intervals and failed_deletes < 10: if self.random.randint(0, 1): u, v = self.random.choice(self.last_data.intervals) else: n = len(self.last_data.buffer) - 1 u, v = sorted(( self.random.choice(self.last_data.intervals) )) if ( v < len(self.last_data.buffer) ) and self.incorporate_new_buffer( self.last_data.buffer[:u] + self.last_data.buffer[v:] ): failed_deletes = 0 else: failed_deletes += 1 self.debug('Structured interval deletes') i = 0 while i < len(self.last_data.intervals): u, v = self.last_data.intervals[i] if not self.incorporate_new_buffer( self.last_data.buffer[:u] + self.last_data.buffer[v:] ): i += 1 if change_counter != self.changed: self.debug('Restarting') continue self.debug('Lexicographical minimization of whole buffer') minimize( self.last_data.buffer, self.incorporate_new_buffer, cautious=True ) if change_counter != self.changed: self.debug('Restarting') continue self.debug('Replacing blocks with simpler blocks') i = 0 while i < len(self.last_data.blocks): u, v = self.last_data.blocks[i] buf = self.last_data.buffer block = buf[u:v] n = v - u all_blocks = sorted(set([bytes(n)] + [ buf[a:a + n] for a in self.last_data.block_starts[n] ])) better_blocks = all_blocks[:all_blocks.index(block)] for b in better_blocks: if self.incorporate_new_buffer( buf[:u] + b + buf[v:] ): break i += 1 self.debug('Simultaneous shrinking of duplicated blocks') block_counter = -1 while block_counter < self.changed: block_counter = self.changed blocks = [ k for k, count in Counter( self.last_data.buffer[u:v] for u, v in self.last_data.blocks).items() if count > 1 ] for block in blocks: parts = [ self.last_data.buffer[r:s] for r, s in self.last_data.blocks ] def replace(b): return b''.join( bytes(b if c == block else c) for c in parts ) minimize( block, lambda b: self.incorporate_new_buffer(replace(b)), self.random ) self.debug('Shrinking of individual blocks') i = 0 while i < len(self.last_data.blocks): u, v = self.last_data.blocks[i] minimize( self.last_data.buffer[u:v], lambda b: self.incorporate_new_buffer( self.last_data.buffer[:u] + b + self.last_data.buffer[v:], ), self.random ) i += 1 self.debug('Replacing intervals with simpler intervals') interval_counter = -1 while interval_counter != self.changed: interval_counter = self.changed i = 0 alternatives = None while i < len(self.last_data.intervals): if alternatives is None: alternatives = sorted(set( self.last_data.buffer[u:v] for u, v in self.last_data.intervals), key=len) u, v = self.last_data.intervals[i] for a in alternatives: buf = self.last_data.buffer if ( len(a) < v - u or (len(a) == (v - u) and a < buf[u:v]) ): if self.incorporate_new_buffer( buf[:u] + a + buf[v:] ): alternatives = None break i += 1 if change_counter != self.changed: self.debug('Restarting') continue self.debug('Shuffling suffixes while shrinking %r' % ( self.last_data.bind_points, )) b = 0 while b < len(self.last_data.bind_points): cutoff = sorted(self.last_data.bind_points)[b] def test_value(prefix): for t in hrange(5): alphabet = {} for i, j in self.last_data.blocks[b:]: alphabet.setdefault(j - i, []).append((i, j)) if t > 0: for v in alphabet.values(): self.random.shuffle(v) buf = bytearray(prefix) for i, j in self.last_data.blocks[b:]: u, v = alphabet[j - i].pop() buf.extend(self.last_data.buffer[u:v]) if self.incorporate_new_buffer(hbytes(buf)): return True return False minimize( self.last_data.buffer[:cutoff], test_value, cautious=True ) b += 1 self.exit_reason = ExitReason.finished
def wrapped_test(*arguments, **kwargs): settings = wrapped_test._hypothesis_internal_use_settings if wrapped_test._hypothesis_internal_use_seed is not None: random = Random( wrapped_test._hypothesis_internal_use_seed) elif settings.derandomize: random = Random(function_digest(test)) else: random = new_random() import hypothesis.strategies as sd selfy = None arguments, kwargs = convert_positional_arguments( wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). if argspec.args: selfy = kwargs.get(argspec.args[0]) elif arguments: selfy = arguments[0] test_runner = new_style_executor(selfy) for example in reversed(getattr( wrapped_test, 'hypothesis_explicit_examples', () )): if example.args: if len(example.args) > len(original_argspec.args): raise InvalidArgument( 'example has too many arguments for test. ' 'Expected at most %d but got %d' % ( len(original_argspec.args), len(example.args))) example_kwargs = dict(zip( original_argspec.args[-len(example.args):], example.args )) else: example_kwargs = example.kwargs if Phase.explicit not in settings.phases: continue example_kwargs.update(kwargs) # Note: Test may mutate arguments and we can't rerun explicit # examples, so we have to calculate the failure message at this # point rather than than later. message_on_failure = 'Falsifying example: %s(%s)' % ( test.__name__, arg_string(test, arguments, example_kwargs) ) try: with BuildContext(None) as b: test_runner( None, lambda data: test(*arguments, **example_kwargs) ) except BaseException: traceback.print_exc() report(message_on_failure) for n in b.notes: report(n) raise if settings.max_examples <= 0: return arguments = tuple(arguments) given_specifier = sd.tuples( sd.just(arguments), sd.fixed_dictionaries(generator_kwargs).map( lambda args: dict(args, **kwargs) ) ) def fail_health_check(message, label): if label in settings.suppress_health_check: return message += ( '\nSee https://hypothesis.readthedocs.io/en/latest/health' 'checks.html for more information about this. ' ) message += ( 'If you want to disable just this health check, add %s ' 'to the suppress_health_check settings for this test.' ) % (label,) raise FailedHealthCheck(message) search_strategy = given_specifier if selfy is not None: search_strategy = WithRunner(search_strategy, selfy) search_strategy.validate() perform_health_check = settings.perform_health_check perform_health_check &= Settings.default.perform_health_check from hypothesis.internal.conjecture.data import ConjectureData, \ Status, StopTest if not ( Phase.reuse in settings.phases or Phase.generate in settings.phases ): return if perform_health_check: health_check_random = Random(random.getrandbits(128)) # We "pre warm" the health check with one draw to give it some # time to calculate any cached data. This prevents the case # where the first draw of the health check takes ages because # of loading unicode data the first time. data = ConjectureData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): try: test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) except BaseException: pass count = 0 overruns = 0 filtered_draws = 0 start = time.time() while ( count < 10 and time.time() < start + 1 and filtered_draws < 50 and overruns < 20 ): try: data = ConjectureData( max_length=settings.buffer_size, draw_bytes=lambda data, n, distribution: distribution(health_check_random, n) ) with Settings(settings, verbosity=Verbosity.quiet): test_runner(data, reify_and_execute( search_strategy, lambda *args, **kwargs: None, )) count += 1 except UnsatisfiedAssumption: filtered_draws += 1 except StopTest: if data.status == Status.INVALID: filtered_draws += 1 else: assert data.status == Status.OVERRUN overruns += 1 except InvalidArgument: raise except Exception: if ( HealthCheck.exception_in_generation in settings.suppress_health_check ): raise report(traceback.format_exc()) if test_runner is default_new_style_executor: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' "an error in a function you've passed to " 'it to construct your data.', HealthCheck.exception_in_generation, ) else: fail_health_check( 'An exception occurred during data ' 'generation in initial health check. ' 'This indicates a bug in the strategy. ' 'This could either be a Hypothesis bug or ' 'an error in a function you\'ve passed to ' 'it to construct your data. Additionally, ' 'you have a custom executor, which means ' 'that this could be your executor failing ' 'to handle a function which returns None. ', HealthCheck.exception_in_generation, ) if overruns >= 20 or ( not count and overruns > 0 ): fail_health_check(( 'Examples routinely exceeded the max allowable size. ' '(%d examples overran while generating %d valid ones)' '. Generating examples this large will usually lead to' ' bad results. You should try setting average_size or ' 'max_size parameters on your collections and turning ' 'max_leaves down on recursive() calls.') % ( overruns, count ), HealthCheck.data_too_large) if filtered_draws >= 50 or ( not count and filtered_draws > 0 ): fail_health_check(( 'It looks like your strategy is filtering out a lot ' 'of data. Health check found %d filtered examples but ' 'only %d good ones. This will make your tests much ' 'slower, and also will probably distort the data ' 'generation quite a lot. You should adapt your ' 'strategy to filter less. This can also be caused by ' 'a low max_leaves parameter in recursive() calls') % ( filtered_draws, count ), HealthCheck.filter_too_much) runtime = time.time() - start if runtime > 1.0 or count < 10: fail_health_check(( 'Data generation is extremely slow: Only produced ' '%d valid examples in %.2f seconds (%d invalid ones ' 'and %d exceeded maximum size). Try decreasing ' "size of the data you're generating (with e.g." 'average_size or max_leaves parameters).' ) % (count, runtime, filtered_draws, overruns), HealthCheck.too_slow, ) last_exception = [None] repr_for_last_exception = [None] def evaluate_test_data(data): try: result = test_runner(data, reify_and_execute( search_strategy, test, )) if result is not None and settings.perform_health_check: fail_health_check(( 'Tests run under @given should return None, but ' '%s returned %r instead.' ) % (test.__name__, result), HealthCheck.return_value) return False except UnsatisfiedAssumption: data.mark_invalid() except ( HypothesisDeprecationWarning, FailedHealthCheck, StopTest, ): raise except Exception: last_exception[0] = traceback.format_exc() verbose_report(last_exception[0]) data.mark_interesting() from hypothesis.internal.conjecture.engine import ConjectureRunner falsifying_example = None database_key = str_to_bytes(fully_qualified_name(test)) start_time = time.time() runner = ConjectureRunner( evaluate_test_data, settings=settings, random=random, database_key=database_key, ) runner.run() note_engine_for_statistics(runner) run_time = time.time() - start_time timed_out = ( settings.timeout > 0 and run_time >= settings.timeout ) if runner.last_data is None: return if runner.last_data.status == Status.INTERESTING: falsifying_example = runner.last_data.buffer if settings.database is not None: settings.database.save( database_key, falsifying_example ) else: if runner.valid_examples < min( settings.min_satisfying_examples, settings.max_examples, ): if timed_out: raise Timeout(( 'Ran out of time before finding a satisfying ' 'example for ' '%s. Only found %d examples in ' + '%.2fs.' ) % ( get_pretty_function_description(test), runner.valid_examples, run_time )) else: raise Unsatisfiable(( 'Unable to satisfy assumptions of hypothesis ' '%s. Only %d examples considered ' 'satisfied assumptions' ) % ( get_pretty_function_description(test), runner.valid_examples,)) return assert last_exception[0] is not None try: with settings: test_runner( ConjectureData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test, print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): report(traceback.format_exc()) raise Flaky( 'Unreliable assumption: An example which satisfied ' 'assumptions on the first run now fails it.' ) report( 'Failed to reproduce exception. Expected: \n' + last_exception[0], ) filter_message = ( 'Unreliable test data: Failed to reproduce a failure ' 'and then when it came to recreating the example in ' 'order to print the test data with a flaky result ' 'the example was filtered out (by e.g. a ' 'call to filter in your strategy) when we didn\'t ' 'expect it to be.' ) try: test_runner( ConjectureData.for_buffer(falsifying_example), reify_and_execute( search_strategy, test_is_flaky(test, repr_for_last_exception[0]), print_example=True, is_final=True )) except (UnsatisfiedAssumption, StopTest): raise Flaky(filter_message)
def new_conjecture_data(self, draw_bytes): return ConjectureData( draw_bytes=draw_bytes, max_length=BUFFER_SIZE, observer=self.tree.new_observer(), )
def _run(self): self.last_data = None mutations = 0 start_time = time.time() if ( self.settings.database is not None and self.database_key is not None ): corpus = sorted( self.settings.database.fetch(self.database_key), key=lambda d: (len(d), d) ) for existing in corpus: if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max( self.settings.max_iterations, self.settings.max_examples ): self.exit_reason = ExitReason.max_iterations return data = ConjectureData.for_buffer(existing) self.test_function(data) data.freeze() self.last_data = data self.consider_new_test_data(data) if data.status < Status.VALID: self.settings.database.delete( self.database_key, existing) elif data.status == Status.VALID: # Incremental garbage collection! we store a lot of # examples in the DB as we shrink: Those that stay # interesting get kept, those that become invalid get # dropped, but those that are merely valid gradually go # away over time. if self.random.randint(0, 2) == 0: self.settings.database.delete( self.database_key, existing) else: assert data.status == Status.INTERESTING self.last_data = data break if ( Phase.generate in self.settings.phases and not self.__tree_is_exhausted() ): if ( self.last_data is None or self.last_data.status < Status.INTERESTING ): self.new_buffer() mutator = self._new_mutator() while ( self.last_data.status != Status.INTERESTING and not self.__tree_is_exhausted() ): if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max( self.settings.max_iterations, self.settings.max_examples ): self.exit_reason = ExitReason.max_iterations return if ( self.settings.timeout > 0 and time.time() >= start_time + self.settings.timeout ): self.exit_reason = ExitReason.timeout return if mutations >= self.settings.max_mutations: mutations = 0 self.new_buffer() mutator = self._new_mutator() else: data = ConjectureData( draw_bytes=mutator, max_length=self.settings.buffer_size ) self.test_function(data) data.freeze() prev_data = self.last_data if self.consider_new_test_data(data): self.last_data = data if data.status > prev_data.status: mutations = 0 else: mutator = self._new_mutator() mutations += 1 if self.__tree_is_exhausted(): self.exit_reason = ExitReason.finished return data = self.last_data if data is None: self.exit_reason = ExitReason.finished return assert isinstance(data.output, text_type) if self.settings.max_shrinks <= 0: self.exit_reason = ExitReason.max_shrinks return if Phase.shrink not in self.settings.phases: self.exit_reason = ExitReason.finished return data = ConjectureData.for_buffer(self.last_data.buffer) self.test_function(data) if data.status != Status.INTERESTING: self.exit_reason = ExitReason.flaky return self.shrink()
def _run(self): self.last_data = None mutations = 0 start_time = time.time() if (self.settings.database is not None and self.database_key is not None): corpus = sorted(self.settings.database.fetch(self.database_key), key=lambda d: (len(d), d)) for existing in corpus: if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max(self.settings.max_iterations, self.settings.max_examples): self.exit_reason = ExitReason.max_iterations return data = ConjectureData.for_buffer(existing) self.test_function(data) data.freeze() self.last_data = data self.consider_new_test_data(data) if data.status < Status.VALID: self.settings.database.delete(self.database_key, existing) elif data.status == Status.VALID: # Incremental garbage collection! we store a lot of # examples in the DB as we shrink: Those that stay # interesting get kept, those that become invalid get # dropped, but those that are merely valid gradually go # away over time. if self.random.randint(0, 2) == 0: self.settings.database.delete(self.database_key, existing) else: assert data.status == Status.INTERESTING self.last_data = data break if (Phase.generate in self.settings.phases and not self.__tree_is_exhausted()): if (self.last_data is None or self.last_data.status < Status.INTERESTING): self.new_buffer() mutator = self._new_mutator() while (self.last_data.status != Status.INTERESTING and not self.__tree_is_exhausted()): if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max(self.settings.max_iterations, self.settings.max_examples): self.exit_reason = ExitReason.max_iterations return if (self.settings.timeout > 0 and time.time() >= start_time + self.settings.timeout): self.exit_reason = ExitReason.timeout return if mutations >= self.settings.max_mutations: mutations = 0 self.new_buffer() mutator = self._new_mutator() else: data = ConjectureData(draw_bytes=mutator, max_length=self.settings.buffer_size) self.test_function(data) data.freeze() prev_data = self.last_data if self.consider_new_test_data(data): self.last_data = data if data.status > prev_data.status: mutations = 0 else: mutator = self._new_mutator() mutations += 1 if self.__tree_is_exhausted(): self.exit_reason = ExitReason.finished return data = self.last_data if data is None: self.exit_reason = ExitReason.finished return assert isinstance(data.output, text_type) if self.settings.max_shrinks <= 0: self.exit_reason = ExitReason.max_shrinks return if Phase.shrink not in self.settings.phases: self.exit_reason = ExitReason.finished return data = ConjectureData.for_buffer(self.last_data.buffer) self.test_function(data) if data.status != Status.INTERESTING: self.exit_reason = ExitReason.flaky return change_counter = -1 while self.changed > change_counter: change_counter = self.changed self.debug('Structured interval deletes') k = len(self.last_data.intervals) // 2 while k > 0: i = 0 while i + k <= len(self.last_data.intervals): bitmask = [True] * len(self.last_data.buffer) for u, v in self.last_data.intervals[i:i + k]: for t in range(u, v): bitmask[t] = False u, v = self.last_data.intervals[i] if not self.incorporate_new_buffer( hbytes( b for b, v in zip(self.last_data.buffer, bitmask) if v)): i += k k //= 2 if change_counter != self.changed: self.debug('Restarting') continue self.debug('Bulk replacing blocks with simpler blocks') i = 0 while i < len(self.last_data.blocks): u, v = self.last_data.blocks[i] buf = self.last_data.buffer block = buf[u:v] n = v - u buffer = bytearray() for r, s in self.last_data.blocks: if s - r == n and self.last_data.buffer[r:s] > block: buffer.extend(block) else: buffer.extend(self.last_data.buffer[r:s]) self.incorporate_new_buffer(hbytes(buffer)) i += 1 self.debug('Replacing individual blocks with simpler blocks') i = 0 while i < len(self.last_data.blocks): u, v = self.last_data.blocks[i] buf = self.last_data.buffer block = buf[u:v] n = v - u all_blocks = sorted( set([hbytes(n)] + [buf[a:a + n] for a in self.last_data.block_starts[n]])) better_blocks = all_blocks[:all_blocks.index(block)] for b in better_blocks: if self.incorporate_new_buffer(buf[:u] + b + buf[v:]): break i += 1 self.debug('Simultaneous shrinking of duplicated blocks') block_counter = -1 while block_counter < self.changed: block_counter = self.changed blocks = [ k for k, count in Counter( self.last_data.buffer[u:v] for u, v in self.last_data.blocks).items() if count > 1 ] for block in blocks: parts = [ self.last_data.buffer[r:s] for r, s in self.last_data.blocks ] def replace(b): return hbytes( EMPTY_BYTES.join( hbytes(b if c == block else c) for c in parts)) minimize(block, lambda b: self.incorporate_new_buffer(replace(b)), self.random) if change_counter != self.changed: self.debug('Restarting') continue self.debug('Lexicographical minimization of whole buffer') minimize(self.last_data.buffer, self.incorporate_new_buffer, cautious=True) self.debug('Shrinking of individual blocks') i = 0 while i < len(self.last_data.blocks): u, v = self.last_data.blocks[i] minimize( self.last_data.buffer[u:v], lambda b: self.incorporate_new_buffer( self.last_data.buffer[:u] + b + self.last_data.buffer[ v:], ), self.random) i += 1 if change_counter != self.changed: self.debug('Restarting') continue self.debug('Reordering blocks') block_lengths = sorted(self.last_data.block_starts, reverse=True) for n in block_lengths: i = 1 while i < len(self.last_data.block_starts.get(n, ())): j = i while j > 0: buf = self.last_data.buffer blocks = self.last_data.block_starts[n] a_start = blocks[j - 1] b_start = blocks[j] a = buf[a_start:a_start + n] b = buf[b_start:b_start + n] if a <= b: break swapped = (buf[:a_start] + b + buf[a_start + n:b_start] + a + buf[b_start + n:]) assert len(swapped) == len(buf) assert swapped < buf if self.incorporate_new_buffer(swapped): j -= 1 else: break i += 1 if change_counter != self.changed: self.debug('Restarting') continue self.debug('Shuffling suffixes while shrinking %r' % (self.last_data.bind_points, )) b = 0 while b < len(self.last_data.bind_points): cutoff = sorted(self.last_data.bind_points)[b] def test_value(prefix): for t in hrange(5): alphabet = {} for i, j in self.last_data.blocks[b:]: alphabet.setdefault(j - i, []).append((i, j)) if t > 0: for v in alphabet.values(): self.random.shuffle(v) buf = bytearray(prefix) for i, j in self.last_data.blocks[b:]: u, v = alphabet[j - i].pop() buf.extend(self.last_data.buffer[u:v]) if self.incorporate_new_buffer(hbytes(buf)): return True return False minimize(self.last_data.buffer[:cutoff], test_value, cautious=True) b += 1 self.exit_reason = ExitReason.finished
def generate_new_examples(self): if Phase.generate not in self.settings.phases: return zero_data = ConjectureData(max_length=self.settings.buffer_size, draw_bytes=lambda data, n: self. __rewrite_for_novelty(data, hbytes(n))) self.test_function(zero_data) count = 0 while count < 10 and not self.interesting_examples: def draw_bytes(data, n): return self.__rewrite_for_novelty( data, self.__zero_bound(data, uniform(self.random, n))) targets_found = len(self.covering_examples) self.last_data = ConjectureData( max_length=self.settings.buffer_size, draw_bytes=draw_bytes) self.test_function(self.last_data) self.last_data.freeze() if len(self.covering_examples) > targets_found: count = 0 else: count += 1 mutations = 0 mutator = self._new_mutator() zero_bound_queue = [] while not self.interesting_examples: if zero_bound_queue: # Whenever we generated an example and it hits a bound # which forces zero blocks into it, this creates a weird # distortion effect by making certain parts of the data # stream (especially ones to the right) much more likely # to be zero. We fix this by redistributing the generated # data by shuffling it randomly. This results in the # zero data being spread evenly throughout the buffer. # Hopefully the shrinking this causes will cause us to # naturally fail to hit the bound. # If it doesn't then we will queue the new version up again # (now with more zeros) and try again. overdrawn = zero_bound_queue.pop() buffer = bytearray(overdrawn.buffer) # These will have values written to them that are different # from what's in them anyway, so the value there doesn't # really "count" for distributional purposes, and if we # leave them in then they can cause the fraction of non # zero bytes to increase on redraw instead of decrease. for i in overdrawn.forced_indices: buffer[i] = 0 self.random.shuffle(buffer) buffer = hbytes(buffer) def draw_bytes(data, n): result = buffer[data.index:data.index + n] if len(result) < n: result += hbytes(n - len(result)) return self.__rewrite(data, result) data = ConjectureData( draw_bytes=draw_bytes, max_length=self.settings.buffer_size, ) self.test_function(data) data.freeze() else: target, last_data = self.target_selector.select() mutations += 1 targets_found = len(self.covering_examples) prev_data = self.last_data data = ConjectureData(draw_bytes=mutator, max_length=self.settings.buffer_size) self.test_function(data) data.freeze() if (data.status > prev_data.status or len(self.covering_examples) > targets_found): mutations = 0 elif (data.status < prev_data.status or not self.target_selector.has_tag(target, data) or mutations >= self.settings.max_mutations): mutations = 0 mutator = self._new_mutator() if getattr(data, 'hit_zero_bound', False): zero_bound_queue.append(data) mutations += 1
def _run(self): self.last_data = None mutations = 0 start_time = time.time() self.reuse_existing_examples() if ( Phase.generate in self.settings.phases and not self.__tree_is_exhausted() ): if ( self.last_data is None or self.last_data.status < Status.INTERESTING ): self.new_buffer() mutator = self._new_mutator() zero_bound_queue = [] while ( self.last_data.status != Status.INTERESTING and not self.__tree_is_exhausted() ): if self.valid_examples >= self.settings.max_examples: self.exit_reason = ExitReason.max_examples return if self.call_count >= max( self.settings.max_iterations, self.settings.max_examples ): self.exit_reason = ExitReason.max_iterations return if ( self.settings.timeout > 0 and time.time() >= start_time + self.settings.timeout ): self.exit_reason = ExitReason.timeout return if zero_bound_queue: # Whenever we generated an example and it hits a bound # which forces zero blocks into it, this creates a weird # distortion effect by making certain parts of the data # stream (especially ones to the right) much more likely # to be zero. We fix this by redistributing the generated # data by shuffling it randomly. This results in the # zero data being spread evenly throughout the buffer. # Hopefully the shrinking this causes will cause us to # naturally fail to hit the bound. # If it doesn't then we will queue the new version up again # (now with more zeros) and try again. overdrawn = zero_bound_queue.pop() buffer = bytearray(overdrawn.buffer) # These will have values written to them that are different # from what's in them anyway, so the value there doesn't # really "count" for distributional purposes, and if we # leave them in then they can cause the fraction of non # zero bytes to increase on redraw instead of decrease. for i in overdrawn.forced_indices: buffer[i] = 0 self.random.shuffle(buffer) buffer = hbytes(buffer) if buffer == overdrawn.buffer: continue def draw_bytes(data, n): result = buffer[data.index:data.index + n] if len(result) < n: result += hbytes(n - len(result)) return self.__rewrite(data, result) data = ConjectureData( draw_bytes=draw_bytes, max_length=self.settings.buffer_size, ) self.test_function(data) data.freeze() elif mutations >= self.settings.max_mutations: mutations = 0 data = self.new_buffer() mutator = self._new_mutator() else: data = ConjectureData( draw_bytes=mutator, max_length=self.settings.buffer_size ) self.test_function(data) data.freeze() prev_data = self.last_data if self.consider_new_test_data(data): self.last_data = data if data.status > prev_data.status: mutations = 0 else: mutator = self._new_mutator() if getattr(data, 'hit_zero_bound', False): zero_bound_queue.append(data) mutations += 1 if self.__tree_is_exhausted(): self.exit_reason = ExitReason.finished return data = self.last_data if data is None: self.exit_reason = ExitReason.finished return assert isinstance(data.output, text_type) if Phase.shrink not in self.settings.phases: self.exit_reason = ExitReason.finished return data = ConjectureData.for_buffer(self.last_data.buffer) self.test_function(data) if data.status != Status.INTERESTING: self.exit_reason = ExitReason.flaky return while len(self.shrunk_examples) < len(self.interesting_examples): target, d = min([ (k, v) for k, v in self.interesting_examples.items() if k not in self.shrunk_examples], key=lambda kv: (sort_key(kv[1].buffer), sort_key(repr(kv[0]))), ) self.debug('Shrinking %r' % (target,)) self.last_data = d assert self.last_data.interesting_origin == target self.shrink() self.shrunk_examples.add(target)