def new_conjecture_data(self, prefix, parameter, max_length=BUFFER_SIZE):
return ConjectureData(
prefix=prefix,
parameter=parameter,
max_length=max_length,
observer=self.tree.new_observer(),
)
def generate_corpus(experiment, seed, buffer_size, count):
random.seed(seed)
try:
os.unlink(os.path.join(CORPORA, experiment + '.tar'))
except FileNotFoundError:
pass
shutil.rmtree(os.path.join(CORPORA, experiment), ignore_errors=True)
mkdirp(os.path.join(CORPORA, experiment))
experiment = EXPERIMENTS[experiment]
completed = 0
while completed < count:
try:
data = ConjectureData(
draw_bytes=lambda data, n: uniform(random, n),
max_length=buffer_size,
)
gen = experiment.generator(data)
info = experiment.calculate_info(gen)
error_pred = experiment.calculate_error_predicate(info)
except StopTest:
continue
except Exception:
continue
print(info)
with open(os.path.join(CORPORA, experiment.name, hashlib.sha1(data.buffer).hexdigest()[:16]), "wb") as o:
o.write(data.buffer)
completed += 1
subprocess.check_call(["apack", experiment.name + ".tar", experiment.name], cwd=CORPORA)
shutil.rmtree(os.path.join(CORPORA, experiment.name))
def fuzz():
def try_data(data):
with tempfile.TemporaryDirectory() as d:
prog = os.path.join(d, 'test.c')
try:
gen(data, prog)
except StopTest:
rmf(prog)
return
name = hashlib.sha1(data.buffer).hexdigest()[:8]
print(name)
if is_interesting(prog):
with open(os.path.join(raw, name), 'wb') as o:
o.write(data.buffer)
shutil.move(prog, os.path.join(programs, name + '.c'))
else:
rmf(os.path.join(raw, name))
rmf(os.path.join(programs, name + '.c'))
random = Random()
while True:
try_data(ConjectureData(
max_length=BUFFER_SIZE,
draw_bytes=lambda self, n: random.getrandbits(n * 8).to_bytes(n, 'big')
))
def new_conjecture_data(self, prefix, max_length=BUFFER_SIZE, observer=None):
return ConjectureData(
prefix=prefix,
max_length=max_length,
random=self.random,
observer=observer or self.tree.new_observer(),
)
def generate_novel_prefix(self, random):
"""Generate a short random string that (after rewriting) is not
a prefix of any buffer previously added to the tree.
This is logically equivalent to generating the test case uniformly
at random and returning the first point at which we hit unknown
territory, but with an optimisation for the only common case where
that would be inefficient.
"""
assert not self.is_exhausted
initial = self.find_necessary_prefix_for_novelty()
while True:
def draw_bytes(data, n):
i = data.index
if i < len(initial):
return initial[i : i + n]
else:
return uniform(random, n)
data = ConjectureData(draw_bytes=draw_bytes, max_length=float("inf"))
try:
self.simulate_test_function(data)
except PreviouslyUnseenBehaviour:
return hbytes(data.buffer)
def generate_new_examples(self):
def draw_bytes(data, n):
return hbytes([255] * n)
self.test_function(
ConjectureData(draw_bytes=draw_bytes,
max_length=self.settings.buffer_size))
def test_partial_buffer(n, rnd):
data = ConjectureData(
prefix=[n],
random=rnd,
max_length=2,
)
assert data.draw_bytes(2)[0] == n
def new_buffer(self):
assert not self.__tree_is_exhausted()
self.last_data = ConjectureData(
max_length=self.settings.buffer_size,
draw_bytes=lambda data, n, distribution: self.
__rewrite_for_novelty(data, distribution(self.random, n)))
self.test_function(self.last_data)
self.last_data.freeze()
def new_buffer(self):
self.last_data = ConjectureData(
max_length=self.settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(self.random, n)
)
self.test_function(self.last_data)
self.last_data.freeze()
def test_distribution_is_correctly_translated(inter, rnd):
assert inter == sorted(inter)
lower, c1, c2, upper = inter
d = ConjectureData(
draw_bytes=lambda data, n, distribution: distribution(rnd, n),
max_length=10**6)
assert d.draw(interval(lower, upper, c1, lambda r: c2)) == c2
assert d.draw(interval(lower, upper, c2, lambda r: c1)) == c1
def new_buffer(self):
assert not self.__tree_is_exhausted()
def draw_bytes(data, n):
return self.__rewrite_for_novelty(
data, self.__zero_bound(data, uniform(self.random, n))
)
self.last_data = ConjectureData(
max_length=self.settings.buffer_size,
draw_bytes=draw_bytes
)
self.test_function(self.last_data)
self.last_data.freeze()
def generate(ctx):
random = Random(ctx.obj['seed'])
log_data = ctx.obj['log_data']
run_tstl = ctx.obj['run_tstl']
while True:
target = ConjectureData(
prefix=b'',
random=random,
max_length=eng.BUFFER_SIZE,
)
try:
run_tstl(target)
except StopTest:
pass
log_data(target)
if target.status == Status.INTERESTING:
break
ctx.obj['log'].close()
def attempt_to_improve(self, example_index, parameter):
"""Part of our hill climbing implementation. Attempts to improve a
given score by regenerating an example in the data based on a new
parameter."""
data = self.current_data
self.current_score
ex = data.examples[example_index]
assert ex.length > 0
prefix_size = ex.start
prefix = data.buffer[:prefix_size]
dummy = ConjectureData(
draw_bytes=draw_bytes_with(prefix, parameter), max_length=BUFFER_SIZE
)
try:
self.engine.tree.simulate_test_function(dummy)
# If this didn't throw an exception then we've already seen this
# behaviour before and are trying something too similar to what
# we already have.
return False
except PreviouslyUnseenBehaviour:
pass
attempt = self.engine.new_conjecture_data(
draw_bytes_with(dummy.buffer, parameter)
)
self.engine.test_function(attempt)
if self.consider_new_test_data(attempt):
return True
ex_attempt = attempt.examples[example_index]
replacement = attempt.buffer[ex_attempt.start : ex_attempt.end]
return self.consider_new_buffer(prefix + replacement + data.buffer[ex.end :])
def generate_new_examples(self):
if Phase.generate not in self.settings.phases:
return
zero_data = self.cached_test_function(hbytes(
self.settings.buffer_size))
if zero_data.status == Status.OVERRUN or (
zero_data.status == Status.VALID
and len(zero_data.buffer) * 2 > self.settings.buffer_size):
fail_health_check(
self.settings,
"The smallest natural example for your test is extremely "
"large. This makes it difficult for Hypothesis to generate "
"good examples, especially when trying to reduce failing ones "
"at the end. Consider reducing the size of your data if it is "
"of a fixed size. You could also fix this by improving how "
"your data shrinks (see https://hypothesis.readthedocs.io/en/"
"latest/data.html#shrinking for details), or by introducing "
"default values inside your strategy. e.g. could you replace "
"some arguments with their defaults by using "
"one_of(none(), some_complex_strategy)?",
HealthCheck.large_base_example,
)
if zero_data is not Overrun:
# If the language starts with writes of length >= cap then there is
# only one string in it: Everything after cap is forced to be zero (or
# to be whatever value is written there). That means that once we've
# tried the zero value, there's nothing left for us to do, so we
# exit early here.
for i in hrange(self.cap):
if i not in zero_data.forced_indices:
break
else:
self.exit_with(ExitReason.finished)
self.health_check_state = HealthCheckState()
count = 0
while not self.interesting_examples and (
count < 10 or self.health_check_state is not None):
prefix = self.generate_novel_prefix()
def draw_bytes(data, n):
if data.index < len(prefix):
result = prefix[data.index:data.index + n]
if len(result) < n:
result += uniform(self.random, n - len(result))
else:
result = uniform(self.random, n)
return self.__zero_bound(data, result)
targets_found = len(self.covering_examples)
last_data = ConjectureData(max_length=self.settings.buffer_size,
draw_bytes=draw_bytes)
self.test_function(last_data)
last_data.freeze()
count += 1
mutations = 0
mutator = self._new_mutator()
zero_bound_queue = []
while not self.interesting_examples:
if zero_bound_queue:
# Whenever we generated an example and it hits a bound
# which forces zero blocks into it, this creates a weird
# distortion effect by making certain parts of the data
# stream (especially ones to the right) much more likely
# to be zero. We fix this by redistributing the generated
# data by shuffling it randomly. This results in the
# zero data being spread evenly throughout the buffer.
# Hopefully the shrinking this causes will cause us to
# naturally fail to hit the bound.
# If it doesn't then we will queue the new version up again
# (now with more zeros) and try again.
overdrawn = zero_bound_queue.pop()
buffer = bytearray(overdrawn.buffer)
# These will have values written to them that are different
# from what's in them anyway, so the value there doesn't
# really "count" for distributional purposes, and if we
# leave them in then they can cause the fraction of non
# zero bytes to increase on redraw instead of decrease.
for i in overdrawn.forced_indices:
buffer[i] = 0
self.random.shuffle(buffer)
buffer = hbytes(buffer)
def draw_bytes(data, n):
result = buffer[data.index:data.index + n]
if len(result) < n:
result += hbytes(n - len(result))
return self.__rewrite(data, result)
data = ConjectureData(draw_bytes=draw_bytes,
max_length=self.settings.buffer_size)
self.test_function(data)
data.freeze()
else:
origin = self.target_selector.select()
mutations += 1
targets_found = len(self.covering_examples)
data = ConjectureData(draw_bytes=mutator(origin),
max_length=self.settings.buffer_size)
self.test_function(data)
data.freeze()
if (data.status > origin.status
or len(self.covering_examples) > targets_found):
mutations = 0
elif data.status < origin.status or mutations >= 10:
# Cap the variations of a single example and move on to
# an entirely fresh start. Ten is an entirely arbitrary
# constant, but it's been working well for years.
mutations = 0
mutator = self._new_mutator()
if getattr(data, "hit_zero_bound", False):
zero_bound_queue.append(data)
mutations += 1
def _run(self):
self.last_data = None
mutations = 0
start_time = time.time()
self.reuse_existing_examples()
if (
Phase.generate in self.settings.phases and not
self.__tree_is_exhausted()
):
if (
self.last_data is None or
self.last_data.status < Status.INTERESTING
):
self.new_buffer()
mutator = self._new_mutator()
zero_bound_queue = []
while (
self.last_data.status != Status.INTERESTING and
not self.__tree_is_exhausted()
):
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(
self.settings.max_iterations, self.settings.max_examples
):
self.exit_reason = ExitReason.max_iterations
return
if (
self.settings.timeout > 0 and
time.time() >= start_time + self.settings.timeout
):
self.exit_reason = ExitReason.timeout
return
if zero_bound_queue:
# Whenever we generated an example and it hits a bound
# which forces zero blocks into it, this creates a weird
# distortion effect by making certain parts of the data
# stream (especially ones to the right) much more likely
# to be zero. We fix this by redistributing the generated
# data by shuffling it randomly. This results in the
# zero data being spread evenly throughout the buffer.
# Hopefully the shrinking this causes will cause us to
# naturally fail to hit the bound.
# If it doesn't then we will queue the new version up again
# (now with more zeros) and try again.
overdrawn = zero_bound_queue.pop()
buffer = bytearray(overdrawn.buffer)
self.random.shuffle(buffer)
buffer = hbytes(buffer)
if buffer == overdrawn.buffer:
continue
def draw_bytes(data, n, distribution):
result = buffer[data.index:data.index + n]
if len(result) < n:
result += hbytes(n - len(result))
return self.__rewrite(data, result)
data = ConjectureData(
draw_bytes=draw_bytes,
max_length=self.settings.buffer_size,
)
self.test_function(data)
data.freeze()
elif mutations >= self.settings.max_mutations:
mutations = 0
data = self.new_buffer()
mutator = self._new_mutator()
else:
data = ConjectureData(
draw_bytes=mutator,
max_length=self.settings.buffer_size
)
self.test_function(data)
data.freeze()
prev_data = self.last_data
if self.consider_new_test_data(data):
self.last_data = data
if data.status > prev_data.status:
mutations = 0
else:
mutator = self._new_mutator()
if getattr(data, 'hit_zero_bound', False):
zero_bound_queue.append(data)
mutations += 1
if self.__tree_is_exhausted():
self.exit_reason = ExitReason.finished
return
data = self.last_data
if data is None:
self.exit_reason = ExitReason.finished
return
assert isinstance(data.output, text_type)
if self.settings.max_shrinks <= 0:
self.exit_reason = ExitReason.max_shrinks
return
if Phase.shrink not in self.settings.phases:
self.exit_reason = ExitReason.finished
return
data = ConjectureData.for_buffer(self.last_data.buffer)
self.test_function(data)
if data.status != Status.INTERESTING:
self.exit_reason = ExitReason.flaky
return
self.shrink()
def new_conjecture_data(self, draw_bytes):
return ConjectureData(
draw_bytes=draw_bytes,
max_length=self.settings.buffer_size,
observer=self.tree.new_observer(),
)
if __name__ == '__main__':
try:
os.makedirs(EXAMPLES)
except FileExistsError:
pass
rnd = Random()
found = 0
i = 0
while found < LIMIT:
i += 1
print("Iter", i)
data = ConjectureData(max_length=2**29,
draw_bytes=lambda self, n: rnd.getrandbits(8 * n)
.to_bytes(n, byteorder='big'))
sourcename = "example.c"
gen(data, sourcename)
print("Generated")
if results_for_o1_and_o2_differ(sourcename):
found += 1
print("Found", found)
result = bytes(data.buffer)
name = hashlib.sha1(result).hexdigest()[:16]
with open(os.path.join(EXAMPLES, name), 'wb') as outfile:
outfile.write(result)
def perform_health_checks(random, settings, test_runner, search_strategy):
if not settings.perform_health_check:
return
if not Settings.default.perform_health_check:
return
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = ConjectureData(max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n))
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(
data,
reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while (count < 10 and time.time() < start + 1 and filtered_draws < 50
and overruns < 20):
try:
data = ConjectureData(max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n))
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(
data,
reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except InvalidArgument:
raise
except Exception:
escalate_hypothesis_internal_error()
if (HealthCheck.exception_in_generation
in settings.suppress_health_check):
raise
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
settings,
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.',
HealthCheck.exception_in_generation,
)
else:
fail_health_check(
settings,
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ',
HealthCheck.exception_in_generation,
)
if overruns >= 20 or (not count and overruns > 0):
fail_health_check(
settings,
('Examples routinely exceeded the max allowable size. '
'(%d examples overran while generating %d valid ones)'
'. Generating examples this large will usually lead to'
' bad results. You should try setting average_size or '
'max_size parameters on your collections and turning '
'max_leaves down on recursive() calls.') % (overruns, count),
HealthCheck.data_too_large)
if filtered_draws >= 50 or (not count and filtered_draws > 0):
fail_health_check(
settings, ('It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less. This can also be caused by '
'a low max_leaves parameter in recursive() calls') %
(filtered_draws, count), HealthCheck.filter_too_much)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check(
settings,
('Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds (%d invalid ones '
'and %d exceeded maximum size). Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).') %
(count, runtime, filtered_draws, overruns),
HealthCheck.too_slow,
)
def _run(self):
self.last_data = None
mutations = 0
start_time = time.time()
if (
self.settings.database is not None and
self.database_key is not None
):
corpus = sorted(
self.settings.database.fetch(self.database_key),
key=lambda d: (len(d), d)
)
for existing in corpus:
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(
self.settings.max_iterations, self.settings.max_examples
):
self.exit_reason = ExitReason.max_iterations
return
data = ConjectureData.for_buffer(existing)
self.test_function(data)
data.freeze()
self.last_data = data
if data.status < Status.VALID:
self.settings.database.delete(
self.database_key, existing)
elif data.status == Status.VALID:
# Incremental garbage collection! we store a lot of
# examples in the DB as we shrink: Those that stay
# interesting get kept, those that become invalid get
# dropped, but those that are merely valid gradually go
# away over time.
if self.random.randint(0, 2) == 0:
self.settings.database.delete(
self.database_key, existing)
else:
assert data.status == Status.INTERESTING
self.last_data = data
break
if Phase.generate in self.settings.phases:
if (
self.last_data is None or
self.last_data.status < Status.INTERESTING
):
self.new_buffer()
mutator = self._new_mutator()
while self.last_data.status != Status.INTERESTING:
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(
self.settings.max_iterations, self.settings.max_examples
):
self.exit_reason = ExitReason.max_iterations
return
if (
self.settings.timeout > 0 and
time.time() >= start_time + self.settings.timeout
):
self.exit_reason = ExitReason.timeout
return
if mutations >= self.settings.max_mutations:
mutations = 0
self.new_buffer()
mutator = self._new_mutator()
else:
data = ConjectureData(
draw_bytes=mutator,
max_length=self.settings.buffer_size
)
self.test_function(data)
data.freeze()
prev_data = self.last_data
if self.consider_new_test_data(data):
self.last_data = data
if data.status > prev_data.status:
mutations = 0
else:
mutator = self._new_mutator()
mutations += 1
data = self.last_data
if data is None:
self.exit_reason = ExitReason.finished
return
assert isinstance(data.output, text_type)
if self.settings.max_shrinks <= 0:
self.exit_reason = ExitReason.max_shrinks
return
if Phase.shrink not in self.settings.phases:
self.exit_reason = ExitReason.finished
return
if not self.last_data.buffer:
self.exit_reason = ExitReason.finished
return
data = ConjectureData.for_buffer(self.last_data.buffer)
self.test_function(data)
if data.status != Status.INTERESTING:
self.exit_reason = ExitReason.flaky
return
change_counter = -1
while self.changed > change_counter:
change_counter = self.changed
self.debug('Random interval deletes')
failed_deletes = 0
while self.last_data.intervals and failed_deletes < 10:
if self.random.randint(0, 1):
u, v = self.random.choice(self.last_data.intervals)
else:
n = len(self.last_data.buffer) - 1
u, v = sorted((
self.random.choice(self.last_data.intervals)
))
if (
v < len(self.last_data.buffer)
) and self.incorporate_new_buffer(
self.last_data.buffer[:u] +
self.last_data.buffer[v:]
):
failed_deletes = 0
else:
failed_deletes += 1
self.debug('Structured interval deletes')
i = 0
while i < len(self.last_data.intervals):
u, v = self.last_data.intervals[i]
if not self.incorporate_new_buffer(
self.last_data.buffer[:u] +
self.last_data.buffer[v:]
):
i += 1
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Lexicographical minimization of whole buffer')
minimize(
self.last_data.buffer, self.incorporate_new_buffer,
cautious=True
)
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Replacing blocks with simpler blocks')
i = 0
while i < len(self.last_data.blocks):
u, v = self.last_data.blocks[i]
buf = self.last_data.buffer
block = buf[u:v]
n = v - u
all_blocks = sorted(set([bytes(n)] + [
buf[a:a + n]
for a in self.last_data.block_starts[n]
]))
better_blocks = all_blocks[:all_blocks.index(block)]
for b in better_blocks:
if self.incorporate_new_buffer(
buf[:u] + b + buf[v:]
):
break
i += 1
self.debug('Simultaneous shrinking of duplicated blocks')
block_counter = -1
while block_counter < self.changed:
block_counter = self.changed
blocks = [
k for k, count in
Counter(
self.last_data.buffer[u:v]
for u, v in self.last_data.blocks).items()
if count > 1
]
for block in blocks:
parts = [
self.last_data.buffer[r:s]
for r, s in self.last_data.blocks
]
def replace(b):
return b''.join(
bytes(b if c == block else c) for c in parts
)
minimize(
block,
lambda b: self.incorporate_new_buffer(replace(b)),
self.random
)
self.debug('Shrinking of individual blocks')
i = 0
while i < len(self.last_data.blocks):
u, v = self.last_data.blocks[i]
minimize(
self.last_data.buffer[u:v],
lambda b: self.incorporate_new_buffer(
self.last_data.buffer[:u] + b +
self.last_data.buffer[v:],
), self.random
)
i += 1
self.debug('Replacing intervals with simpler intervals')
interval_counter = -1
while interval_counter != self.changed:
interval_counter = self.changed
i = 0
alternatives = None
while i < len(self.last_data.intervals):
if alternatives is None:
alternatives = sorted(set(
self.last_data.buffer[u:v]
for u, v in self.last_data.intervals), key=len)
u, v = self.last_data.intervals[i]
for a in alternatives:
buf = self.last_data.buffer
if (
len(a) < v - u or
(len(a) == (v - u) and a < buf[u:v])
):
if self.incorporate_new_buffer(
buf[:u] + a + buf[v:]
):
alternatives = None
break
i += 1
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Shuffling suffixes while shrinking %r' % (
self.last_data.bind_points,
))
b = 0
while b < len(self.last_data.bind_points):
cutoff = sorted(self.last_data.bind_points)[b]
def test_value(prefix):
for t in hrange(5):
alphabet = {}
for i, j in self.last_data.blocks[b:]:
alphabet.setdefault(j - i, []).append((i, j))
if t > 0:
for v in alphabet.values():
self.random.shuffle(v)
buf = bytearray(prefix)
for i, j in self.last_data.blocks[b:]:
u, v = alphabet[j - i].pop()
buf.extend(self.last_data.buffer[u:v])
if self.incorporate_new_buffer(hbytes(buf)):
return True
return False
minimize(
self.last_data.buffer[:cutoff], test_value, cautious=True
)
b += 1
self.exit_reason = ExitReason.finished
def wrapped_test(*arguments, **kwargs):
settings = wrapped_test._hypothesis_internal_use_settings
if wrapped_test._hypothesis_internal_use_seed is not None:
random = Random(
wrapped_test._hypothesis_internal_use_seed)
elif settings.derandomize:
random = Random(function_digest(test))
else:
random = new_random()
import hypothesis.strategies as sd
selfy = None
arguments, kwargs = convert_positional_arguments(
wrapped_test, arguments, kwargs)
# If the test function is a method of some kind, the bound object
# will be the first named argument if there are any, otherwise the
# first vararg (if any).
if argspec.args:
selfy = kwargs.get(argspec.args[0])
elif arguments:
selfy = arguments[0]
test_runner = new_style_executor(selfy)
for example in reversed(getattr(
wrapped_test, 'hypothesis_explicit_examples', ()
)):
if example.args:
if len(example.args) > len(original_argspec.args):
raise InvalidArgument(
'example has too many arguments for test. '
'Expected at most %d but got %d' % (
len(original_argspec.args), len(example.args)))
example_kwargs = dict(zip(
original_argspec.args[-len(example.args):],
example.args
))
else:
example_kwargs = example.kwargs
if Phase.explicit not in settings.phases:
continue
example_kwargs.update(kwargs)
# Note: Test may mutate arguments and we can't rerun explicit
# examples, so we have to calculate the failure message at this
# point rather than than later.
message_on_failure = 'Falsifying example: %s(%s)' % (
test.__name__, arg_string(test, arguments, example_kwargs)
)
try:
with BuildContext(None) as b:
test_runner(
None,
lambda data: test(*arguments, **example_kwargs)
)
except BaseException:
traceback.print_exc()
report(message_on_failure)
for n in b.notes:
report(n)
raise
if settings.max_examples <= 0:
return
arguments = tuple(arguments)
given_specifier = sd.tuples(
sd.just(arguments),
sd.fixed_dictionaries(generator_kwargs).map(
lambda args: dict(args, **kwargs)
)
)
def fail_health_check(message, label):
if label in settings.suppress_health_check:
return
message += (
'\nSee https://hypothesis.readthedocs.io/en/latest/health'
'checks.html for more information about this. '
)
message += (
'If you want to disable just this health check, add %s '
'to the suppress_health_check settings for this test.'
) % (label,)
raise FailedHealthCheck(message)
search_strategy = given_specifier
if selfy is not None:
search_strategy = WithRunner(search_strategy, selfy)
search_strategy.validate()
perform_health_check = settings.perform_health_check
perform_health_check &= Settings.default.perform_health_check
from hypothesis.internal.conjecture.data import ConjectureData, \
Status, StopTest
if not (
Phase.reuse in settings.phases or
Phase.generate in settings.phases
):
return
if perform_health_check:
health_check_random = Random(random.getrandbits(128))
# We "pre warm" the health check with one draw to give it some
# time to calculate any cached data. This prevents the case
# where the first draw of the health check takes ages because
# of loading unicode data the first time.
data = ConjectureData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
try:
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
except BaseException:
pass
count = 0
overruns = 0
filtered_draws = 0
start = time.time()
while (
count < 10 and time.time() < start + 1 and
filtered_draws < 50 and overruns < 20
):
try:
data = ConjectureData(
max_length=settings.buffer_size,
draw_bytes=lambda data, n, distribution:
distribution(health_check_random, n)
)
with Settings(settings, verbosity=Verbosity.quiet):
test_runner(data, reify_and_execute(
search_strategy,
lambda *args, **kwargs: None,
))
count += 1
except UnsatisfiedAssumption:
filtered_draws += 1
except StopTest:
if data.status == Status.INVALID:
filtered_draws += 1
else:
assert data.status == Status.OVERRUN
overruns += 1
except InvalidArgument:
raise
except Exception:
if (
HealthCheck.exception_in_generation in
settings.suppress_health_check
):
raise
report(traceback.format_exc())
if test_runner is default_new_style_executor:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
"an error in a function you've passed to "
'it to construct your data.',
HealthCheck.exception_in_generation,
)
else:
fail_health_check(
'An exception occurred during data '
'generation in initial health check. '
'This indicates a bug in the strategy. '
'This could either be a Hypothesis bug or '
'an error in a function you\'ve passed to '
'it to construct your data. Additionally, '
'you have a custom executor, which means '
'that this could be your executor failing '
'to handle a function which returns None. ',
HealthCheck.exception_in_generation,
)
if overruns >= 20 or (
not count and overruns > 0
):
fail_health_check((
'Examples routinely exceeded the max allowable size. '
'(%d examples overran while generating %d valid ones)'
'. Generating examples this large will usually lead to'
' bad results. You should try setting average_size or '
'max_size parameters on your collections and turning '
'max_leaves down on recursive() calls.') % (
overruns, count
), HealthCheck.data_too_large)
if filtered_draws >= 50 or (
not count and filtered_draws > 0
):
fail_health_check((
'It looks like your strategy is filtering out a lot '
'of data. Health check found %d filtered examples but '
'only %d good ones. This will make your tests much '
'slower, and also will probably distort the data '
'generation quite a lot. You should adapt your '
'strategy to filter less. This can also be caused by '
'a low max_leaves parameter in recursive() calls') % (
filtered_draws, count
), HealthCheck.filter_too_much)
runtime = time.time() - start
if runtime > 1.0 or count < 10:
fail_health_check((
'Data generation is extremely slow: Only produced '
'%d valid examples in %.2f seconds (%d invalid ones '
'and %d exceeded maximum size). Try decreasing '
"size of the data you're generating (with e.g."
'average_size or max_leaves parameters).'
) % (count, runtime, filtered_draws, overruns),
HealthCheck.too_slow,
)
last_exception = [None]
repr_for_last_exception = [None]
def evaluate_test_data(data):
try:
result = test_runner(data, reify_and_execute(
search_strategy, test,
))
if result is not None and settings.perform_health_check:
fail_health_check((
'Tests run under @given should return None, but '
'%s returned %r instead.'
) % (test.__name__, result), HealthCheck.return_value)
return False
except UnsatisfiedAssumption:
data.mark_invalid()
except (
HypothesisDeprecationWarning, FailedHealthCheck,
StopTest,
):
raise
except Exception:
last_exception[0] = traceback.format_exc()
verbose_report(last_exception[0])
data.mark_interesting()
from hypothesis.internal.conjecture.engine import ConjectureRunner
falsifying_example = None
database_key = str_to_bytes(fully_qualified_name(test))
start_time = time.time()
runner = ConjectureRunner(
evaluate_test_data,
settings=settings, random=random,
database_key=database_key,
)
runner.run()
note_engine_for_statistics(runner)
run_time = time.time() - start_time
timed_out = (
settings.timeout > 0 and
run_time >= settings.timeout
)
if runner.last_data is None:
return
if runner.last_data.status == Status.INTERESTING:
falsifying_example = runner.last_data.buffer
if settings.database is not None:
settings.database.save(
database_key, falsifying_example
)
else:
if runner.valid_examples < min(
settings.min_satisfying_examples,
settings.max_examples,
):
if timed_out:
raise Timeout((
'Ran out of time before finding a satisfying '
'example for '
'%s. Only found %d examples in ' +
'%.2fs.'
) % (
get_pretty_function_description(test),
runner.valid_examples, run_time
))
else:
raise Unsatisfiable((
'Unable to satisfy assumptions of hypothesis '
'%s. Only %d examples considered '
'satisfied assumptions'
) % (
get_pretty_function_description(test),
runner.valid_examples,))
return
assert last_exception[0] is not None
try:
with settings:
test_runner(
ConjectureData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy, test,
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
report(traceback.format_exc())
raise Flaky(
'Unreliable assumption: An example which satisfied '
'assumptions on the first run now fails it.'
)
report(
'Failed to reproduce exception. Expected: \n' +
last_exception[0],
)
filter_message = (
'Unreliable test data: Failed to reproduce a failure '
'and then when it came to recreating the example in '
'order to print the test data with a flaky result '
'the example was filtered out (by e.g. a '
'call to filter in your strategy) when we didn\'t '
'expect it to be.'
)
try:
test_runner(
ConjectureData.for_buffer(falsifying_example),
reify_and_execute(
search_strategy,
test_is_flaky(test, repr_for_last_exception[0]),
print_example=True, is_final=True
))
except (UnsatisfiedAssumption, StopTest):
raise Flaky(filter_message)
def new_conjecture_data(self, draw_bytes):
return ConjectureData(
draw_bytes=draw_bytes,
max_length=BUFFER_SIZE,
observer=self.tree.new_observer(),
)
def _run(self):
self.last_data = None
mutations = 0
start_time = time.time()
if (
self.settings.database is not None and
self.database_key is not None
):
corpus = sorted(
self.settings.database.fetch(self.database_key),
key=lambda d: (len(d), d)
)
for existing in corpus:
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(
self.settings.max_iterations, self.settings.max_examples
):
self.exit_reason = ExitReason.max_iterations
return
data = ConjectureData.for_buffer(existing)
self.test_function(data)
data.freeze()
self.last_data = data
self.consider_new_test_data(data)
if data.status < Status.VALID:
self.settings.database.delete(
self.database_key, existing)
elif data.status == Status.VALID:
# Incremental garbage collection! we store a lot of
# examples in the DB as we shrink: Those that stay
# interesting get kept, those that become invalid get
# dropped, but those that are merely valid gradually go
# away over time.
if self.random.randint(0, 2) == 0:
self.settings.database.delete(
self.database_key, existing)
else:
assert data.status == Status.INTERESTING
self.last_data = data
break
if (
Phase.generate in self.settings.phases and not
self.__tree_is_exhausted()
):
if (
self.last_data is None or
self.last_data.status < Status.INTERESTING
):
self.new_buffer()
mutator = self._new_mutator()
while (
self.last_data.status != Status.INTERESTING and
not self.__tree_is_exhausted()
):
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(
self.settings.max_iterations, self.settings.max_examples
):
self.exit_reason = ExitReason.max_iterations
return
if (
self.settings.timeout > 0 and
time.time() >= start_time + self.settings.timeout
):
self.exit_reason = ExitReason.timeout
return
if mutations >= self.settings.max_mutations:
mutations = 0
self.new_buffer()
mutator = self._new_mutator()
else:
data = ConjectureData(
draw_bytes=mutator,
max_length=self.settings.buffer_size
)
self.test_function(data)
data.freeze()
prev_data = self.last_data
if self.consider_new_test_data(data):
self.last_data = data
if data.status > prev_data.status:
mutations = 0
else:
mutator = self._new_mutator()
mutations += 1
if self.__tree_is_exhausted():
self.exit_reason = ExitReason.finished
return
data = self.last_data
if data is None:
self.exit_reason = ExitReason.finished
return
assert isinstance(data.output, text_type)
if self.settings.max_shrinks <= 0:
self.exit_reason = ExitReason.max_shrinks
return
if Phase.shrink not in self.settings.phases:
self.exit_reason = ExitReason.finished
return
data = ConjectureData.for_buffer(self.last_data.buffer)
self.test_function(data)
if data.status != Status.INTERESTING:
self.exit_reason = ExitReason.flaky
return
self.shrink()
def _run(self):
self.last_data = None
mutations = 0
start_time = time.time()
if (self.settings.database is not None
and self.database_key is not None):
corpus = sorted(self.settings.database.fetch(self.database_key),
key=lambda d: (len(d), d))
for existing in corpus:
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(self.settings.max_iterations,
self.settings.max_examples):
self.exit_reason = ExitReason.max_iterations
return
data = ConjectureData.for_buffer(existing)
self.test_function(data)
data.freeze()
self.last_data = data
self.consider_new_test_data(data)
if data.status < Status.VALID:
self.settings.database.delete(self.database_key, existing)
elif data.status == Status.VALID:
# Incremental garbage collection! we store a lot of
# examples in the DB as we shrink: Those that stay
# interesting get kept, those that become invalid get
# dropped, but those that are merely valid gradually go
# away over time.
if self.random.randint(0, 2) == 0:
self.settings.database.delete(self.database_key,
existing)
else:
assert data.status == Status.INTERESTING
self.last_data = data
break
if (Phase.generate in self.settings.phases
and not self.__tree_is_exhausted()):
if (self.last_data is None
or self.last_data.status < Status.INTERESTING):
self.new_buffer()
mutator = self._new_mutator()
while (self.last_data.status != Status.INTERESTING
and not self.__tree_is_exhausted()):
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(self.settings.max_iterations,
self.settings.max_examples):
self.exit_reason = ExitReason.max_iterations
return
if (self.settings.timeout > 0
and time.time() >= start_time + self.settings.timeout):
self.exit_reason = ExitReason.timeout
return
if mutations >= self.settings.max_mutations:
mutations = 0
self.new_buffer()
mutator = self._new_mutator()
else:
data = ConjectureData(draw_bytes=mutator,
max_length=self.settings.buffer_size)
self.test_function(data)
data.freeze()
prev_data = self.last_data
if self.consider_new_test_data(data):
self.last_data = data
if data.status > prev_data.status:
mutations = 0
else:
mutator = self._new_mutator()
mutations += 1
if self.__tree_is_exhausted():
self.exit_reason = ExitReason.finished
return
data = self.last_data
if data is None:
self.exit_reason = ExitReason.finished
return
assert isinstance(data.output, text_type)
if self.settings.max_shrinks <= 0:
self.exit_reason = ExitReason.max_shrinks
return
if Phase.shrink not in self.settings.phases:
self.exit_reason = ExitReason.finished
return
data = ConjectureData.for_buffer(self.last_data.buffer)
self.test_function(data)
if data.status != Status.INTERESTING:
self.exit_reason = ExitReason.flaky
return
change_counter = -1
while self.changed > change_counter:
change_counter = self.changed
self.debug('Structured interval deletes')
k = len(self.last_data.intervals) // 2
while k > 0:
i = 0
while i + k <= len(self.last_data.intervals):
bitmask = [True] * len(self.last_data.buffer)
for u, v in self.last_data.intervals[i:i + k]:
for t in range(u, v):
bitmask[t] = False
u, v = self.last_data.intervals[i]
if not self.incorporate_new_buffer(
hbytes(
b
for b, v in zip(self.last_data.buffer, bitmask)
if v)):
i += k
k //= 2
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Bulk replacing blocks with simpler blocks')
i = 0
while i < len(self.last_data.blocks):
u, v = self.last_data.blocks[i]
buf = self.last_data.buffer
block = buf[u:v]
n = v - u
buffer = bytearray()
for r, s in self.last_data.blocks:
if s - r == n and self.last_data.buffer[r:s] > block:
buffer.extend(block)
else:
buffer.extend(self.last_data.buffer[r:s])
self.incorporate_new_buffer(hbytes(buffer))
i += 1
self.debug('Replacing individual blocks with simpler blocks')
i = 0
while i < len(self.last_data.blocks):
u, v = self.last_data.blocks[i]
buf = self.last_data.buffer
block = buf[u:v]
n = v - u
all_blocks = sorted(
set([hbytes(n)] +
[buf[a:a + n]
for a in self.last_data.block_starts[n]]))
better_blocks = all_blocks[:all_blocks.index(block)]
for b in better_blocks:
if self.incorporate_new_buffer(buf[:u] + b + buf[v:]):
break
i += 1
self.debug('Simultaneous shrinking of duplicated blocks')
block_counter = -1
while block_counter < self.changed:
block_counter = self.changed
blocks = [
k for k, count in Counter(
self.last_data.buffer[u:v]
for u, v in self.last_data.blocks).items() if count > 1
]
for block in blocks:
parts = [
self.last_data.buffer[r:s]
for r, s in self.last_data.blocks
]
def replace(b):
return hbytes(
EMPTY_BYTES.join(
hbytes(b if c == block else c) for c in parts))
minimize(block,
lambda b: self.incorporate_new_buffer(replace(b)),
self.random)
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Lexicographical minimization of whole buffer')
minimize(self.last_data.buffer,
self.incorporate_new_buffer,
cautious=True)
self.debug('Shrinking of individual blocks')
i = 0
while i < len(self.last_data.blocks):
u, v = self.last_data.blocks[i]
minimize(
self.last_data.buffer[u:v],
lambda b: self.incorporate_new_buffer(
self.last_data.buffer[:u] + b + self.last_data.buffer[
v:], ), self.random)
i += 1
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Reordering blocks')
block_lengths = sorted(self.last_data.block_starts, reverse=True)
for n in block_lengths:
i = 1
while i < len(self.last_data.block_starts.get(n, ())):
j = i
while j > 0:
buf = self.last_data.buffer
blocks = self.last_data.block_starts[n]
a_start = blocks[j - 1]
b_start = blocks[j]
a = buf[a_start:a_start + n]
b = buf[b_start:b_start + n]
if a <= b:
break
swapped = (buf[:a_start] + b +
buf[a_start + n:b_start] + a +
buf[b_start + n:])
assert len(swapped) == len(buf)
assert swapped < buf
if self.incorporate_new_buffer(swapped):
j -= 1
else:
break
i += 1
if change_counter != self.changed:
self.debug('Restarting')
continue
self.debug('Shuffling suffixes while shrinking %r' %
(self.last_data.bind_points, ))
b = 0
while b < len(self.last_data.bind_points):
cutoff = sorted(self.last_data.bind_points)[b]
def test_value(prefix):
for t in hrange(5):
alphabet = {}
for i, j in self.last_data.blocks[b:]:
alphabet.setdefault(j - i, []).append((i, j))
if t > 0:
for v in alphabet.values():
self.random.shuffle(v)
buf = bytearray(prefix)
for i, j in self.last_data.blocks[b:]:
u, v = alphabet[j - i].pop()
buf.extend(self.last_data.buffer[u:v])
if self.incorporate_new_buffer(hbytes(buf)):
return True
return False
minimize(self.last_data.buffer[:cutoff],
test_value,
cautious=True)
b += 1
self.exit_reason = ExitReason.finished
def generate_new_examples(self):
if Phase.generate not in self.settings.phases:
return
zero_data = ConjectureData(max_length=self.settings.buffer_size,
draw_bytes=lambda data, n: self.
__rewrite_for_novelty(data, hbytes(n)))
self.test_function(zero_data)
count = 0
while count < 10 and not self.interesting_examples:
def draw_bytes(data, n):
return self.__rewrite_for_novelty(
data, self.__zero_bound(data, uniform(self.random, n)))
targets_found = len(self.covering_examples)
self.last_data = ConjectureData(
max_length=self.settings.buffer_size, draw_bytes=draw_bytes)
self.test_function(self.last_data)
self.last_data.freeze()
if len(self.covering_examples) > targets_found:
count = 0
else:
count += 1
mutations = 0
mutator = self._new_mutator()
zero_bound_queue = []
while not self.interesting_examples:
if zero_bound_queue:
# Whenever we generated an example and it hits a bound
# which forces zero blocks into it, this creates a weird
# distortion effect by making certain parts of the data
# stream (especially ones to the right) much more likely
# to be zero. We fix this by redistributing the generated
# data by shuffling it randomly. This results in the
# zero data being spread evenly throughout the buffer.
# Hopefully the shrinking this causes will cause us to
# naturally fail to hit the bound.
# If it doesn't then we will queue the new version up again
# (now with more zeros) and try again.
overdrawn = zero_bound_queue.pop()
buffer = bytearray(overdrawn.buffer)
# These will have values written to them that are different
# from what's in them anyway, so the value there doesn't
# really "count" for distributional purposes, and if we
# leave them in then they can cause the fraction of non
# zero bytes to increase on redraw instead of decrease.
for i in overdrawn.forced_indices:
buffer[i] = 0
self.random.shuffle(buffer)
buffer = hbytes(buffer)
def draw_bytes(data, n):
result = buffer[data.index:data.index + n]
if len(result) < n:
result += hbytes(n - len(result))
return self.__rewrite(data, result)
data = ConjectureData(
draw_bytes=draw_bytes,
max_length=self.settings.buffer_size,
)
self.test_function(data)
data.freeze()
else:
target, last_data = self.target_selector.select()
mutations += 1
targets_found = len(self.covering_examples)
prev_data = self.last_data
data = ConjectureData(draw_bytes=mutator,
max_length=self.settings.buffer_size)
self.test_function(data)
data.freeze()
if (data.status > prev_data.status
or len(self.covering_examples) > targets_found):
mutations = 0
elif (data.status < prev_data.status
or not self.target_selector.has_tag(target, data)
or mutations >= self.settings.max_mutations):
mutations = 0
mutator = self._new_mutator()
if getattr(data, 'hit_zero_bound', False):
zero_bound_queue.append(data)
mutations += 1
def _run(self):
self.last_data = None
mutations = 0
start_time = time.time()
self.reuse_existing_examples()
if (
Phase.generate in self.settings.phases and not
self.__tree_is_exhausted()
):
if (
self.last_data is None or
self.last_data.status < Status.INTERESTING
):
self.new_buffer()
mutator = self._new_mutator()
zero_bound_queue = []
while (
self.last_data.status != Status.INTERESTING and
not self.__tree_is_exhausted()
):
if self.valid_examples >= self.settings.max_examples:
self.exit_reason = ExitReason.max_examples
return
if self.call_count >= max(
self.settings.max_iterations, self.settings.max_examples
):
self.exit_reason = ExitReason.max_iterations
return
if (
self.settings.timeout > 0 and
time.time() >= start_time + self.settings.timeout
):
self.exit_reason = ExitReason.timeout
return
if zero_bound_queue:
# Whenever we generated an example and it hits a bound
# which forces zero blocks into it, this creates a weird
# distortion effect by making certain parts of the data
# stream (especially ones to the right) much more likely
# to be zero. We fix this by redistributing the generated
# data by shuffling it randomly. This results in the
# zero data being spread evenly throughout the buffer.
# Hopefully the shrinking this causes will cause us to
# naturally fail to hit the bound.
# If it doesn't then we will queue the new version up again
# (now with more zeros) and try again.
overdrawn = zero_bound_queue.pop()
buffer = bytearray(overdrawn.buffer)
# These will have values written to them that are different
# from what's in them anyway, so the value there doesn't
# really "count" for distributional purposes, and if we
# leave them in then they can cause the fraction of non
# zero bytes to increase on redraw instead of decrease.
for i in overdrawn.forced_indices:
buffer[i] = 0
self.random.shuffle(buffer)
buffer = hbytes(buffer)
if buffer == overdrawn.buffer:
continue
def draw_bytes(data, n):
result = buffer[data.index:data.index + n]
if len(result) < n:
result += hbytes(n - len(result))
return self.__rewrite(data, result)
data = ConjectureData(
draw_bytes=draw_bytes,
max_length=self.settings.buffer_size,
)
self.test_function(data)
data.freeze()
elif mutations >= self.settings.max_mutations:
mutations = 0
data = self.new_buffer()
mutator = self._new_mutator()
else:
data = ConjectureData(
draw_bytes=mutator,
max_length=self.settings.buffer_size
)
self.test_function(data)
data.freeze()
prev_data = self.last_data
if self.consider_new_test_data(data):
self.last_data = data
if data.status > prev_data.status:
mutations = 0
else:
mutator = self._new_mutator()
if getattr(data, 'hit_zero_bound', False):
zero_bound_queue.append(data)
mutations += 1
if self.__tree_is_exhausted():
self.exit_reason = ExitReason.finished
return
data = self.last_data
if data is None:
self.exit_reason = ExitReason.finished
return
assert isinstance(data.output, text_type)
if Phase.shrink not in self.settings.phases:
self.exit_reason = ExitReason.finished
return
data = ConjectureData.for_buffer(self.last_data.buffer)
self.test_function(data)
if data.status != Status.INTERESTING:
self.exit_reason = ExitReason.flaky
return
while len(self.shrunk_examples) < len(self.interesting_examples):
target, d = min([
(k, v) for k, v in self.interesting_examples.items()
if k not in self.shrunk_examples],
key=lambda kv: (sort_key(kv[1].buffer), sort_key(repr(kv[0]))),
)
self.debug('Shrinking %r' % (target,))
self.last_data = d
assert self.last_data.interesting_origin == target
self.shrink()
self.shrunk_examples.add(target)
