def test_the_slow_test_health_only_runs_if_health_checks_are_on():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all(), deadline=None)
def a(x):
time.sleep(1000)
a()
def test_returning_non_none_does_not_fail_if_health_check_disabled():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all())
def a(x):
return 1
a()
def test_database_clears_secondary_key():
key = b"key"
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) == 10:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=1,
buffer_size=1024,
database=database,
suppress_health_check=HealthCheck.all(),
),
database_key=key,
)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.cached_test_function([10])
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 0
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=5000,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug,
),
)
runner.cached_test_function(buf)
runner.run()
out, _ = capsys.readouterr()
assert re.match(u"\\d+ bytes \\[.*\\] -> ", out)
assert "INTERESTING" in out
def test_contains_the_test_function_name_in_the_exception_string():
look_for_one = settings(
max_examples=1, suppress_health_check=HealthCheck.all())
@given(integers())
@look_for_one
def this_has_a_totally_unique_name(x):
reject()
with raises(Unsatisfiable) as e:
this_has_a_totally_unique_name()
assert this_has_a_totally_unique_name.__name__ in e.value.args[0]
class Foo(object):
@given(integers())
@look_for_one
def this_has_a_unique_name_and_lives_on_a_class(self, x):
reject()
with raises(Unsatisfiable) as e:
Foo().this_has_a_unique_name_and_lives_on_a_class()
assert (
Foo.this_has_a_unique_name_and_lives_on_a_class.__name__
) in e.value.args[0]
def test_failure_sequence_inducing(building, testing, rnd):
buildit = iter(building)
testit = iter(testing)
def build(x):
try:
assume(not next(buildit))
except StopIteration:
pass
return x
@given(integers().map(build))
@settings(
verbosity=Verbosity.quiet, database=None,
suppress_health_check=HealthCheck.all(), phases=no_shrink
)
def test(x):
try:
i = next(testit)
except StopIteration:
return
if i == 1:
return
elif i == 2:
reject()
else:
raise Nope()
try:
test()
except (Nope, Unsatisfiable, Flaky):
pass
except UnsatisfiedAssumption:
raise SatisfyMe()
def test_database_uses_values_from_secondary_key():
key = b'key'
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) >= 5:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, buffer_size=1024,
database=database, suppress_health_check=HealthCheck.all(),
), database_key=key)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.test_function(ConjectureData.for_buffer(hbytes([10])))
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert set(
map(int_from_bytes, database.fetch(runner.secondary_key))
) == set(range(6, 11))
v, = runner.interesting_examples.values()
assert list(v.buffer) == [5]
def test_notes_hard_to_satisfy():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all())
def test(i):
assume(i == 0)
stats = call_for_statistics(test)
assert 'satisfied assumptions' in stats.exit_reason
def test_raises_unsatisfiable_if_all_false():
@given(integers())
@settings(max_examples=50, suppress_health_check=HealthCheck.all())
def test_assume_false(x):
reject()
with pytest.raises(Unsatisfiable):
test_assume_false()
def test_default_health_check_can_weaken_specific():
import random
@settings(suppress_health_check=HealthCheck.all())
@given(st.lists(st.integers(), min_size=1))
def test(x):
random.choice(x)
test()
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(f, random=Random(0), settings=settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited,
phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug
), database_key=None)
runner.test_function(ConjectureData.for_buffer(blob))
assert runner.interesting_examples
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.clear_passes()
shrinker.add_new_pass('minimize_individual_blocks')
shrinker.shrink()
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed])
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
def run_to_buffer(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return hbytes(last_data.buffer)
def accept(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.test_function(ConjectureData.for_buffer(start))
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return runner.new_shrinker(
last_data, lambda d: d.status == Status.INTERESTING
)
def test_large_initial_write():
big = hbytes(b'\xff') * 512
def f(data):
data.write(big)
data.draw_bits(63)
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.exit_reason == ExitReason.finished
def test_exit_because_max_iterations():
def f(data):
data.draw_bits(64)
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.call_count <= 1000
assert runner.exit_reason == ExitReason.max_iterations
def test_can_increase_number_of_bytes_drawn_in_tail():
# This is designed to trigger a case where the zero bound queue will end up
# increasing the size of data drawn because moving zeroes into the initial
# prefix will increase the amount drawn.
def f(data):
x = data.draw_bytes(5)
n = x.count(0)
b = data.draw_bytes(n + 1)
assert not any(b)
runner = ConjectureRunner(
f, settings=settings(
max_examples=100,
buffer_size=11, suppress_health_check=HealthCheck.all()))
runner.run()
def test_will_reset_the_tree_as_it_goes(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
data.draw_bits(8)
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree[0]) > 1
step(2)
assert len(runner.tree[0]) == 1
def run_language_test_for(root, data, seed):
random.seed(seed)
def test(local_data):
node = root
while not isinstance(node, Terminal):
if isinstance(node, Write):
local_data.write(hbytes(node.value))
node = node.child
else:
assert isinstance(node, Branch)
c = local_data.draw_bits(node.bits)
try:
node = node.children[c]
except KeyError:
if data is None:
return
node = node.children.setdefault(c, data.draw(nodes))
assert isinstance(node, Terminal)
if node.status == Status.INTERESTING:
local_data.mark_interesting(node.payload)
elif node.status == Status.INVALID:
local_data.mark_invalid()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=1,
buffer_size=512,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.quiet,
phases=list(Phase),
),
)
try:
runner.run()
finally:
if data is not None:
note(root)
assume(runner.interesting_examples)
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
if data.draw_bits(8) == 7:
data.mark_interesting()
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree) > 1
step(7)
assert len(runner.tree) > 1
t = len(runner.tree)
runner.shrink_interesting_examples()
assert len(runner.tree) > t
m = merged([schema, schema])
assert m == canonicalish(schema)
def _merge_semantics_helper(data, s1, s2, combined):
note(combined)
ic = data.draw(from_schema(combined), label="combined")
i1 = data.draw(from_schema(s1), label="s1")
i2 = data.draw(from_schema(s2), label="s2")
assert is_valid(ic, s1)
assert is_valid(ic, s2)
assert is_valid(i1, s2) == is_valid(i1, combined)
assert is_valid(i2, s1) == is_valid(i2, combined)
@settings(suppress_health_check=HealthCheck.all(), deadline=None)
@given(st.data(), json_schemata(), json_schemata())
def test_merge_semantics(data, s1, s2):
assume(canonicalish(s1) != FALSEY and canonicalish(s2) != FALSEY)
combined = merged([s1, s2])
assume(combined is not None)
assume(combined != FALSEY)
_merge_semantics_helper(data, s1, s2, combined)
@settings(suppress_health_check=HealthCheck.all(), deadline=None)
@given(
st.data(),
gen_number(kind="integer") | gen_number(kind="number"),
gen_number(kind="integer") | gen_number(kind="number"),
)
# We draw n as two separate calls so that it doesn't show up as a
# single block. If it did, the heuristics that allow us to move
# blocks around would fire and it would move right, which would
# then allow us to shrink it more easily.
n = (data.draw_bits(16) << 16) | data.draw_bits(16)
if n == MAX_INT:
return (POISON,)
else:
return (None,)
LOTS = 10 ** 6
TEST_SETTINGS = settings(
database=None, suppress_health_check=HealthCheck.all(), max_examples=LOTS,
deadline=None, timeout=unlimited
)
@pytest.mark.parametrize('size', [2, 5, 10])
@pytest.mark.parametrize('seed', [0, 15993493061449915028])
def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
@st.composite
def problem(draw):
b = draw(st.binary(min_size=1, max_size=8))
m = int_from_bytes(b) * 256
assume(m > 0)
marker = draw(st.binary(max_size=8))
bound = draw(st.integers(0, m - 1))
return (b, marker, bound)
base_settings = settings(
database=None,
deadline=None,
suppress_health_check=HealthCheck.all(),
max_examples=10,
verbosity=Verbosity.normal,
phases=(Phase.explicit, Phase.generate),
)
@example((b"\x10\x00\x00\x00\x00\x00", b"", 2861143707951135))
@example((b"\x05Cn", b"%\x1b\xa0\xfa", 12394667))
@example((b"\x179 f", b"\xf5|", 24300326997))
@example((b"\x05*\xf5\xe5\nh", b"", 1076887621690235))
@example((b"=", b"", 2508))
@example((b"\x01\x00", b"", 20048))
@example((b"\x01", b"", 0))
@example((b"\x02", b"", 258))
@example((b"\x08", b"", 1792))
respond_yaml,
respond_dot,
respond_text,
respond_cytoscape,
respond_graphml,
)
from tests.resotocore.hypothesis_extension import (
json_array_gen,
json_simple_element_gen,
node_gen,
graph_stream,
)
@given(json_array_gen)
@settings(max_examples=20, suppress_health_check=HealthCheck.all())
@pytest.mark.asyncio
async def test_json(elements: List[JsonElement]) -> None:
async with stream.iterate(elements).stream() as streamer:
result = ""
async for elem in respond_json(streamer):
result += elem
assert json.loads(result) == elements
@given(json_array_gen)
@settings(max_examples=20, suppress_health_check=HealthCheck.all())
@pytest.mark.asyncio
async def test_ndjson(elements: List[JsonElement]) -> None:
async with stream.iterate(elements).stream() as streamer:
result = []
def reduce_with_hypothesis_base(experiment, name, suppress_intervals):
experiment = EXPERIMENTS[experiment]
base_predicate = experiment.calculate_error_predicate(info(experiment, name))
generated = generate(experiment, name)
classified = base_predicate(generated)
assert classified == Classification.INTERESTING, (classified, base_predicate)
results, predicate = tracking_predicate(base_predicate)
generation_stats = {
c: ExampleStatistics()
for c in Status
}
input_to_outputs = []
def test_function(data):
if suppress_intervals:
start = data.start_example
stop = data.stop_example
def start_example(label):
if data.depth >= 0:
data.depth += 1
if data.depth > data.max_depth:
data.max_depth = data.depth
else:
start(label)
def stop_example(discard=False):
if data.depth >= 1:
data.depth -= 1
else:
stop(discard)
data.start_example = start_example
data.stop_example = stop_example
generation_start = time.monotonic()
try:
try:
source = data.draw(experiment.generator)
except UnsatisfiedAssumption:
data.mark_invalid()
finally:
generation_time = time.monotonic() - generation_start
result = predicate(source)
input_to_outputs.append((encode_failure(data.buffer).decode('ascii'), source, result.name))
if trace_memory_usage:
display_top(tracemalloc.take_snapshot())
if result == Classification.INTERESTING:
data.mark_interesting()
elif result in (Classification.INVALIDCHEAP, Classification.INVALIDEXPENSIVE):
data.mark_invalid()
finally:
generation_stats[data.status].record(size=len(data.buffer), runtime=generation_time)
buffer = raw_buffer(experiment, name)
runner = eng.ConjectureRunner(
test_function,
settings=settings(
database=None,
max_examples=1,
suppress_health_check=HealthCheck.all(),
deadline=None,
verbosity=Verbosity.debug,
buffer_size=BUFFER_SIZE,
),
random=Random(int.from_bytes(hashlib.sha1(buffer).digest(), "big")),
)
def debug_data(data):
runner.debug(
f"DATA {hashlib.sha1(data.buffer).hexdigest()[:8]}: {len(data.buffer)} bytes, {data.status.name}"
)
runner.debug_data = debug_data
runner.cached_test_function(buffer)
assert runner.interesting_examples
results.start()
runner.shrink_interesting_examples()
results.finish()
v, = runner.interesting_examples.values()
return {
"final": {
"buffer": base64.b64encode(v.buffer).decode("ascii"),
"generated": buffer_to_value(experiment, v.buffer),
},
"reductionstats": attr.asdict(results),
"input_to_outputs": input_to_outputs,
"generationstats": {k.name: attr.asdict(v) for k, v in generation_stats.items()},
}
def learner_for(strategy):
"""Returns an LStar learner that predicts whether a buffer
corresponds to a discard free choice sequence leading to
a valid value for this strategy."""
try:
return LEARNERS[strategy]
except KeyError:
pass
def test_function(data):
try:
data.draw(strategy)
except UnsatisfiedAssumption:
data.mark_invalid()
data.mark_interesting()
runner = ConjectureRunner(
test_function,
settings=settings(
database=None,
verbosity=Verbosity.quiet,
suppress_health_check=HealthCheck.all(),
),
random=Random(0),
ignore_limits=True,
)
def predicate(s):
result = runner.cached_test_function(s)
if result.status < Status.VALID:
return False
if result.has_discards:
return False
return result.buffer == s
learner = LStar(predicate)
runner.run()
(v, ) = runner.interesting_examples.values()
# We make sure the learner has properly learned small examples.
# This is all fairly ad hoc but is mostly designed to get it
# to understand what the smallest example is and avoid any
# loops at the beginning of the DFA that don't really exist.
learner.learn(v.buffer)
for n in [1, 2, 3]:
for _ in range(5):
learner.learn(uniform(runner.random, n) + v.buffer)
prev = -1
while learner.generation != prev:
prev = learner.generation
for _ in range(10):
s = uniform(runner.random, len(v.buffer)) + bytes(BUFFER_SIZE)
learner.learn(s)
data = runner.cached_test_function(s)
if data.status >= Status.VALID:
learner.learn(data.buffer)
LEARNERS[strategy] = learner
return learner
from hypothesis import given, example
from hypothesis import HealthCheck, settings
from hypothesis.strategies import binary,\
dictionaries, integers, lists, one_of, recursive, text
from homework03 import encode, decode
settings.suppres_health_check = HealthCheck.all()
"""http://www.bittorrent.org/beps/bep_0003.html"""
SIMPLE_TYPES = one_of(integers(), binary())
# Strings are length-prefixed base ten followed by a colon and the string.
# For example 4:spam corresponds to 'spam'.
@given(text())
@example('spam')
def test_string(s):
assert decode(encode(s)) == bytes(s, 'utf-8')
@given(binary())
def test_bytestring(s):
assert decode(encode(s)) == s
# Integers are represented by an 'i' followed by the number in base 10 followed by an 'e'.
# For example i3e corresponds to 3 and i-3e corresponds to -3.
# Integers have no size limitation. i-0e is invalid.
# All encodings with a leading zero, such as i03e, are invalid,
# other than i0e, which of course corresponds to 0.
import bz2
import gzip
import unittest
from hypothesis import HealthCheck, given, settings, strategies as st
no_health_checks = settings(suppress_health_check=HealthCheck.all())
class TestBz2(unittest.TestCase):
@given(payload=st.binary(), compresslevel=st.integers(1, 9))
def test_bz2_round_trip(self, payload, compresslevel):
result = bz2.decompress(
bz2.compress(payload, compresslevel=compresslevel))
self.assertEqual(payload, result)
@given(payloads=st.lists(st.binary()), compresslevel=st.integers(1, 9))
def test_bz2_incremental_compress_eq_oneshot(self, payloads,
compresslevel):
c = bz2.BZ2Compressor(compresslevel)
compressed = b"".join(c.compress(p) for p in payloads) + c.flush()
self.assertEqual(compressed,
bz2.compress(b"".join(payloads), compresslevel))
@no_health_checks
@given(payload=st.binary(),
compresslevel=st.integers(1, 9),
data=st.data())
def test_bz2_incremental_decompress_eq_oneshot(self, payload,
compresslevel, data):
compressed = bz2.compress(payload, compresslevel=compresslevel)
assert minimal(st.floats(), lambda x: x > 1) == 2.0
assert minimal(st.floats(), lambda x: x > 0) == 1.0
@pytest.mark.parametrize("n", [1, 2, 3, 8, 10])
def test_can_shrink_in_variable_sized_context(n):
x = minimal(st.lists(st.floats(), min_size=n), any)
assert len(x) == n
assert x.count(0.0) == n - 1
assert 1 in x
@example(1.7976931348623157e308)
@example(1.5)
@given(st.floats(min_value=0, allow_infinity=False, allow_nan=False))
@settings(deadline=None, suppress_health_check=HealthCheck.all())
def test_shrinks_downwards_to_integers(f):
g = minimal(st.floats(), lambda x: x >= f, settings(verbosity=Verbosity.quiet))
assert g == ceil(f)
@example(1)
@given(st.integers(1, 2 ** 16 - 1))
@settings(deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10)
def test_shrinks_downwards_to_integers_when_fractional(b):
g = minimal(
st.floats(),
lambda x: assume((0 < x < (2 ** 53)) and int(x) != x) and x >= b,
settings=settings(verbosity=Verbosity.quiet, max_examples=10 ** 6),
)
assert g == b + 0.5
def normalize(
base_name,
test_function,
*,
required_successes=100,
allowed_to_update=False,
max_dfas=10,
):
"""Attempt to ensure that this test function successfully normalizes - i.e.
whenever it declares a test case to be interesting, we are able
to shrink that to the same interesting test case (which logically should
be the shortlex minimal interesting test case, though we may not be able
to detect if it is).
Will run until we have seen ``required_successes`` many interesting test
cases in a row normalize to the same value.
If ``allowed_to_update`` is True, whenever we fail to normalize we will
learn a new DFA-based shrink pass that allows us to make progress. Any
learned DFAs will be written back into the learned DFA file at the end
of this function. If ``allowed_to_update`` is False, this will raise an
error as soon as it encounters a failure to normalize.
Additionally, if more than ``max_dfas` DFAs are required to normalize
this test function, this function will raise an error - it's essentially
designed for small patches that other shrink passes don't cover, and
if it's learning too many patches then you need a better shrink pass
than this can provide.
"""
# Need import inside the function to avoid circular imports
from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
runner = ConjectureRunner(
test_function,
settings=settings(database=None, suppress_health_check=HealthCheck.all()),
ignore_limits=True,
)
seen = set()
dfas_added = 0
found_interesting = False
consecutive_successes = 0
failures_to_find_interesting = 0
while consecutive_successes < required_successes:
attempt = runner.cached_test_function(b"", extend=BUFFER_SIZE)
if attempt.status < Status.INTERESTING:
failures_to_find_interesting += 1
assert (
found_interesting or failures_to_find_interesting <= 1000
), "Test function seems to have no interesting test cases"
continue
found_interesting = True
target = attempt.interesting_origin
def shrinking_predicate(d):
return d.status == Status.INTERESTING and d.interesting_origin == target
if target not in seen:
seen.add(target)
runner.shrink(attempt, shrinking_predicate)
continue
previous = fully_shrink(
runner, runner.interesting_examples[target], shrinking_predicate
)
current = fully_shrink(runner, attempt, shrinking_predicate)
if current.buffer == previous.buffer:
consecutive_successes += 1
continue
consecutive_successes = 0
if not allowed_to_update:
raise FailedToNormalise(
"Shrinker failed to normalize %r to %r and we are not allowed to learn new DFAs."
% (previous.buffer, current.buffer)
)
if dfas_added >= max_dfas:
raise FailedToNormalise(
"Test function is too hard to learn: Added %d DFAs and still not done."
% (dfas_added,)
)
dfas_added += 1
new_dfa = learn_a_new_dfa(
runner, previous.buffer, current.buffer, shrinking_predicate
)
name = (
base_name
+ "-"
+ hashlib.sha256(repr(new_dfa).encode("utf-8")).hexdigest()[:10]
)
# If there is a name collision this DFA should already be being
# used for shrinking, so we should have already been able to shrink
# v further.
assert name not in SHRINKING_DFAS
SHRINKING_DFAS[name] = new_dfa
if dfas_added > 0:
# We've learned one or more DFAs in the course of normalising, so now
# we update the file to record those for posterity.
update_learned_dfas()
assert minimal(st.floats(), lambda x: x > 1) == 2.0
assert minimal(st.floats(), lambda x: x > 0) == 1.0
@pytest.mark.parametrize("n", [1, 2, 3, 8, 10])
def test_can_shrink_in_variable_sized_context(n):
x = minimal(st.lists(st.floats(), min_size=n), any)
assert len(x) == n
assert x.count(0.0) == n - 1
assert 1 in x
@example(1.7976931348623157e308)
@example(1.5)
@given(st.floats(min_value=0, allow_infinity=False, allow_nan=False))
@settings(deadline=None, suppress_health_check=HealthCheck.all())
def test_shrinks_downwards_to_integers(f):
g = minimal(st.floats(), lambda x: x >= f,
settings(verbosity=Verbosity.quiet))
assert g == ceil(f)
@example(1)
@given(st.integers(1, 2**16 - 1))
@settings(deadline=None,
suppress_health_check=HealthCheck.all(),
max_examples=10)
def test_shrinks_downwards_to_integers_when_fractional(b):
g = minimal(
st.floats(),
lambda x: assume((0 < x < (2**53)) and int(x) != x) and x >= b,
index_to_add_word = draw(st.integers(min_value=0, max_value=len(text) - 1))
new_word = vocabulary[draw(st.integers(min_value=0, max_value=vocab_size))]
text.append(text[index_to_add_word] + " " + new_word)
return (text, vocabulary)
# TODO: Add a set of tests for passing in instantiated classes
# TODO: Test that DocVectorizer transform preserves column order and size on new data
@given(generate_test_text_info())
@settings(
deadline=None,
suppress_health_check=[HealthCheck(3)],
max_examples=DEFAULT_MAX_EXAMPLES,
)
@example((test_text_example, None))
def test_joint_nobasistransformer(test_text_info):
test_text = test_text_info[0]
model = JointWordDocVectorizer(feature_basis_converter=None,
token_contractor_kwds={"min_score": 8})
result = model.fit_transform(test_text)
assert isinstance(result, scipy.sparse.csr_matrix)
if test_text == test_text_example:
assert result.shape == (12, 7)
else:
assert result.shape[0] == model.n_words_ + len(test_text)
assert result.shape[1] == model.n_words_
class TestSummaries(TestCase):
def assertDictAlmostEqual(self, d1, d2, msg=None, places=7):
# check if both inputs are dicts
self.assertIsInstance(d1, dict, 'First argument is not a dictionary')
self.assertIsInstance(d2, dict, 'Second argument is not a dictionary')
# check if both inputs have the same keys
self.assertEqual(d1.keys(), d2.keys())
# check each key
for key, value in d1.items():
if isinstance(value, dict):
self.assertDictAlmostEqual(d1[key], d2[key], msg=msg)
else:
self.assertAlmostEqual(d1[key],
d2[key],
places=places,
msg=msg)
def assertSummaryIsValid(self,
loc_df,
gul_inputs,
exp_summary,
perils_expected=None):
cov_types = ['buildings', 'other', 'bi', 'contents']
lookup_status = [
'success', 'fail', 'nomatch', 'fail_ap', 'fail_v', 'notatrisk'
]
loc_rename_cols = {
'bitiv': 'bi',
'buildingtiv': 'buildings',
'contentstiv': 'contents',
'othertiv': 'other'
}
# Check each returned peril
for peril in perils_expected:
peril_summary = exp_summary[peril]
# Check the 'All' section
supported_tivs = loc_df[[
'buildingtiv', 'othertiv', 'bitiv', 'contentstiv'
]].sum(0).rename(loc_rename_cols)
self.assertAlmostEqual(supported_tivs.sum(0),
peril_summary['all']['tiv'])
for cov in cov_types:
self.assertAlmostEqual(
supported_tivs[cov],
peril_summary['all']['tiv_by_coverage'][cov])
# Check each lookup status
peril_expected = gul_inputs[gul_inputs.peril_id == peril]
for status in lookup_status:
peril_status = peril_expected[peril_expected.status == status]
self.assertAlmostEqual(peril_status.tiv.sum(),
peril_summary[status]['tiv'])
self.assertEqual(len(peril_status.loc_id.unique()),
peril_summary[status]['number_of_locations'])
for cov in cov_types:
cov_type_id = SUPPORTED_COVERAGE_TYPES[cov]['id']
cov_type_tiv = peril_status[peril_status.coverage_type_id
== cov_type_id].tiv.sum()
self.assertAlmostEqual(
cov_type_tiv,
peril_summary[status]['tiv_by_coverage'][cov])
# Check 'noreturn' status
tiv_returned = sum([
s[1]['tiv'] for s in peril_summary.items()
if s[0] in lookup_status
])
self.assertAlmostEqual(peril_summary['all']['tiv'] - tiv_returned,
peril_summary['noreturn']['tiv'])
for cov in cov_types:
cov_tiv_returned = sum([
s[1]['tiv_by_coverage'][cov]
for s in peril_summary.items() if s[0] in lookup_status
])
self.assertAlmostEqual(
peril_summary['all']['tiv_by_coverage'][cov] -
cov_tiv_returned,
peril_summary['noreturn']['tiv_by_coverage'][cov])
@given(st.data())
@settings(max_examples=20, deadline=None)
def test_single_peril__totals_correct(self, data):
# Shared Values between Loc / keys
loc_size = data.draw(integers(10, 20))
supported_cov = data.draw(
st.lists(integers(1, 4), unique=True, min_size=1, max_size=4))
perils = 'WTC'
# Create Mock keys_df
keys_data = list()
for i in supported_cov:
keys_data += data.draw(
keys(size=loc_size,
from_peril_ids=just(perils),
from_coverage_type_ids=just(i),
from_area_peril_ids=just(1),
from_vulnerability_ids=just(1),
from_messages=just('str')))
keys_df = pd.DataFrame.from_dict(keys_data)
# Create Mock location_df
loc_df = pd.DataFrame.from_dict(
data.draw(
min_source_exposure(size=loc_size,
from_location_perils_covered=just(perils),
from_location_perils=just(perils),
from_building_tivs=integers(1000, 1000000),
from_other_tivs=integers(100, 100000),
from_contents_tivs=integers(50, 50000),
from_bi_tivs=integers(20, 20000))))
loc_df['loc_id'] = get_ids(loc_df,
['portnumber', 'accnumber', 'locnumber'])
# Run exposure_summary
exp_summary = get_exposure_summary(
exposure_df=loc_df,
keys_df=keys_df,
)
# Run Gul Proccessing
gul_inputs = get_gul_input_items(loc_df, keys_df)
gul_inputs = gul_inputs[gul_inputs['status'].isin(
OASIS_KEYS_STATUS_MODELLED)]
# Fetch expected TIVS
tiv_portfolio = loc_df[[
'buildingtiv', 'othertiv', 'bitiv', 'contentstiv'
]].sum(1).sum(0)
tiv_modelled = gul_inputs['tiv'].sum()
tiv_not_modelled = tiv_portfolio - tiv_modelled
# Check TIV values
self.assertEqual(tiv_portfolio,
exp_summary['total']['portfolio']['tiv'])
self.assertEqual(tiv_modelled, exp_summary['total']['modelled']['tiv'])
self.assertEqual(tiv_not_modelled,
exp_summary['total']['not-modelled']['tiv'])
# Check number of locs
self.assertEqual(
len(loc_df),
exp_summary['total']['portfolio']['number_of_locations'])
self.assertEqual(
len(gul_inputs.loc_id.unique()),
exp_summary['total']['modelled']['number_of_locations'])
# Check number of not-modelled
# WARNING: current assumption is that all cov types must be covered to be modelled
#moddeled = 0
#moddeld_loc_ids = gul_inputs[gul_inputs['status'] == 'success'].loc_id.unique()
#for loc_id in moddeld_loc_ids:
# if len(gul_inputs[gul_inputs.loc_id == loc_id].coverage_type_id.unique()) == 4:
# moddeled+=1
#self.assertEqual(len(loc_df) - moddeled, exp_summary['total']['not-modelled']['number_of_locations'])
@given(st.data())
@settings(max_examples=10, deadline=None)
def test_multi_perils__single_covarage(self, data):
loc_size = data.draw(integers(10, 20))
supported_cov = data.draw(integers(1, 4))
perils = data.draw(
st.lists(st.text(alphabet=(string.ascii_letters + string.digits),
min_size=2,
max_size=6),
min_size=2,
max_size=6,
unique=True))
# Create Mock keys_df
keys_data = list()
for p in perils:
keys_data += data.draw(
keys(size=loc_size,
from_peril_ids=just(p),
from_coverage_type_ids=just(supported_cov),
from_area_peril_ids=just(1),
from_vulnerability_ids=just(1),
from_messages=just('str')))
keys_df = pd.DataFrame.from_dict(keys_data)
perils_returned = keys_df.peril_id.unique().tolist()
# Create Mock location_df
perils_covered = ';'.join(perils)
loc_df = pd.DataFrame.from_dict(
data.draw(
min_source_exposure(
size=loc_size,
from_location_perils_covered=just(perils_covered),
from_location_perils=just(perils_covered),
from_building_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_other_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_contents_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_bi_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)))))
loc_df['loc_id'] = get_ids(loc_df,
['portnumber', 'accnumber', 'locnumber'])
# Run Summary output check
self.assertSummaryIsValid(
loc_df, get_gul_input_items(loc_df, keys_df),
get_exposure_summary(exposure_df=loc_df, keys_df=keys_df),
perils_returned)
@given(st.data())
@settings(max_examples=10,
deadline=None,
suppress_health_check=HealthCheck.all())
def test_multi_perils__multi_covarage(self, data):
loc_size = data.draw(integers(10, 20))
supported_cov = data.draw(
st.lists(integers(1, 4), unique=True, min_size=1, max_size=4))
perils = data.draw(
st.lists(st.text(alphabet=(string.ascii_letters + string.digits),
min_size=2,
max_size=6),
min_size=2,
max_size=6,
unique=True))
# Create Mock keys_df
keys_data = list()
for c in supported_cov:
for p in perils:
keys_data += data.draw(
keys(size=loc_size,
from_peril_ids=just(p),
from_coverage_type_ids=just(c),
from_area_peril_ids=just(1),
from_vulnerability_ids=just(1),
from_messages=just('str')))
keys_df = pd.DataFrame.from_dict(keys_data)
perils_returned = keys_df.peril_id.unique().tolist()
# Create Mock location_df
loc_df = pd.DataFrame.from_dict(
data.draw(
min_source_exposure(
size=loc_size,
from_location_perils_covered=st.sampled_from(perils),
from_location_perils=st.sampled_from(perils),
from_building_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_other_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_contents_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_bi_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)))))
loc_df['loc_id'] = get_ids(loc_df,
['portnumber', 'accnumber', 'locnumber'])
# Run Summary output check
exp_summary = get_exposure_summary(exposure_df=loc_df, keys_df=keys_df)
gul_inputs = get_gul_input_items(loc_df, keys_df)
self.assertSummaryIsValid(loc_df, gul_inputs, exp_summary,
perils_returned)
@given(st.data())
@settings(max_examples=10, deadline=None)
def test_summary_file_written(self, data):
loc_size = data.draw(integers(10, 20))
# Create Mock keys_data
keys_data = data.draw(
keys(size=loc_size,
from_area_peril_ids=just(1),
from_vulnerability_ids=just(1),
from_messages=just('str')))
# Create Mock location_data
loc_data = data.draw(
min_source_exposure(
size=loc_size,
from_building_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_other_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_contents_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000)),
from_bi_tivs=st.one_of(st.floats(1.0, 1000.0),
st.integers(1, 1000))))
# Prepare arguments for write_exposure_summary
with TemporaryDirectory() as tmp_dir:
keys_fp = os.path.join(tmp_dir, 'keys.csv')
successes = [k for k in keys_data if k['status'] in ['success']]
keys_errors_fp = os.path.join(tmp_dir, 'keys_errors.csv')
nonsuccesses = [
k for k in keys_data if k['status'] not in ['success']
]
write_keys_files(keys=successes,
keys_file_path=keys_fp,
keys_errors=nonsuccesses,
keys_errors_file_path=keys_errors_fp)
location_fp = os.path.join(tmp_dir, 'location.csv')
write_source_files(exposure=loc_data, exposure_fp=location_fp)
location_df = get_location_df(location_fp)
exposure_summary_fp = write_exposure_summary(
tmp_dir, location_df, keys_fp, keys_errors_fp,
get_default_exposure_profile())
self.assertTrue(os.path.isfile(exposure_summary_fp))
with open(exposure_summary_fp) as f:
data = json.load(f)
loc_df = pd.DataFrame.from_dict(loc_data)
loc_df['loc_id'] = get_ids(
loc_df, ['portnumber', 'accnumber', 'locnumber'])
keys_df = pd.DataFrame.from_dict(keys_data)
exp_summary = get_exposure_summary(loc_df, keys_df)
self.assertDictAlmostEqual(data, exp_summary)
def weights(draw):
parts = draw(st.lists(st.integers()))
parts.reverse()
base = Fraction(1, 1)
for p in parts:
base = Fraction(1) / (1 + base)
return base
@example([Fraction(1, 3), Fraction(1, 3), Fraction(1, 3)])
@example([Fraction(1, 1), Fraction(1, 2)])
@example([Fraction(1, 2), Fraction(4, 10)])
@example([Fraction(1, 1), Fraction(3, 5), Fraction(1, 1)])
@example([Fraction(2, 257), Fraction(2, 5), Fraction(1, 11)])
@settings(
deadline=None, suppress_health_check=HealthCheck.all(),
max_examples=0 if IN_COVERAGE_TESTS else settings.default.max_examples,
)
@given(st.lists(weights(), min_size=1))
def test_sampler_distribution(weights):
total = sum(weights)
n = len(weights)
assume(total > 0)
probabilities = [w / total for w in weights]
sampler = cu.Sampler(weights)
calculated = [Fraction(0)] * n
for base, alternate, p_alternate in sampler.table:
result = []
count = 0
last = None
for line in lines:
if line:
if last is not None and last != line:
result.append(f'{last},{count}')
count = 0
last = line
count += 1
if last:
result.append(f'{last},{count}')
return '\n'.join(result) + '\n'
@given(inputs())
@settings(database=ExampleDatabase(':memory:'), max_examples=100 * int(os.environ.get('TEST_FACTOR', 1)), suppress_health_check=HealthCheck.all()) # type: ignore
def test_props(args):
csv = args
result = expected(csv)
assert result == run(csv, 'bsv | bcounteach | bschema *,i64:a | csv')
def test_basic():
stdin = """
a
a
a
b
b
a
"""
stdout = """
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(
f,
random=Random(0),
settings=settings(
suppress_health_check=HealthCheck.all(),
phases=(Phase.shrink,),
database=None,
verbosity=Verbosity.debug,
),
database_key=None,
)
runner.cached_test_function(blob)
assert runner.interesting_examples
(v,) = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.fixate_shrink_passes(["minimize_individual_blocks"])
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed]
)
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
a coverage-guided fuzzer I'm working on.
"""
import hypothesmith
from hypothesis import HealthCheck, given, settings, strategies as st
import black
# This test uses the Hypothesis and Hypothesmith libraries to generate random
# syntatically-valid Python source code and run Black in odd modes.
@settings(
max_examples=1000, # roughly 1k tests/minute, or half that under coverage
derandomize=True, # deterministic mode to avoid CI flakiness
deadline=None, # ignore Hypothesis' health checks; we already know that
suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy.
)
@given(
# Note that while Hypothesmith might generate code unlike that written by
# humans, it's a general test that should pass for any *valid* source code.
# (so e.g. running it against code scraped of the internet might also help)
src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(),
# Using randomly-varied modes helps us to exercise less common code paths.
mode=st.builds(
black.FileMode,
line_length=st.just(88) | st.integers(0, 200),
string_normalization=st.booleans(),
is_pyi=st.booleans(),
),
)
def test_idempotent_any_syntatically_valid_python(
import pandas as pd
from pyrle import Rle
import numpy as np
strandedness = [False, "same", "opposite"]
methods = ["join", "intersection", "overlap"]
# strandedness = strandedness[1:2]
# methods = methods[1:2]
coverage_cmd = "Rscript --vanilla tests/compute_coverage.R {} {}"
@pytest.mark.r
@settings(max_examples=max_examples, deadline=deadline, timeout=unlimited, suppress_health_check=HealthCheck.all())
@given(df=dfs_min_single_chromosome())
def test_coverage(df):
print("---" * 10)
p = pr.PyRanges(df)
print("pyranges\n", p)
c = p.coverage()["chr1"]
result_df = None
with tempfile.TemporaryDirectory() as temp_dir:
f1 = "{}/f1.txt".format(temp_dir)
outfile = "{}/result.txt".format(temp_dir)
R_df = df
R_df.End = R_df.End - 1
def shrink(filename, sizes):
with open(filename, 'rb') as i:
initial_data = ConjectureData.for_buffer(i.read())
printing_to_stdout = False
def data_info(data):
name = hashlib.sha1(data.buffer).hexdigest()[:8]
input_length = len(data.buffer)
output_length = data.extra_information.generated_length
return {
"name": name, "input_length": input_length, "output_length": output_length,
"interesting": data.status == Status.INTERESTING
}
if sizes is not None:
if sizes == "-":
writer = sys.stdout
printing_to_stdout = True
else:
writer = open(sizes, 'w')
def log_data(data):
if data.status >= Status.VALID and hasattr(data.extra_information, 'generated_length'):
writer.write(json.dumps(data_info(data)) + "\n")
writer.flush()
else:
writer = None
def log_data(data):
pass
initial = mktemp(suffix='.c')
gen(initial_data, initial)
errtype, gcc, opt = interesting_reason(initial, printing=not printing_to_stdout)
if errtype == 'error':
def test_function(data):
with tempfile.TemporaryDirectory() as d:
prog = os.path.join(d, 'test.c')
try:
gen(data, prog)
data.extra_information.generated_length = os.stat(prog).st_size
run_gcc(gcc, opt, prog)
except subprocess.TimeoutExpired:
return
except subprocess.SubprocessError:
name = hashlib.sha1(data.buffer).hexdigest()[:8]
data.unique_name = name
data.mark_interesting()
finally:
log_data(data)
else:
assert errtype == "differs"
def test_function(data):
with tempfile.TemporaryDirectory() as d:
prog = os.path.join(d, 'test.c')
try:
gen(data, prog)
data.generated_length = os.stat(prog).st_size
if run_gcc(gcc, opt, prog) != run_gcc(
'/opt/compiler-explorer/gcc-8.2.0/bin/gcc', '-O0', prog
):
name = hashlib.sha1(data.buffer).hexdigest()[:8]
data.unique_name = name
data.mark_interesting()
except subprocess.SubprocessError:
pass
finally:
log_data(data)
eng.MAX_SHRINKS = 10 ** 6
with open(filename, 'rb') as i:
buffer = i.read()
# tracemalloc.start()
runner = eng.ConjectureRunner(test_function, settings=settings(
database=None,
max_examples=1000,
timeout=unlimited, suppress_health_check=HealthCheck.all(),
deadline=None,
verbosity=Verbosity.quiet if printing_to_stdout else Verbosity.debug,
buffer_size=BUFFER_SIZE
), random=Random(int.from_bytes(hashlib.sha1(buffer).digest(), 'big')))
class FakeTree(object):
def add(self, data):
pass
def rewrite(self, buffer):
return (buffer, None)
def generate_novel_prefix(self, random):
return b''
is_exhausted = False
def uncached_test_function(buffer):
data = ConjectureData.for_buffer(buffer)
runner.test_function(data)
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.statistics('lineno')
print("[ Top 10 ]")
for stat in top_stats[:10]:
print(stat)
return data.as_result()
# runner.tree = FakeTree()
# runner.cached_test_function = uncached_test_function
# runner.target_selector.add = lambda x: None
runner.test_function(ConjectureData.for_buffer(buffer))
assert runner.interesting_examples
runner.debug_data = lambda data: runner.debug(
f"DATA({getattr(data, 'unique_name', None)}): {len(data.buffer)} bytes, {data.status.name}, {data.interesting_origin}"
)
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(
v, lambda d: d.status == Status.INTERESTING and d.interesting_origin == v.interesting_origin
)
initial_calls = runner.call_count
initial_valid = runner.valid_examples
start = time.monotonic()
shrinker.shrink()
end = time.monotonic()
final = data_info(shrinker.shrink_target)
final["bytes"] = encode_failure(shrinker.shrink_target.buffer).decode('ascii')
if writer is not None:
writer.write(json.dumps({
"runtime": end - start,
"calls": runner.call_count - initial_calls,
"valid": runner.valid_examples - initial_valid,
"final": final,
}) + "\n")
result = runner.interesting_examples[v.interesting_origin]
runner.debug_data(result)
gen(ConjectureData.for_buffer(result.buffer), 'shrunk.c')
if writer is not None:
writer.close()
def csr_slow(divider=2):
dft = settings.default
return settings(dft,
deadline=None,
suppress_health_check=HealthCheck.all(),
max_examples=dft.max_examples // divider)
# obtain one at https://mozilla.org/MPL/2.0/.
#
# END HEADER
from __future__ import absolute_import, division, print_function
import pytest
import hypothesis.strategies as st
from hypothesis import HealthCheck, Verbosity, find, given, reject, settings
from hypothesis.errors import NoSuchExample
from tests.common.utils import no_shrink
@pytest.mark.parametrize("strat", [st.text(min_size=5)])
@settings(phases=no_shrink, deadline=None, suppress_health_check=HealthCheck.all())
@given(st.data())
def test_explore_arbitrary_function(strat, data):
cache = {}
def predicate(x):
try:
return cache[x]
except KeyError:
return cache.setdefault(x, data.draw(st.booleans(), label=repr(x)))
try:
find(
strat,
predicate,
settings=settings(
ls = minimal(
st.lists(st.recursive(
st.booleans(),
lambda x: st.lists(
x, min_size=2 * (target - 1), max_size=2 * target
).map(tuple),
max_leaves=2 * target - 1)),
lambda x: len(set(x)) >= target,
timeout_after=None
)
assert len(ls) == target
@given(st.randoms())
@settings(
max_examples=50, phases=no_shrink, suppress_health_check=HealthCheck.all(),
deadline=None
)
def test_can_use_recursive_data_in_sets(rnd):
nested_sets = st.recursive(st.booleans(), st.frozensets, max_leaves=3)
find_any(nested_sets, random=rnd)
def flatten(x):
if isinstance(x, bool):
return frozenset((x,))
else:
result = frozenset()
for t in x:
result |= flatten(t)
if len(result) == 2:
break
from hypothesis.internal.conjecture.engine import ConjectureRunner
@st.composite
def problem(draw):
b = hbytes(draw(st.binary(min_size=1, max_size=8)))
m = int_from_bytes(b) * 256
assume(m > 0)
marker = draw(st.binary(max_size=8))
bound = draw(st.integers(0, m - 1))
return (b, marker, bound)
base_settings = settings(
database=None,
deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10,
verbosity=Verbosity.normal, timeout=unlimited,
phases=(
Phase.explicit,
Phase.generate
)
)
@example((b'\x10\x00\x00\x00\x00\x00', b'', 2861143707951135))
@example((b'\x05Cn', b'%\x1b\xa0\xfa', 12394667))
@example((b'\x179 f', b'\xf5|', 24300326997))
@example((b'\x05*\xf5\xe5\nh', b'', 1076887621690235))
@example((b'=', b'', 2508))
@example((b'\x01\x00', b'', 20048))
@example((b'\x01', b'', 0))
def main(filename):
target = os.path.splitext(filename)[0] + ".json"
random = Random(
int.from_bytes(
hashlib.sha1(os.path.basename(filename).encode("utf-8")).digest(),
"big"))
results = []
with open(filename, "r") as i:
source = i.read()
compiled = compile(source, filename, "exec")
for _ in trange(100):
seed = random.getrandbits(64)
namespace = {}
exec(compiled, namespace)
test = namespace["test"]
assert getattr(test, "is_hypothesis_test", False)
base_function = test.hypothesis.inner_test
stats = {
"seed": seed,
"evaluations": 0,
}
def record(kwargs, interesting):
if interesting:
kwargs = {name: repr(value) for name, value in kwargs.items()}
if "original" not in stats:
stats["original"] = kwargs
stats["shrunk"] = kwargs
if "original" in stats:
stats["evaluations"] += 1
@proxies(base_function)
def replacement_function(**kwargs):
try:
base_function(**kwargs)
record(kwargs, False)
except Exception:
record(kwargs, True)
raise
test.hypothesis.inner_test = replacement_function
test = with_seed(seed)(settings(
database=None,
suppress_health_check=HealthCheck.all(),
max_examples=10**6,
phases=[Phase.generate, Phase.shrink],
verbosity=Verbosity.quiet,
)(test))
try:
test()
except Exception:
if "original" not in stats:
raise
results.append(stats)
with open(target, "w") as o:
o.write(json.dumps(
results,
sort_keys=True,
indent=4,
))
def test_runner(schema_path):
schema = loaders.from_path(schema_path, validate_schema=False)
runner = from_schema(
schema,
dry_run=True,
count_operations=False,
hypothesis_settings=hypothesis.settings(
max_examples=1, suppress_health_check=HealthCheck.all(), phases=[Phase.explicit, Phase.generate]
),
)
schema_id = get_id(schema_path)
def check_xfailed(ev) -> bool:
if schema_id in XFAILING and ev.current_operation in XFAILING[schema_id]:
if ev.result.errors:
message = XFAILING[schema_id][ev.current_operation]
# If is is failed for some other reason, then an assertion will be risen
return any(message in err.exception_with_traceback for err in ev.result.errors)
pytest.fail("Expected a failure")
return False
def is_unsatisfiable(text):
return "Unable to satisfy schema parameters for this API operation" in text
def check_flaky(ev) -> bool:
if schema_id in FLAKY_SCHEMAS and ev.current_operation in FLAKY_SCHEMAS[schema_id]:
if ev.result.errors:
# NOTE. There could be other errors if the "Unsatisfiable" case wasn't triggered.
# Could be added to expected errors later
return any(is_unsatisfiable(err.exception_with_traceback) for err in ev.result.errors)
return False
def check_unsatisfiable(ev):
# In some cases Schemathesis can't generate data - either due to a contradiction within the schema
if schema_id in UNSATISFIABLE_SCHEMAS and ev.current_operation in UNSATISFIABLE_SCHEMAS[schema_id]:
exception = ev.result.errors[0].exception
if (
"Unable to satisfy schema parameters for this API operation" not in exception
and "Cannot create non-empty lists with elements" not in exception
and "Cannot create a collection of " not in exception
):
pytest.fail(f"Expected unsatisfiable, but there is a different error: {exception}")
return True
return False
def check_recursive_references(ev):
if schema_id in RECURSIVE_REFERENCES and ev.current_operation in RECURSIVE_REFERENCES[schema_id]:
for err in ev.result.errors:
if RECURSIVE_REFERENCE_ERROR_MESSAGE in err.exception_with_traceback:
# It is OK
return True
# These errors may be triggered not every time
return False
def check_not_parsable(ev):
return schema_id in NOT_PARSABLE_SCHEMAS and NOT_PARSABLE_SCHEMAS[schema_id] in ev.exception_with_traceback
def check_invalid(ev):
if schema_id in INVALID_SCHEMAS:
if ev.result.errors:
return any(
"The API schema contains non-string keys" in err.exception_with_traceback
for err in ev.result.errors
)
return pytest.fail("Expected YAML parsing error")
return False
for event in runner.execute():
if isinstance(event, events.AfterExecution):
if check_xfailed(event):
continue
if check_flaky(event):
continue
if check_recursive_references(event):
continue
if check_unsatisfiable(event):
continue
if check_invalid(event):
continue
assert not event.result.has_errors, event.current_operation
assert not event.result.has_failures, event.current_operation
if isinstance(event, events.InternalError):
if check_not_parsable(event):
continue
assert not isinstance(event, events.InternalError), event.exception_with_traceback
and not unquote(x.get("job_id", "a")).startswith("\n")
)
@pytest.mark.usefixtures("nocommit_transaction")
@schema.hooks.apply(before_generate_path_parameters)
@schema.parametrize(method=["GET", "DELETE"])
def test_api(case, with_auth, auth_token):
if with_auth:
upd = {"Authorization": f"Bearer {auth_token}"}
if case.headers is None:
case.headers = upd
else:
case.headers.update(upd)
response = case.call_asgi()
case.validate_response(response)
@pytest.mark.usefixtures("nocommit_transaction")
@schema.parametrize(method=["POST"], operation_id="^(?!.*update.*system.*).*")
@hypsettings(max_examples=5, suppress_health_check=HealthCheck.all())
def test_api_post(case, with_auth, auth_token):
if with_auth:
upd = {"Authorization": f"Bearer {auth_token}"}
if case.headers is None:
case.headers = upd
else:
case.headers.update(upd)
response = case.call_asgi()
case.validate_response(response)
from __future__ import absolute_import, division, print_function
from random import Random
import pytest
from hypothesis import HealthCheck, settings
from hypothesis.errors import Flaky
from hypothesis.internal.compat import hbytes, hrange
from hypothesis.internal.conjecture.data import ConjectureData, Status, StopTest
from hypothesis.internal.conjecture.datatree import DataTree
from hypothesis.internal.conjecture.engine import ConjectureRunner
TEST_SETTINGS = settings(
max_examples=5000, database=None, suppress_health_check=HealthCheck.all()
)
def runner_for(*examples):
if len(examples) == 1 and isinstance(examples[0], list):
examples = examples[0]
def accept(tf):
runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0))
runner.exit_with = lambda reason: None
ran_examples = []
for e in examples:
e = hbytes(e)
data = runner.cached_test_function(e)
ran_examples.append((e, data))
def _test_func_is_valid_kernel(kernel, sample_dim, num_samples):
"""A func is a valid kernel if the kernel matrix generated by it is PSD.
Not including this in tests for all kernels to allow for non-PSD kernels in the future
"""
KM = KernelMatrix(kernel, name='TestKM')
KM.attach_to(gen_random_sample(num_samples, sample_dim))
is_psd = is_positive_semidefinite(KM.full, verbose=True)
if not is_psd:
raise ValueError('{} is not PSD'.format(str(KM)))
@hyp_settings(max_examples=num_tests_psd_kernel, deadline=None,
timeout=unlimited, suppress_health_check=HealthCheck.all())
@given(strategies.integers(range_feature_dim[0], range_feature_dim[1]),
strategies.integers(range_num_samples[0], range_num_samples[1]),
strategies.integers(range_polynomial_degree[0], range_polynomial_degree[1]),
strategies.floats(min_value=0, max_value=1e3,
allow_nan=False, allow_infinity=False))
def test_polynomial_kernel(sample_dim, num_samples,
poly_degree, poly_intercept):
"""Tests specific for Polynomial kernel."""
poly = PolyKernel(degree=poly_degree, b=poly_intercept, skip_input_checks=False)
_test_for_all_kernels(poly, sample_dim)
_test_func_is_valid_kernel(poly, sample_dim, num_samples)
@hyp_settings(max_examples=num_tests_psd_kernel, deadline=None,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.
#
# END HEADER
from __future__ import absolute_import, division, print_function
from random import Random
from hypothesis import HealthCheck, settings
from hypothesis.internal.compat import hbytes
from hypothesis.internal.conjecture.data import ConjectureData, Status
from hypothesis.internal.conjecture.engine import ConjectureRunner, RunIsComplete
TEST_SETTINGS = settings(
max_examples=5000, database=None, suppress_health_check=HealthCheck.all()
)
def runner_for(*examples):
if len(examples) == 1 and isinstance(examples[0], list):
examples = examples[0]
def accept(tf):
runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0))
ran_examples = []
for e in examples:
e = hbytes(e)
data = ConjectureData.for_buffer(e)
try:
runner.test_function(data)
buf.pop()
try:
d = ConjectureData.for_buffer(buf)
k = d.draw(st.integers())
stop = d.draw_bits(8)
if stop > 0 and k > 0:
return (draw(st.integers(0, k - 1)), hbytes(d.buffer))
except (StopTest, IndexError):
pass
@example((2, b'\x00\x00\n\x01'))
@example((1, b'\x00\x00\x06\x01'))
@example(problem=(32768, b'\x03\x01\x00\x00\x00\x00\x00\x01\x00\x02\x01'))
@settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited, deadline=None,
max_examples=10, verbosity=Verbosity.normal
)
@given(problems())
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
#
# END HEADER
from __future__ import absolute_import, division, print_function
import pytest
import hypothesis.strategies as st
from hypothesis import HealthCheck, Verbosity, given, reject, settings
from hypothesis.errors import Unsatisfiable
from tests.common.debug import minimal
from tests.common.utils import no_shrink
@pytest.mark.parametrize("strat", [st.text(min_size=5)])
@settings(phases=no_shrink, deadline=None, suppress_health_check=HealthCheck.all())
@given(st.data())
def test_explore_arbitrary_function(strat, data):
cache = {}
def predicate(x):
try:
return cache[x]
except KeyError:
return cache.setdefault(x, data.draw(st.booleans(), label=repr(x)))
try:
minimal(
strat,
predicate,
settings=settings(
import ast
import dis
import io
import lib2to3.pgen2.tokenize as lib2to3_tokenize
import tokenize
import unittest
import hypothesmith
from hypothesis import HealthCheck, example, given, reject, settings, strategies as st
# Used to mark tests which generate arbitrary source code,
# because that's a relatively expensive thing to get right.
settings.register_profile(
"slow", deadline=None, suppress_health_check=HealthCheck.all(),
)
class TestAST(unittest.TestCase):
@unittest.skipIf(not hasattr(ast, "unparse"), reason="new in 3.9")
@given(source_code=hypothesmith.from_grammar())
@settings.get_profile("slow")
def test_ast_unparse(self, source_code):
"""Unparsing always produces code which parses back to the same ast."""
first = ast.parse(source_code)
unparsed = ast.unparse(first)
second = ast.parse(unparsed)
assert ast.dump(first) == ast.dump(second)
class TestDis(unittest.TestCase):
@given(
from __future__ import division, print_function, absolute_import
import pytest
import hypothesis.strategies as st
from hypothesis import Verbosity, HealthCheck, find, given, reject, \
settings, unlimited
from hypothesis.errors import NoSuchExample
from tests.common.utils import no_shrink
@pytest.mark.parametrize('strat', [st.text(min_size=5)])
@settings(
phases=no_shrink, deadline=None,
suppress_health_check=HealthCheck.all()
)
@given(st.data())
def test_explore_arbitrary_function(strat, data):
cache = {}
def predicate(x):
try:
return cache[x]
except KeyError:
return cache.setdefault(x, data.draw(st.booleans(), label=repr(x)))
try:
find(
strat, predicate,
settings=settings(
# See issue https://github.com/HypothesisWorks/hypothesis-python/issues/38
# for details
pass
Litter = namedtuple('Litter', ('kitten1', 'kitten2'))
@given(builds(Litter, integers(), integers()))
def test_named_tuples_are_of_right_type(litter):
assert isinstance(litter, Litter)
@fails_with(AttributeError)
@given(integers().map(lambda x: x.nope))
@settings(suppress_health_check=HealthCheck.all())
def test_fails_in_reify(x):
pass
@given(text(u'a'))
def test_a_text(x):
assert set(x).issubset(set(u'a'))
@given(text(u''))
def test_empty_text(x):
assert not x
@given(text(u'abcdefg'))
