def test_the_slow_test_health_only_runs_if_health_checks_are_on(): @given(st.integers()) @settings(suppress_health_check=HealthCheck.all(), deadline=None) def a(x): time.sleep(1000) a()
def test_returning_non_none_does_not_fail_if_health_check_disabled(): @given(st.integers()) @settings(suppress_health_check=HealthCheck.all()) def a(x): return 1 a()
def test_database_clears_secondary_key(): key = b"key" database = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) == 10: data.mark_interesting() else: data.mark_invalid() runner = ConjectureRunner( f, settings=settings( max_examples=1, buffer_size=1024, database=database, suppress_health_check=HealthCheck.all(), ), database_key=key, ) for i in range(10): database.save(runner.secondary_key, hbytes([i])) runner.cached_test_function([10]) assert runner.interesting_examples assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 10 runner.clear_secondary_key() assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 0
def test_debug_data(capsys): buf = [0, 1, 2] def f(data): for x in hbytes(buf): if data.draw_bits(8) != x: data.mark_invalid() data.start_example(1) data.stop_example() data.mark_interesting() runner = ConjectureRunner( f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.debug, ), ) runner.cached_test_function(buf) runner.run() out, _ = capsys.readouterr() assert re.match(u"\\d+ bytes \\[.*\\] -> ", out) assert "INTERESTING" in out
def test_contains_the_test_function_name_in_the_exception_string(): look_for_one = settings( max_examples=1, suppress_health_check=HealthCheck.all()) @given(integers()) @look_for_one def this_has_a_totally_unique_name(x): reject() with raises(Unsatisfiable) as e: this_has_a_totally_unique_name() assert this_has_a_totally_unique_name.__name__ in e.value.args[0] class Foo(object): @given(integers()) @look_for_one def this_has_a_unique_name_and_lives_on_a_class(self, x): reject() with raises(Unsatisfiable) as e: Foo().this_has_a_unique_name_and_lives_on_a_class() assert ( Foo.this_has_a_unique_name_and_lives_on_a_class.__name__ ) in e.value.args[0]
def test_failure_sequence_inducing(building, testing, rnd): buildit = iter(building) testit = iter(testing) def build(x): try: assume(not next(buildit)) except StopIteration: pass return x @given(integers().map(build)) @settings( verbosity=Verbosity.quiet, database=None, suppress_health_check=HealthCheck.all(), phases=no_shrink ) def test(x): try: i = next(testit) except StopIteration: return if i == 1: return elif i == 2: reject() else: raise Nope() try: test() except (Nope, Unsatisfiable, Flaky): pass except UnsatisfiedAssumption: raise SatisfyMe()
def test_database_uses_values_from_secondary_key(): key = b'key' database = InMemoryExampleDatabase() def f(data): if data.draw_bits(8) >= 5: data.mark_interesting() else: data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, buffer_size=1024, database=database, suppress_health_check=HealthCheck.all(), ), database_key=key) for i in range(10): database.save(runner.secondary_key, hbytes([i])) runner.test_function(ConjectureData.for_buffer(hbytes([10]))) assert runner.interesting_examples assert len(set(database.fetch(key))) == 1 assert len(set(database.fetch(runner.secondary_key))) == 10 runner.clear_secondary_key() assert len(set(database.fetch(key))) == 1 assert set( map(int_from_bytes, database.fetch(runner.secondary_key)) ) == set(range(6, 11)) v, = runner.interesting_examples.values() assert list(v.buffer) == [5]
def test_notes_hard_to_satisfy(): @given(st.integers()) @settings(suppress_health_check=HealthCheck.all()) def test(i): assume(i == 0) stats = call_for_statistics(test) assert 'satisfied assumptions' in stats.exit_reason
def test_raises_unsatisfiable_if_all_false(): @given(integers()) @settings(max_examples=50, suppress_health_check=HealthCheck.all()) def test_assume_false(x): reject() with pytest.raises(Unsatisfiable): test_assume_false()
def test_default_health_check_can_weaken_specific(): import random @settings(suppress_health_check=HealthCheck.all()) @given(st.lists(st.integers(), min_size=1)) def test(x): random.choice(x) test()
def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem blob = hbytes(blob) try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0) def f(data): k = data.draw(st.integers()) data.output = repr(k) if data.draw_bits(8) == stop and k >= n: data.mark_interesting() runner = ConjectureRunner(f, random=Random(0), settings=settings( suppress_health_check=HealthCheck.all(), timeout=unlimited, phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug ), database_key=None) runner.test_function(ConjectureData.for_buffer(blob)) assert runner.interesting_examples v, = runner.interesting_examples.values() shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING) shrinker.clear_passes() shrinker.add_new_pass('minimize_individual_blocks') shrinker.shrink() v = shrinker.shrink_target m = ConjectureData.for_buffer(v.buffer).draw(st.integers()) assert m == n # Upper bound on the length needed is calculated as follows: # * We have an initial byte at the beginning to decide the length of the # integer. # * We have a terminal byte as the stop value. # * The rest is the integer payload. This should be n. Including the sign # bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only # have power of two sizes, so it may be up to a factor of two more than # that. bits_needed = 1 + n.bit_length() actual_bits_needed = min( [s for s in WideRangeIntStrategy.sizes if s >= bits_needed]) bytes_needed = actual_bits_needed // 8 # 3 extra bytes: two for the sampler, one for the capping value. assert len(v.buffer) == 3 + bytes_needed
def run_to_buffer(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.run() assert runner.interesting_examples last_data, = runner.interesting_examples.values() return hbytes(last_data.buffer)
def accept(f): with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.test_function(ConjectureData.for_buffer(start)) assert runner.interesting_examples last_data, = runner.interesting_examples.values() return runner.new_shrinker( last_data, lambda d: d.status == Status.INTERESTING )
def test_large_initial_write(): big = hbytes(b'\xff') * 512 def f(data): data.write(big) data.draw_bits(63) with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( max_examples=5000, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.run() assert runner.exit_reason == ExitReason.finished
def test_exit_because_max_iterations(): def f(data): data.draw_bits(64) data.mark_invalid() runner = ConjectureRunner(f, settings=settings( max_examples=1, buffer_size=1024, database=None, suppress_health_check=HealthCheck.all(), )) runner.run() assert runner.call_count <= 1000 assert runner.exit_reason == ExitReason.max_iterations
def test_can_increase_number_of_bytes_drawn_in_tail(): # This is designed to trigger a case where the zero bound queue will end up # increasing the size of data drawn because moving zeroes into the initial # prefix will increase the amount drawn. def f(data): x = data.draw_bytes(5) n = x.count(0) b = data.draw_bytes(n + 1) assert not any(b) runner = ConjectureRunner( f, settings=settings( max_examples=100, buffer_size=11, suppress_health_check=HealthCheck.all())) runner.run()
def test_will_reset_the_tree_as_it_goes(monkeypatch): monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3) def f(data): data.draw_bits(8) with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( database=None, suppress_health_check=HealthCheck.all(), )) def step(n): runner.test_function(ConjectureData.for_buffer([n])) step(0) step(1) assert len(runner.tree[0]) > 1 step(2) assert len(runner.tree[0]) == 1
def run_language_test_for(root, data, seed): random.seed(seed) def test(local_data): node = root while not isinstance(node, Terminal): if isinstance(node, Write): local_data.write(hbytes(node.value)) node = node.child else: assert isinstance(node, Branch) c = local_data.draw_bits(node.bits) try: node = node.children[c] except KeyError: if data is None: return node = node.children.setdefault(c, data.draw(nodes)) assert isinstance(node, Terminal) if node.status == Status.INTERESTING: local_data.mark_interesting(node.payload) elif node.status == Status.INVALID: local_data.mark_invalid() runner = ConjectureRunner( test, settings=settings( max_examples=1, buffer_size=512, database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.quiet, phases=list(Phase), ), ) try: runner.run() finally: if data is not None: note(root) assume(runner.interesting_examples)
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch): monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3) def f(data): if data.draw_bits(8) == 7: data.mark_interesting() with deterministic_PRNG(): runner = ConjectureRunner(f, settings=settings( database=None, suppress_health_check=HealthCheck.all(), )) def step(n): runner.test_function(ConjectureData.for_buffer([n])) step(0) step(1) assert len(runner.tree) > 1 step(7) assert len(runner.tree) > 1 t = len(runner.tree) runner.shrink_interesting_examples() assert len(runner.tree) > t
m = merged([schema, schema]) assert m == canonicalish(schema) def _merge_semantics_helper(data, s1, s2, combined): note(combined) ic = data.draw(from_schema(combined), label="combined") i1 = data.draw(from_schema(s1), label="s1") i2 = data.draw(from_schema(s2), label="s2") assert is_valid(ic, s1) assert is_valid(ic, s2) assert is_valid(i1, s2) == is_valid(i1, combined) assert is_valid(i2, s1) == is_valid(i2, combined) @settings(suppress_health_check=HealthCheck.all(), deadline=None) @given(st.data(), json_schemata(), json_schemata()) def test_merge_semantics(data, s1, s2): assume(canonicalish(s1) != FALSEY and canonicalish(s2) != FALSEY) combined = merged([s1, s2]) assume(combined is not None) assume(combined != FALSEY) _merge_semantics_helper(data, s1, s2, combined) @settings(suppress_health_check=HealthCheck.all(), deadline=None) @given( st.data(), gen_number(kind="integer") | gen_number(kind="number"), gen_number(kind="integer") | gen_number(kind="number"), )
# We draw n as two separate calls so that it doesn't show up as a # single block. If it did, the heuristics that allow us to move # blocks around would fire and it would move right, which would # then allow us to shrink it more easily. n = (data.draw_bits(16) << 16) | data.draw_bits(16) if n == MAX_INT: return (POISON,) else: return (None,) LOTS = 10 ** 6 TEST_SETTINGS = settings( database=None, suppress_health_check=HealthCheck.all(), max_examples=LOTS, deadline=None, timeout=unlimited ) @pytest.mark.parametrize('size', [2, 5, 10]) @pytest.mark.parametrize('seed', [0, 15993493061449915028]) def test_can_reduce_poison_from_any_subtree(size, seed): """This test validates that we can minimize to any leaf node of a binary tree, regardless of where in the tree the leaf is.""" random = Random(seed) # Initially we create the minimal tree of size n, regardless of whether it # is poisoned (which it won't be - the poison event essentially never # happens when drawing uniformly at random).
@st.composite def problem(draw): b = draw(st.binary(min_size=1, max_size=8)) m = int_from_bytes(b) * 256 assume(m > 0) marker = draw(st.binary(max_size=8)) bound = draw(st.integers(0, m - 1)) return (b, marker, bound) base_settings = settings( database=None, deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10, verbosity=Verbosity.normal, phases=(Phase.explicit, Phase.generate), ) @example((b"\x10\x00\x00\x00\x00\x00", b"", 2861143707951135)) @example((b"\x05Cn", b"%\x1b\xa0\xfa", 12394667)) @example((b"\x179 f", b"\xf5|", 24300326997)) @example((b"\x05*\xf5\xe5\nh", b"", 1076887621690235)) @example((b"=", b"", 2508)) @example((b"\x01\x00", b"", 20048)) @example((b"\x01", b"", 0)) @example((b"\x02", b"", 258)) @example((b"\x08", b"", 1792))
respond_yaml, respond_dot, respond_text, respond_cytoscape, respond_graphml, ) from tests.resotocore.hypothesis_extension import ( json_array_gen, json_simple_element_gen, node_gen, graph_stream, ) @given(json_array_gen) @settings(max_examples=20, suppress_health_check=HealthCheck.all()) @pytest.mark.asyncio async def test_json(elements: List[JsonElement]) -> None: async with stream.iterate(elements).stream() as streamer: result = "" async for elem in respond_json(streamer): result += elem assert json.loads(result) == elements @given(json_array_gen) @settings(max_examples=20, suppress_health_check=HealthCheck.all()) @pytest.mark.asyncio async def test_ndjson(elements: List[JsonElement]) -> None: async with stream.iterate(elements).stream() as streamer: result = []
def reduce_with_hypothesis_base(experiment, name, suppress_intervals): experiment = EXPERIMENTS[experiment] base_predicate = experiment.calculate_error_predicate(info(experiment, name)) generated = generate(experiment, name) classified = base_predicate(generated) assert classified == Classification.INTERESTING, (classified, base_predicate) results, predicate = tracking_predicate(base_predicate) generation_stats = { c: ExampleStatistics() for c in Status } input_to_outputs = [] def test_function(data): if suppress_intervals: start = data.start_example stop = data.stop_example def start_example(label): if data.depth >= 0: data.depth += 1 if data.depth > data.max_depth: data.max_depth = data.depth else: start(label) def stop_example(discard=False): if data.depth >= 1: data.depth -= 1 else: stop(discard) data.start_example = start_example data.stop_example = stop_example generation_start = time.monotonic() try: try: source = data.draw(experiment.generator) except UnsatisfiedAssumption: data.mark_invalid() finally: generation_time = time.monotonic() - generation_start result = predicate(source) input_to_outputs.append((encode_failure(data.buffer).decode('ascii'), source, result.name)) if trace_memory_usage: display_top(tracemalloc.take_snapshot()) if result == Classification.INTERESTING: data.mark_interesting() elif result in (Classification.INVALIDCHEAP, Classification.INVALIDEXPENSIVE): data.mark_invalid() finally: generation_stats[data.status].record(size=len(data.buffer), runtime=generation_time) buffer = raw_buffer(experiment, name) runner = eng.ConjectureRunner( test_function, settings=settings( database=None, max_examples=1, suppress_health_check=HealthCheck.all(), deadline=None, verbosity=Verbosity.debug, buffer_size=BUFFER_SIZE, ), random=Random(int.from_bytes(hashlib.sha1(buffer).digest(), "big")), ) def debug_data(data): runner.debug( f"DATA {hashlib.sha1(data.buffer).hexdigest()[:8]}: {len(data.buffer)} bytes, {data.status.name}" ) runner.debug_data = debug_data runner.cached_test_function(buffer) assert runner.interesting_examples results.start() runner.shrink_interesting_examples() results.finish() v, = runner.interesting_examples.values() return { "final": { "buffer": base64.b64encode(v.buffer).decode("ascii"), "generated": buffer_to_value(experiment, v.buffer), }, "reductionstats": attr.asdict(results), "input_to_outputs": input_to_outputs, "generationstats": {k.name: attr.asdict(v) for k, v in generation_stats.items()}, }
def learner_for(strategy): """Returns an LStar learner that predicts whether a buffer corresponds to a discard free choice sequence leading to a valid value for this strategy.""" try: return LEARNERS[strategy] except KeyError: pass def test_function(data): try: data.draw(strategy) except UnsatisfiedAssumption: data.mark_invalid() data.mark_interesting() runner = ConjectureRunner( test_function, settings=settings( database=None, verbosity=Verbosity.quiet, suppress_health_check=HealthCheck.all(), ), random=Random(0), ignore_limits=True, ) def predicate(s): result = runner.cached_test_function(s) if result.status < Status.VALID: return False if result.has_discards: return False return result.buffer == s learner = LStar(predicate) runner.run() (v, ) = runner.interesting_examples.values() # We make sure the learner has properly learned small examples. # This is all fairly ad hoc but is mostly designed to get it # to understand what the smallest example is and avoid any # loops at the beginning of the DFA that don't really exist. learner.learn(v.buffer) for n in [1, 2, 3]: for _ in range(5): learner.learn(uniform(runner.random, n) + v.buffer) prev = -1 while learner.generation != prev: prev = learner.generation for _ in range(10): s = uniform(runner.random, len(v.buffer)) + bytes(BUFFER_SIZE) learner.learn(s) data = runner.cached_test_function(s) if data.status >= Status.VALID: learner.learn(data.buffer) LEARNERS[strategy] = learner return learner
from hypothesis import given, example from hypothesis import HealthCheck, settings from hypothesis.strategies import binary,\ dictionaries, integers, lists, one_of, recursive, text from homework03 import encode, decode settings.suppres_health_check = HealthCheck.all() """http://www.bittorrent.org/beps/bep_0003.html""" SIMPLE_TYPES = one_of(integers(), binary()) # Strings are length-prefixed base ten followed by a colon and the string. # For example 4:spam corresponds to 'spam'. @given(text()) @example('spam') def test_string(s): assert decode(encode(s)) == bytes(s, 'utf-8') @given(binary()) def test_bytestring(s): assert decode(encode(s)) == s # Integers are represented by an 'i' followed by the number in base 10 followed by an 'e'. # For example i3e corresponds to 3 and i-3e corresponds to -3. # Integers have no size limitation. i-0e is invalid. # All encodings with a leading zero, such as i03e, are invalid, # other than i0e, which of course corresponds to 0.
import bz2 import gzip import unittest from hypothesis import HealthCheck, given, settings, strategies as st no_health_checks = settings(suppress_health_check=HealthCheck.all()) class TestBz2(unittest.TestCase): @given(payload=st.binary(), compresslevel=st.integers(1, 9)) def test_bz2_round_trip(self, payload, compresslevel): result = bz2.decompress( bz2.compress(payload, compresslevel=compresslevel)) self.assertEqual(payload, result) @given(payloads=st.lists(st.binary()), compresslevel=st.integers(1, 9)) def test_bz2_incremental_compress_eq_oneshot(self, payloads, compresslevel): c = bz2.BZ2Compressor(compresslevel) compressed = b"".join(c.compress(p) for p in payloads) + c.flush() self.assertEqual(compressed, bz2.compress(b"".join(payloads), compresslevel)) @no_health_checks @given(payload=st.binary(), compresslevel=st.integers(1, 9), data=st.data()) def test_bz2_incremental_decompress_eq_oneshot(self, payload, compresslevel, data): compressed = bz2.compress(payload, compresslevel=compresslevel)
assert minimal(st.floats(), lambda x: x > 1) == 2.0 assert minimal(st.floats(), lambda x: x > 0) == 1.0 @pytest.mark.parametrize("n", [1, 2, 3, 8, 10]) def test_can_shrink_in_variable_sized_context(n): x = minimal(st.lists(st.floats(), min_size=n), any) assert len(x) == n assert x.count(0.0) == n - 1 assert 1 in x @example(1.7976931348623157e308) @example(1.5) @given(st.floats(min_value=0, allow_infinity=False, allow_nan=False)) @settings(deadline=None, suppress_health_check=HealthCheck.all()) def test_shrinks_downwards_to_integers(f): g = minimal(st.floats(), lambda x: x >= f, settings(verbosity=Verbosity.quiet)) assert g == ceil(f) @example(1) @given(st.integers(1, 2 ** 16 - 1)) @settings(deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10) def test_shrinks_downwards_to_integers_when_fractional(b): g = minimal( st.floats(), lambda x: assume((0 < x < (2 ** 53)) and int(x) != x) and x >= b, settings=settings(verbosity=Verbosity.quiet, max_examples=10 ** 6), ) assert g == b + 0.5
def normalize( base_name, test_function, *, required_successes=100, allowed_to_update=False, max_dfas=10, ): """Attempt to ensure that this test function successfully normalizes - i.e. whenever it declares a test case to be interesting, we are able to shrink that to the same interesting test case (which logically should be the shortlex minimal interesting test case, though we may not be able to detect if it is). Will run until we have seen ``required_successes`` many interesting test cases in a row normalize to the same value. If ``allowed_to_update`` is True, whenever we fail to normalize we will learn a new DFA-based shrink pass that allows us to make progress. Any learned DFAs will be written back into the learned DFA file at the end of this function. If ``allowed_to_update`` is False, this will raise an error as soon as it encounters a failure to normalize. Additionally, if more than ``max_dfas` DFAs are required to normalize this test function, this function will raise an error - it's essentially designed for small patches that other shrink passes don't cover, and if it's learning too many patches then you need a better shrink pass than this can provide. """ # Need import inside the function to avoid circular imports from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner runner = ConjectureRunner( test_function, settings=settings(database=None, suppress_health_check=HealthCheck.all()), ignore_limits=True, ) seen = set() dfas_added = 0 found_interesting = False consecutive_successes = 0 failures_to_find_interesting = 0 while consecutive_successes < required_successes: attempt = runner.cached_test_function(b"", extend=BUFFER_SIZE) if attempt.status < Status.INTERESTING: failures_to_find_interesting += 1 assert ( found_interesting or failures_to_find_interesting <= 1000 ), "Test function seems to have no interesting test cases" continue found_interesting = True target = attempt.interesting_origin def shrinking_predicate(d): return d.status == Status.INTERESTING and d.interesting_origin == target if target not in seen: seen.add(target) runner.shrink(attempt, shrinking_predicate) continue previous = fully_shrink( runner, runner.interesting_examples[target], shrinking_predicate ) current = fully_shrink(runner, attempt, shrinking_predicate) if current.buffer == previous.buffer: consecutive_successes += 1 continue consecutive_successes = 0 if not allowed_to_update: raise FailedToNormalise( "Shrinker failed to normalize %r to %r and we are not allowed to learn new DFAs." % (previous.buffer, current.buffer) ) if dfas_added >= max_dfas: raise FailedToNormalise( "Test function is too hard to learn: Added %d DFAs and still not done." % (dfas_added,) ) dfas_added += 1 new_dfa = learn_a_new_dfa( runner, previous.buffer, current.buffer, shrinking_predicate ) name = ( base_name + "-" + hashlib.sha256(repr(new_dfa).encode("utf-8")).hexdigest()[:10] ) # If there is a name collision this DFA should already be being # used for shrinking, so we should have already been able to shrink # v further. assert name not in SHRINKING_DFAS SHRINKING_DFAS[name] = new_dfa if dfas_added > 0: # We've learned one or more DFAs in the course of normalising, so now # we update the file to record those for posterity. update_learned_dfas()
assert minimal(st.floats(), lambda x: x > 1) == 2.0 assert minimal(st.floats(), lambda x: x > 0) == 1.0 @pytest.mark.parametrize("n", [1, 2, 3, 8, 10]) def test_can_shrink_in_variable_sized_context(n): x = minimal(st.lists(st.floats(), min_size=n), any) assert len(x) == n assert x.count(0.0) == n - 1 assert 1 in x @example(1.7976931348623157e308) @example(1.5) @given(st.floats(min_value=0, allow_infinity=False, allow_nan=False)) @settings(deadline=None, suppress_health_check=HealthCheck.all()) def test_shrinks_downwards_to_integers(f): g = minimal(st.floats(), lambda x: x >= f, settings(verbosity=Verbosity.quiet)) assert g == ceil(f) @example(1) @given(st.integers(1, 2**16 - 1)) @settings(deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10) def test_shrinks_downwards_to_integers_when_fractional(b): g = minimal( st.floats(), lambda x: assume((0 < x < (2**53)) and int(x) != x) and x >= b,
index_to_add_word = draw(st.integers(min_value=0, max_value=len(text) - 1)) new_word = vocabulary[draw(st.integers(min_value=0, max_value=vocab_size))] text.append(text[index_to_add_word] + " " + new_word) return (text, vocabulary) # TODO: Add a set of tests for passing in instantiated classes # TODO: Test that DocVectorizer transform preserves column order and size on new data @given(generate_test_text_info()) @settings( deadline=None, suppress_health_check=[HealthCheck(3)], max_examples=DEFAULT_MAX_EXAMPLES, ) @example((test_text_example, None)) def test_joint_nobasistransformer(test_text_info): test_text = test_text_info[0] model = JointWordDocVectorizer(feature_basis_converter=None, token_contractor_kwds={"min_score": 8}) result = model.fit_transform(test_text) assert isinstance(result, scipy.sparse.csr_matrix) if test_text == test_text_example: assert result.shape == (12, 7) else: assert result.shape[0] == model.n_words_ + len(test_text) assert result.shape[1] == model.n_words_
class TestSummaries(TestCase): def assertDictAlmostEqual(self, d1, d2, msg=None, places=7): # check if both inputs are dicts self.assertIsInstance(d1, dict, 'First argument is not a dictionary') self.assertIsInstance(d2, dict, 'Second argument is not a dictionary') # check if both inputs have the same keys self.assertEqual(d1.keys(), d2.keys()) # check each key for key, value in d1.items(): if isinstance(value, dict): self.assertDictAlmostEqual(d1[key], d2[key], msg=msg) else: self.assertAlmostEqual(d1[key], d2[key], places=places, msg=msg) def assertSummaryIsValid(self, loc_df, gul_inputs, exp_summary, perils_expected=None): cov_types = ['buildings', 'other', 'bi', 'contents'] lookup_status = [ 'success', 'fail', 'nomatch', 'fail_ap', 'fail_v', 'notatrisk' ] loc_rename_cols = { 'bitiv': 'bi', 'buildingtiv': 'buildings', 'contentstiv': 'contents', 'othertiv': 'other' } # Check each returned peril for peril in perils_expected: peril_summary = exp_summary[peril] # Check the 'All' section supported_tivs = loc_df[[ 'buildingtiv', 'othertiv', 'bitiv', 'contentstiv' ]].sum(0).rename(loc_rename_cols) self.assertAlmostEqual(supported_tivs.sum(0), peril_summary['all']['tiv']) for cov in cov_types: self.assertAlmostEqual( supported_tivs[cov], peril_summary['all']['tiv_by_coverage'][cov]) # Check each lookup status peril_expected = gul_inputs[gul_inputs.peril_id == peril] for status in lookup_status: peril_status = peril_expected[peril_expected.status == status] self.assertAlmostEqual(peril_status.tiv.sum(), peril_summary[status]['tiv']) self.assertEqual(len(peril_status.loc_id.unique()), peril_summary[status]['number_of_locations']) for cov in cov_types: cov_type_id = SUPPORTED_COVERAGE_TYPES[cov]['id'] cov_type_tiv = peril_status[peril_status.coverage_type_id == cov_type_id].tiv.sum() self.assertAlmostEqual( cov_type_tiv, peril_summary[status]['tiv_by_coverage'][cov]) # Check 'noreturn' status tiv_returned = sum([ s[1]['tiv'] for s in peril_summary.items() if s[0] in lookup_status ]) self.assertAlmostEqual(peril_summary['all']['tiv'] - tiv_returned, peril_summary['noreturn']['tiv']) for cov in cov_types: cov_tiv_returned = sum([ s[1]['tiv_by_coverage'][cov] for s in peril_summary.items() if s[0] in lookup_status ]) self.assertAlmostEqual( peril_summary['all']['tiv_by_coverage'][cov] - cov_tiv_returned, peril_summary['noreturn']['tiv_by_coverage'][cov]) @given(st.data()) @settings(max_examples=20, deadline=None) def test_single_peril__totals_correct(self, data): # Shared Values between Loc / keys loc_size = data.draw(integers(10, 20)) supported_cov = data.draw( st.lists(integers(1, 4), unique=True, min_size=1, max_size=4)) perils = 'WTC' # Create Mock keys_df keys_data = list() for i in supported_cov: keys_data += data.draw( keys(size=loc_size, from_peril_ids=just(perils), from_coverage_type_ids=just(i), from_area_peril_ids=just(1), from_vulnerability_ids=just(1), from_messages=just('str'))) keys_df = pd.DataFrame.from_dict(keys_data) # Create Mock location_df loc_df = pd.DataFrame.from_dict( data.draw( min_source_exposure(size=loc_size, from_location_perils_covered=just(perils), from_location_perils=just(perils), from_building_tivs=integers(1000, 1000000), from_other_tivs=integers(100, 100000), from_contents_tivs=integers(50, 50000), from_bi_tivs=integers(20, 20000)))) loc_df['loc_id'] = get_ids(loc_df, ['portnumber', 'accnumber', 'locnumber']) # Run exposure_summary exp_summary = get_exposure_summary( exposure_df=loc_df, keys_df=keys_df, ) # Run Gul Proccessing gul_inputs = get_gul_input_items(loc_df, keys_df) gul_inputs = gul_inputs[gul_inputs['status'].isin( OASIS_KEYS_STATUS_MODELLED)] # Fetch expected TIVS tiv_portfolio = loc_df[[ 'buildingtiv', 'othertiv', 'bitiv', 'contentstiv' ]].sum(1).sum(0) tiv_modelled = gul_inputs['tiv'].sum() tiv_not_modelled = tiv_portfolio - tiv_modelled # Check TIV values self.assertEqual(tiv_portfolio, exp_summary['total']['portfolio']['tiv']) self.assertEqual(tiv_modelled, exp_summary['total']['modelled']['tiv']) self.assertEqual(tiv_not_modelled, exp_summary['total']['not-modelled']['tiv']) # Check number of locs self.assertEqual( len(loc_df), exp_summary['total']['portfolio']['number_of_locations']) self.assertEqual( len(gul_inputs.loc_id.unique()), exp_summary['total']['modelled']['number_of_locations']) # Check number of not-modelled # WARNING: current assumption is that all cov types must be covered to be modelled #moddeled = 0 #moddeld_loc_ids = gul_inputs[gul_inputs['status'] == 'success'].loc_id.unique() #for loc_id in moddeld_loc_ids: # if len(gul_inputs[gul_inputs.loc_id == loc_id].coverage_type_id.unique()) == 4: # moddeled+=1 #self.assertEqual(len(loc_df) - moddeled, exp_summary['total']['not-modelled']['number_of_locations']) @given(st.data()) @settings(max_examples=10, deadline=None) def test_multi_perils__single_covarage(self, data): loc_size = data.draw(integers(10, 20)) supported_cov = data.draw(integers(1, 4)) perils = data.draw( st.lists(st.text(alphabet=(string.ascii_letters + string.digits), min_size=2, max_size=6), min_size=2, max_size=6, unique=True)) # Create Mock keys_df keys_data = list() for p in perils: keys_data += data.draw( keys(size=loc_size, from_peril_ids=just(p), from_coverage_type_ids=just(supported_cov), from_area_peril_ids=just(1), from_vulnerability_ids=just(1), from_messages=just('str'))) keys_df = pd.DataFrame.from_dict(keys_data) perils_returned = keys_df.peril_id.unique().tolist() # Create Mock location_df perils_covered = ';'.join(perils) loc_df = pd.DataFrame.from_dict( data.draw( min_source_exposure( size=loc_size, from_location_perils_covered=just(perils_covered), from_location_perils=just(perils_covered), from_building_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_other_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_contents_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_bi_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000))))) loc_df['loc_id'] = get_ids(loc_df, ['portnumber', 'accnumber', 'locnumber']) # Run Summary output check self.assertSummaryIsValid( loc_df, get_gul_input_items(loc_df, keys_df), get_exposure_summary(exposure_df=loc_df, keys_df=keys_df), perils_returned) @given(st.data()) @settings(max_examples=10, deadline=None, suppress_health_check=HealthCheck.all()) def test_multi_perils__multi_covarage(self, data): loc_size = data.draw(integers(10, 20)) supported_cov = data.draw( st.lists(integers(1, 4), unique=True, min_size=1, max_size=4)) perils = data.draw( st.lists(st.text(alphabet=(string.ascii_letters + string.digits), min_size=2, max_size=6), min_size=2, max_size=6, unique=True)) # Create Mock keys_df keys_data = list() for c in supported_cov: for p in perils: keys_data += data.draw( keys(size=loc_size, from_peril_ids=just(p), from_coverage_type_ids=just(c), from_area_peril_ids=just(1), from_vulnerability_ids=just(1), from_messages=just('str'))) keys_df = pd.DataFrame.from_dict(keys_data) perils_returned = keys_df.peril_id.unique().tolist() # Create Mock location_df loc_df = pd.DataFrame.from_dict( data.draw( min_source_exposure( size=loc_size, from_location_perils_covered=st.sampled_from(perils), from_location_perils=st.sampled_from(perils), from_building_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_other_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_contents_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_bi_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000))))) loc_df['loc_id'] = get_ids(loc_df, ['portnumber', 'accnumber', 'locnumber']) # Run Summary output check exp_summary = get_exposure_summary(exposure_df=loc_df, keys_df=keys_df) gul_inputs = get_gul_input_items(loc_df, keys_df) self.assertSummaryIsValid(loc_df, gul_inputs, exp_summary, perils_returned) @given(st.data()) @settings(max_examples=10, deadline=None) def test_summary_file_written(self, data): loc_size = data.draw(integers(10, 20)) # Create Mock keys_data keys_data = data.draw( keys(size=loc_size, from_area_peril_ids=just(1), from_vulnerability_ids=just(1), from_messages=just('str'))) # Create Mock location_data loc_data = data.draw( min_source_exposure( size=loc_size, from_building_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_other_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_contents_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)), from_bi_tivs=st.one_of(st.floats(1.0, 1000.0), st.integers(1, 1000)))) # Prepare arguments for write_exposure_summary with TemporaryDirectory() as tmp_dir: keys_fp = os.path.join(tmp_dir, 'keys.csv') successes = [k for k in keys_data if k['status'] in ['success']] keys_errors_fp = os.path.join(tmp_dir, 'keys_errors.csv') nonsuccesses = [ k for k in keys_data if k['status'] not in ['success'] ] write_keys_files(keys=successes, keys_file_path=keys_fp, keys_errors=nonsuccesses, keys_errors_file_path=keys_errors_fp) location_fp = os.path.join(tmp_dir, 'location.csv') write_source_files(exposure=loc_data, exposure_fp=location_fp) location_df = get_location_df(location_fp) exposure_summary_fp = write_exposure_summary( tmp_dir, location_df, keys_fp, keys_errors_fp, get_default_exposure_profile()) self.assertTrue(os.path.isfile(exposure_summary_fp)) with open(exposure_summary_fp) as f: data = json.load(f) loc_df = pd.DataFrame.from_dict(loc_data) loc_df['loc_id'] = get_ids( loc_df, ['portnumber', 'accnumber', 'locnumber']) keys_df = pd.DataFrame.from_dict(keys_data) exp_summary = get_exposure_summary(loc_df, keys_df) self.assertDictAlmostEqual(data, exp_summary)
def weights(draw): parts = draw(st.lists(st.integers())) parts.reverse() base = Fraction(1, 1) for p in parts: base = Fraction(1) / (1 + base) return base @example([Fraction(1, 3), Fraction(1, 3), Fraction(1, 3)]) @example([Fraction(1, 1), Fraction(1, 2)]) @example([Fraction(1, 2), Fraction(4, 10)]) @example([Fraction(1, 1), Fraction(3, 5), Fraction(1, 1)]) @example([Fraction(2, 257), Fraction(2, 5), Fraction(1, 11)]) @settings( deadline=None, suppress_health_check=HealthCheck.all(), max_examples=0 if IN_COVERAGE_TESTS else settings.default.max_examples, ) @given(st.lists(weights(), min_size=1)) def test_sampler_distribution(weights): total = sum(weights) n = len(weights) assume(total > 0) probabilities = [w / total for w in weights] sampler = cu.Sampler(weights) calculated = [Fraction(0)] * n for base, alternate, p_alternate in sampler.table:
result = [] count = 0 last = None for line in lines: if line: if last is not None and last != line: result.append(f'{last},{count}') count = 0 last = line count += 1 if last: result.append(f'{last},{count}') return '\n'.join(result) + '\n' @given(inputs()) @settings(database=ExampleDatabase(':memory:'), max_examples=100 * int(os.environ.get('TEST_FACTOR', 1)), suppress_health_check=HealthCheck.all()) # type: ignore def test_props(args): csv = args result = expected(csv) assert result == run(csv, 'bsv | bcounteach | bschema *,i64:a | csv') def test_basic(): stdin = """ a a a b b a """ stdout = """
def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem blob = hbytes(blob) try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0) def f(data): k = data.draw(st.integers()) data.output = repr(k) if data.draw_bits(8) == stop and k >= n: data.mark_interesting() runner = ConjectureRunner( f, random=Random(0), settings=settings( suppress_health_check=HealthCheck.all(), phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug, ), database_key=None, ) runner.cached_test_function(blob) assert runner.interesting_examples (v,) = runner.interesting_examples.values() shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING) shrinker.fixate_shrink_passes(["minimize_individual_blocks"]) v = shrinker.shrink_target m = ConjectureData.for_buffer(v.buffer).draw(st.integers()) assert m == n # Upper bound on the length needed is calculated as follows: # * We have an initial byte at the beginning to decide the length of the # integer. # * We have a terminal byte as the stop value. # * The rest is the integer payload. This should be n. Including the sign # bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only # have power of two sizes, so it may be up to a factor of two more than # that. bits_needed = 1 + n.bit_length() actual_bits_needed = min( [s for s in WideRangeIntStrategy.sizes if s >= bits_needed] ) bytes_needed = actual_bits_needed // 8 # 3 extra bytes: two for the sampler, one for the capping value. assert len(v.buffer) == 3 + bytes_needed
a coverage-guided fuzzer I'm working on. """ import hypothesmith from hypothesis import HealthCheck, given, settings, strategies as st import black # This test uses the Hypothesis and Hypothesmith libraries to generate random # syntatically-valid Python source code and run Black in odd modes. @settings( max_examples=1000, # roughly 1k tests/minute, or half that under coverage derandomize=True, # deterministic mode to avoid CI flakiness deadline=None, # ignore Hypothesis' health checks; we already know that suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy. ) @given( # Note that while Hypothesmith might generate code unlike that written by # humans, it's a general test that should pass for any *valid* source code. # (so e.g. running it against code scraped of the internet might also help) src_contents=hypothesmith.from_grammar() | hypothesmith.from_node(), # Using randomly-varied modes helps us to exercise less common code paths. mode=st.builds( black.FileMode, line_length=st.just(88) | st.integers(0, 200), string_normalization=st.booleans(), is_pyi=st.booleans(), ), ) def test_idempotent_any_syntatically_valid_python(
import pandas as pd from pyrle import Rle import numpy as np strandedness = [False, "same", "opposite"] methods = ["join", "intersection", "overlap"] # strandedness = strandedness[1:2] # methods = methods[1:2] coverage_cmd = "Rscript --vanilla tests/compute_coverage.R {} {}" @pytest.mark.r @settings(max_examples=max_examples, deadline=deadline, timeout=unlimited, suppress_health_check=HealthCheck.all()) @given(df=dfs_min_single_chromosome()) def test_coverage(df): print("---" * 10) p = pr.PyRanges(df) print("pyranges\n", p) c = p.coverage()["chr1"] result_df = None with tempfile.TemporaryDirectory() as temp_dir: f1 = "{}/f1.txt".format(temp_dir) outfile = "{}/result.txt".format(temp_dir) R_df = df R_df.End = R_df.End - 1
def shrink(filename, sizes): with open(filename, 'rb') as i: initial_data = ConjectureData.for_buffer(i.read()) printing_to_stdout = False def data_info(data): name = hashlib.sha1(data.buffer).hexdigest()[:8] input_length = len(data.buffer) output_length = data.extra_information.generated_length return { "name": name, "input_length": input_length, "output_length": output_length, "interesting": data.status == Status.INTERESTING } if sizes is not None: if sizes == "-": writer = sys.stdout printing_to_stdout = True else: writer = open(sizes, 'w') def log_data(data): if data.status >= Status.VALID and hasattr(data.extra_information, 'generated_length'): writer.write(json.dumps(data_info(data)) + "\n") writer.flush() else: writer = None def log_data(data): pass initial = mktemp(suffix='.c') gen(initial_data, initial) errtype, gcc, opt = interesting_reason(initial, printing=not printing_to_stdout) if errtype == 'error': def test_function(data): with tempfile.TemporaryDirectory() as d: prog = os.path.join(d, 'test.c') try: gen(data, prog) data.extra_information.generated_length = os.stat(prog).st_size run_gcc(gcc, opt, prog) except subprocess.TimeoutExpired: return except subprocess.SubprocessError: name = hashlib.sha1(data.buffer).hexdigest()[:8] data.unique_name = name data.mark_interesting() finally: log_data(data) else: assert errtype == "differs" def test_function(data): with tempfile.TemporaryDirectory() as d: prog = os.path.join(d, 'test.c') try: gen(data, prog) data.generated_length = os.stat(prog).st_size if run_gcc(gcc, opt, prog) != run_gcc( '/opt/compiler-explorer/gcc-8.2.0/bin/gcc', '-O0', prog ): name = hashlib.sha1(data.buffer).hexdigest()[:8] data.unique_name = name data.mark_interesting() except subprocess.SubprocessError: pass finally: log_data(data) eng.MAX_SHRINKS = 10 ** 6 with open(filename, 'rb') as i: buffer = i.read() # tracemalloc.start() runner = eng.ConjectureRunner(test_function, settings=settings( database=None, max_examples=1000, timeout=unlimited, suppress_health_check=HealthCheck.all(), deadline=None, verbosity=Verbosity.quiet if printing_to_stdout else Verbosity.debug, buffer_size=BUFFER_SIZE ), random=Random(int.from_bytes(hashlib.sha1(buffer).digest(), 'big'))) class FakeTree(object): def add(self, data): pass def rewrite(self, buffer): return (buffer, None) def generate_novel_prefix(self, random): return b'' is_exhausted = False def uncached_test_function(buffer): data = ConjectureData.for_buffer(buffer) runner.test_function(data) # snapshot = tracemalloc.take_snapshot() # top_stats = snapshot.statistics('lineno') print("[ Top 10 ]") for stat in top_stats[:10]: print(stat) return data.as_result() # runner.tree = FakeTree() # runner.cached_test_function = uncached_test_function # runner.target_selector.add = lambda x: None runner.test_function(ConjectureData.for_buffer(buffer)) assert runner.interesting_examples runner.debug_data = lambda data: runner.debug( f"DATA({getattr(data, 'unique_name', None)}): {len(data.buffer)} bytes, {data.status.name}, {data.interesting_origin}" ) v, = runner.interesting_examples.values() shrinker = runner.new_shrinker( v, lambda d: d.status == Status.INTERESTING and d.interesting_origin == v.interesting_origin ) initial_calls = runner.call_count initial_valid = runner.valid_examples start = time.monotonic() shrinker.shrink() end = time.monotonic() final = data_info(shrinker.shrink_target) final["bytes"] = encode_failure(shrinker.shrink_target.buffer).decode('ascii') if writer is not None: writer.write(json.dumps({ "runtime": end - start, "calls": runner.call_count - initial_calls, "valid": runner.valid_examples - initial_valid, "final": final, }) + "\n") result = runner.interesting_examples[v.interesting_origin] runner.debug_data(result) gen(ConjectureData.for_buffer(result.buffer), 'shrunk.c') if writer is not None: writer.close()
def csr_slow(divider=2): dft = settings.default return settings(dft, deadline=None, suppress_health_check=HealthCheck.all(), max_examples=dft.max_examples // divider)
# obtain one at https://mozilla.org/MPL/2.0/. # # END HEADER from __future__ import absolute_import, division, print_function import pytest import hypothesis.strategies as st from hypothesis import HealthCheck, Verbosity, find, given, reject, settings from hypothesis.errors import NoSuchExample from tests.common.utils import no_shrink @pytest.mark.parametrize("strat", [st.text(min_size=5)]) @settings(phases=no_shrink, deadline=None, suppress_health_check=HealthCheck.all()) @given(st.data()) def test_explore_arbitrary_function(strat, data): cache = {} def predicate(x): try: return cache[x] except KeyError: return cache.setdefault(x, data.draw(st.booleans(), label=repr(x))) try: find( strat, predicate, settings=settings(
ls = minimal( st.lists(st.recursive( st.booleans(), lambda x: st.lists( x, min_size=2 * (target - 1), max_size=2 * target ).map(tuple), max_leaves=2 * target - 1)), lambda x: len(set(x)) >= target, timeout_after=None ) assert len(ls) == target @given(st.randoms()) @settings( max_examples=50, phases=no_shrink, suppress_health_check=HealthCheck.all(), deadline=None ) def test_can_use_recursive_data_in_sets(rnd): nested_sets = st.recursive(st.booleans(), st.frozensets, max_leaves=3) find_any(nested_sets, random=rnd) def flatten(x): if isinstance(x, bool): return frozenset((x,)) else: result = frozenset() for t in x: result |= flatten(t) if len(result) == 2: break
from hypothesis.internal.conjecture.engine import ConjectureRunner @st.composite def problem(draw): b = hbytes(draw(st.binary(min_size=1, max_size=8))) m = int_from_bytes(b) * 256 assume(m > 0) marker = draw(st.binary(max_size=8)) bound = draw(st.integers(0, m - 1)) return (b, marker, bound) base_settings = settings( database=None, deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10, verbosity=Verbosity.normal, timeout=unlimited, phases=( Phase.explicit, Phase.generate ) ) @example((b'\x10\x00\x00\x00\x00\x00', b'', 2861143707951135)) @example((b'\x05Cn', b'%\x1b\xa0\xfa', 12394667)) @example((b'\x179 f', b'\xf5|', 24300326997)) @example((b'\x05*\xf5\xe5\nh', b'', 1076887621690235)) @example((b'=', b'', 2508)) @example((b'\x01\x00', b'', 20048)) @example((b'\x01', b'', 0))
def main(filename): target = os.path.splitext(filename)[0] + ".json" random = Random( int.from_bytes( hashlib.sha1(os.path.basename(filename).encode("utf-8")).digest(), "big")) results = [] with open(filename, "r") as i: source = i.read() compiled = compile(source, filename, "exec") for _ in trange(100): seed = random.getrandbits(64) namespace = {} exec(compiled, namespace) test = namespace["test"] assert getattr(test, "is_hypothesis_test", False) base_function = test.hypothesis.inner_test stats = { "seed": seed, "evaluations": 0, } def record(kwargs, interesting): if interesting: kwargs = {name: repr(value) for name, value in kwargs.items()} if "original" not in stats: stats["original"] = kwargs stats["shrunk"] = kwargs if "original" in stats: stats["evaluations"] += 1 @proxies(base_function) def replacement_function(**kwargs): try: base_function(**kwargs) record(kwargs, False) except Exception: record(kwargs, True) raise test.hypothesis.inner_test = replacement_function test = with_seed(seed)(settings( database=None, suppress_health_check=HealthCheck.all(), max_examples=10**6, phases=[Phase.generate, Phase.shrink], verbosity=Verbosity.quiet, )(test)) try: test() except Exception: if "original" not in stats: raise results.append(stats) with open(target, "w") as o: o.write(json.dumps( results, sort_keys=True, indent=4, ))
def test_runner(schema_path): schema = loaders.from_path(schema_path, validate_schema=False) runner = from_schema( schema, dry_run=True, count_operations=False, hypothesis_settings=hypothesis.settings( max_examples=1, suppress_health_check=HealthCheck.all(), phases=[Phase.explicit, Phase.generate] ), ) schema_id = get_id(schema_path) def check_xfailed(ev) -> bool: if schema_id in XFAILING and ev.current_operation in XFAILING[schema_id]: if ev.result.errors: message = XFAILING[schema_id][ev.current_operation] # If is is failed for some other reason, then an assertion will be risen return any(message in err.exception_with_traceback for err in ev.result.errors) pytest.fail("Expected a failure") return False def is_unsatisfiable(text): return "Unable to satisfy schema parameters for this API operation" in text def check_flaky(ev) -> bool: if schema_id in FLAKY_SCHEMAS and ev.current_operation in FLAKY_SCHEMAS[schema_id]: if ev.result.errors: # NOTE. There could be other errors if the "Unsatisfiable" case wasn't triggered. # Could be added to expected errors later return any(is_unsatisfiable(err.exception_with_traceback) for err in ev.result.errors) return False def check_unsatisfiable(ev): # In some cases Schemathesis can't generate data - either due to a contradiction within the schema if schema_id in UNSATISFIABLE_SCHEMAS and ev.current_operation in UNSATISFIABLE_SCHEMAS[schema_id]: exception = ev.result.errors[0].exception if ( "Unable to satisfy schema parameters for this API operation" not in exception and "Cannot create non-empty lists with elements" not in exception and "Cannot create a collection of " not in exception ): pytest.fail(f"Expected unsatisfiable, but there is a different error: {exception}") return True return False def check_recursive_references(ev): if schema_id in RECURSIVE_REFERENCES and ev.current_operation in RECURSIVE_REFERENCES[schema_id]: for err in ev.result.errors: if RECURSIVE_REFERENCE_ERROR_MESSAGE in err.exception_with_traceback: # It is OK return True # These errors may be triggered not every time return False def check_not_parsable(ev): return schema_id in NOT_PARSABLE_SCHEMAS and NOT_PARSABLE_SCHEMAS[schema_id] in ev.exception_with_traceback def check_invalid(ev): if schema_id in INVALID_SCHEMAS: if ev.result.errors: return any( "The API schema contains non-string keys" in err.exception_with_traceback for err in ev.result.errors ) return pytest.fail("Expected YAML parsing error") return False for event in runner.execute(): if isinstance(event, events.AfterExecution): if check_xfailed(event): continue if check_flaky(event): continue if check_recursive_references(event): continue if check_unsatisfiable(event): continue if check_invalid(event): continue assert not event.result.has_errors, event.current_operation assert not event.result.has_failures, event.current_operation if isinstance(event, events.InternalError): if check_not_parsable(event): continue assert not isinstance(event, events.InternalError), event.exception_with_traceback
and not unquote(x.get("job_id", "a")).startswith("\n") ) @pytest.mark.usefixtures("nocommit_transaction") @schema.hooks.apply(before_generate_path_parameters) @schema.parametrize(method=["GET", "DELETE"]) def test_api(case, with_auth, auth_token): if with_auth: upd = {"Authorization": f"Bearer {auth_token}"} if case.headers is None: case.headers = upd else: case.headers.update(upd) response = case.call_asgi() case.validate_response(response) @pytest.mark.usefixtures("nocommit_transaction") @schema.parametrize(method=["POST"], operation_id="^(?!.*update.*system.*).*") @hypsettings(max_examples=5, suppress_health_check=HealthCheck.all()) def test_api_post(case, with_auth, auth_token): if with_auth: upd = {"Authorization": f"Bearer {auth_token}"} if case.headers is None: case.headers = upd else: case.headers.update(upd) response = case.call_asgi() case.validate_response(response)
from __future__ import absolute_import, division, print_function from random import Random import pytest from hypothesis import HealthCheck, settings from hypothesis.errors import Flaky from hypothesis.internal.compat import hbytes, hrange from hypothesis.internal.conjecture.data import ConjectureData, Status, StopTest from hypothesis.internal.conjecture.datatree import DataTree from hypothesis.internal.conjecture.engine import ConjectureRunner TEST_SETTINGS = settings( max_examples=5000, database=None, suppress_health_check=HealthCheck.all() ) def runner_for(*examples): if len(examples) == 1 and isinstance(examples[0], list): examples = examples[0] def accept(tf): runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0)) runner.exit_with = lambda reason: None ran_examples = [] for e in examples: e = hbytes(e) data = runner.cached_test_function(e) ran_examples.append((e, data))
def _test_func_is_valid_kernel(kernel, sample_dim, num_samples): """A func is a valid kernel if the kernel matrix generated by it is PSD. Not including this in tests for all kernels to allow for non-PSD kernels in the future """ KM = KernelMatrix(kernel, name='TestKM') KM.attach_to(gen_random_sample(num_samples, sample_dim)) is_psd = is_positive_semidefinite(KM.full, verbose=True) if not is_psd: raise ValueError('{} is not PSD'.format(str(KM))) @hyp_settings(max_examples=num_tests_psd_kernel, deadline=None, timeout=unlimited, suppress_health_check=HealthCheck.all()) @given(strategies.integers(range_feature_dim[0], range_feature_dim[1]), strategies.integers(range_num_samples[0], range_num_samples[1]), strategies.integers(range_polynomial_degree[0], range_polynomial_degree[1]), strategies.floats(min_value=0, max_value=1e3, allow_nan=False, allow_infinity=False)) def test_polynomial_kernel(sample_dim, num_samples, poly_degree, poly_intercept): """Tests specific for Polynomial kernel.""" poly = PolyKernel(degree=poly_degree, b=poly_intercept, skip_input_checks=False) _test_for_all_kernels(poly, sample_dim) _test_func_is_valid_kernel(poly, sample_dim, num_samples) @hyp_settings(max_examples=num_tests_psd_kernel, deadline=None,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. # # END HEADER from __future__ import absolute_import, division, print_function from random import Random from hypothesis import HealthCheck, settings from hypothesis.internal.compat import hbytes from hypothesis.internal.conjecture.data import ConjectureData, Status from hypothesis.internal.conjecture.engine import ConjectureRunner, RunIsComplete TEST_SETTINGS = settings( max_examples=5000, database=None, suppress_health_check=HealthCheck.all() ) def runner_for(*examples): if len(examples) == 1 and isinstance(examples[0], list): examples = examples[0] def accept(tf): runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0)) ran_examples = [] for e in examples: e = hbytes(e) data = ConjectureData.for_buffer(e) try: runner.test_function(data)
buf.pop() try: d = ConjectureData.for_buffer(buf) k = d.draw(st.integers()) stop = d.draw_bits(8) if stop > 0 and k > 0: return (draw(st.integers(0, k - 1)), hbytes(d.buffer)) except (StopTest, IndexError): pass @example((2, b'\x00\x00\n\x01')) @example((1, b'\x00\x00\x06\x01')) @example(problem=(32768, b'\x03\x01\x00\x00\x00\x00\x00\x01\x00\x02\x01')) @settings( suppress_health_check=HealthCheck.all(), timeout=unlimited, deadline=None, max_examples=10, verbosity=Verbosity.normal ) @given(problems()) def test_always_reduces_integers_to_smallest_suitable_sizes(problem): n, blob = problem blob = hbytes(blob) try: d = ConjectureData.for_buffer(blob) k = d.draw(st.integers()) stop = blob[len(d.buffer)] except (StopTest, IndexError): reject() assume(k > n) assume(stop > 0)
# # END HEADER from __future__ import absolute_import, division, print_function import pytest import hypothesis.strategies as st from hypothesis import HealthCheck, Verbosity, given, reject, settings from hypothesis.errors import Unsatisfiable from tests.common.debug import minimal from tests.common.utils import no_shrink @pytest.mark.parametrize("strat", [st.text(min_size=5)]) @settings(phases=no_shrink, deadline=None, suppress_health_check=HealthCheck.all()) @given(st.data()) def test_explore_arbitrary_function(strat, data): cache = {} def predicate(x): try: return cache[x] except KeyError: return cache.setdefault(x, data.draw(st.booleans(), label=repr(x))) try: minimal( strat, predicate, settings=settings(
import ast import dis import io import lib2to3.pgen2.tokenize as lib2to3_tokenize import tokenize import unittest import hypothesmith from hypothesis import HealthCheck, example, given, reject, settings, strategies as st # Used to mark tests which generate arbitrary source code, # because that's a relatively expensive thing to get right. settings.register_profile( "slow", deadline=None, suppress_health_check=HealthCheck.all(), ) class TestAST(unittest.TestCase): @unittest.skipIf(not hasattr(ast, "unparse"), reason="new in 3.9") @given(source_code=hypothesmith.from_grammar()) @settings.get_profile("slow") def test_ast_unparse(self, source_code): """Unparsing always produces code which parses back to the same ast.""" first = ast.parse(source_code) unparsed = ast.unparse(first) second = ast.parse(unparsed) assert ast.dump(first) == ast.dump(second) class TestDis(unittest.TestCase): @given(
from __future__ import division, print_function, absolute_import import pytest import hypothesis.strategies as st from hypothesis import Verbosity, HealthCheck, find, given, reject, \ settings, unlimited from hypothesis.errors import NoSuchExample from tests.common.utils import no_shrink @pytest.mark.parametrize('strat', [st.text(min_size=5)]) @settings( phases=no_shrink, deadline=None, suppress_health_check=HealthCheck.all() ) @given(st.data()) def test_explore_arbitrary_function(strat, data): cache = {} def predicate(x): try: return cache[x] except KeyError: return cache.setdefault(x, data.draw(st.booleans(), label=repr(x))) try: find( strat, predicate, settings=settings(
# See issue https://github.com/HypothesisWorks/hypothesis-python/issues/38 # for details pass Litter = namedtuple('Litter', ('kitten1', 'kitten2')) @given(builds(Litter, integers(), integers())) def test_named_tuples_are_of_right_type(litter): assert isinstance(litter, Litter) @fails_with(AttributeError) @given(integers().map(lambda x: x.nope)) @settings(suppress_health_check=HealthCheck.all()) def test_fails_in_reify(x): pass @given(text(u'a')) def test_a_text(x): assert set(x).issubset(set(u'a')) @given(text(u'')) def test_empty_text(x): assert not x @given(text(u'abcdefg'))