def test_errors_with_did_not_reproduce_if_rejected(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): reject() with pytest.raises(DidNotReproduce): test()
def test_errors_if_provided_example_does_not_reproduce_failure(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): assert x == b with pytest.raises(DidNotReproduce): test()
def test_errors_with_did_not_reproduce_if_the_shape_changes(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n + 1, max_size=n + 1)) def test(x): assert x == b with pytest.raises(DidNotReproduce): test()
def test_raises_invalid_if_wrong_version(): b = b"hello world" n = len(b) @reproduce_failure("1.0.0", encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): pass with pytest.raises(InvalidArgument): test()
def test_reproduces_the_failure(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): assert x != b with pytest.raises(AssertionError): test()
def test_raises_invalid_if_wrong_version(): b = b'hello world' n = len(b) @reproduce_failure('1.0.0', encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): pass with pytest.raises(InvalidArgument): test()
def test_errors_with_did_not_reproduce_if_rejected(): b = b"hello world" n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): reject() with pytest.raises(DidNotReproduce): test()
def test_errors_with_did_not_reproduce_if_the_shape_changes(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n + 1, max_size=n + 1)) def test(x): assert x == b with pytest.raises(DidNotReproduce): test()
def test_errors_if_provided_example_does_not_reproduce_failure(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): assert x == b with pytest.raises(DidNotReproduce): test()
def test_reproduces_the_failure(): b = b'hello world' n = len(b) @reproduce_failure(__version__, encode_failure(b)) @given(st.binary(min_size=n, max_size=n)) def test(x): assert x != b with pytest.raises(AssertionError): test()
def test_function(data): if suppress_intervals: start = data.start_example stop = data.stop_example def start_example(label): if data.depth >= 0: data.depth += 1 if data.depth > data.max_depth: data.max_depth = data.depth else: start(label) def stop_example(discard=False): if data.depth >= 1: data.depth -= 1 else: stop(discard) data.start_example = start_example data.stop_example = stop_example generation_start = time.monotonic() try: try: source = data.draw(experiment.generator) except UnsatisfiedAssumption: data.mark_invalid() finally: generation_time = time.monotonic() - generation_start result = predicate(source) input_to_outputs.append((encode_failure(data.buffer).decode('ascii'), source, result.name)) if trace_memory_usage: display_top(tracemalloc.take_snapshot()) if result == Classification.INTERESTING: data.mark_interesting() elif result in (Classification.INVALIDCHEAP, Classification.INVALIDEXPENSIVE): data.mark_invalid() finally: generation_stats[data.status].record(size=len(data.buffer), runtime=generation_time)
def test_encoding_loop(b): assert decode_failure(encode_failure(b)) == b
def shrink(filename, sizes): with open(filename, 'rb') as i: initial_data = ConjectureData.for_buffer(i.read()) printing_to_stdout = False def data_info(data): name = hashlib.sha1(data.buffer).hexdigest()[:8] input_length = len(data.buffer) output_length = data.extra_information.generated_length return { "name": name, "input_length": input_length, "output_length": output_length, "interesting": data.status == Status.INTERESTING } if sizes is not None: if sizes == "-": writer = sys.stdout printing_to_stdout = True else: writer = open(sizes, 'w') def log_data(data): if data.status >= Status.VALID and hasattr(data.extra_information, 'generated_length'): writer.write(json.dumps(data_info(data)) + "\n") writer.flush() else: writer = None def log_data(data): pass initial = mktemp(suffix='.c') gen(initial_data, initial) errtype, gcc, opt = interesting_reason(initial, printing=not printing_to_stdout) if errtype == 'error': def test_function(data): with tempfile.TemporaryDirectory() as d: prog = os.path.join(d, 'test.c') try: gen(data, prog) data.extra_information.generated_length = os.stat(prog).st_size run_gcc(gcc, opt, prog) except subprocess.TimeoutExpired: return except subprocess.SubprocessError: name = hashlib.sha1(data.buffer).hexdigest()[:8] data.unique_name = name data.mark_interesting() finally: log_data(data) else: assert errtype == "differs" def test_function(data): with tempfile.TemporaryDirectory() as d: prog = os.path.join(d, 'test.c') try: gen(data, prog) data.generated_length = os.stat(prog).st_size if run_gcc(gcc, opt, prog) != run_gcc( '/opt/compiler-explorer/gcc-8.2.0/bin/gcc', '-O0', prog ): name = hashlib.sha1(data.buffer).hexdigest()[:8] data.unique_name = name data.mark_interesting() except subprocess.SubprocessError: pass finally: log_data(data) eng.MAX_SHRINKS = 10 ** 6 with open(filename, 'rb') as i: buffer = i.read() # tracemalloc.start() runner = eng.ConjectureRunner(test_function, settings=settings( database=None, max_examples=1000, timeout=unlimited, suppress_health_check=HealthCheck.all(), deadline=None, verbosity=Verbosity.quiet if printing_to_stdout else Verbosity.debug, buffer_size=BUFFER_SIZE ), random=Random(int.from_bytes(hashlib.sha1(buffer).digest(), 'big'))) class FakeTree(object): def add(self, data): pass def rewrite(self, buffer): return (buffer, None) def generate_novel_prefix(self, random): return b'' is_exhausted = False def uncached_test_function(buffer): data = ConjectureData.for_buffer(buffer) runner.test_function(data) # snapshot = tracemalloc.take_snapshot() # top_stats = snapshot.statistics('lineno') print("[ Top 10 ]") for stat in top_stats[:10]: print(stat) return data.as_result() # runner.tree = FakeTree() # runner.cached_test_function = uncached_test_function # runner.target_selector.add = lambda x: None runner.test_function(ConjectureData.for_buffer(buffer)) assert runner.interesting_examples runner.debug_data = lambda data: runner.debug( f"DATA({getattr(data, 'unique_name', None)}): {len(data.buffer)} bytes, {data.status.name}, {data.interesting_origin}" ) v, = runner.interesting_examples.values() shrinker = runner.new_shrinker( v, lambda d: d.status == Status.INTERESTING and d.interesting_origin == v.interesting_origin ) initial_calls = runner.call_count initial_valid = runner.valid_examples start = time.monotonic() shrinker.shrink() end = time.monotonic() final = data_info(shrinker.shrink_target) final["bytes"] = encode_failure(shrinker.shrink_target.buffer).decode('ascii') if writer is not None: writer.write(json.dumps({ "runtime": end - start, "calls": runner.call_count - initial_calls, "valid": runner.valid_examples - initial_valid, "final": final, }) + "\n") result = runner.interesting_examples[v.interesting_origin] runner.debug_data(result) gen(ConjectureData.for_buffer(result.buffer), 'shrunk.c') if writer is not None: writer.close()
def test_encoding_loop(b): assert decode_failure(encode_failure(b)) == b