class TestPoreDispatchServer(TestCase):
# def setUp(self):
# super().setUp()
# self.flow = flow = models.Flowcell.create(consumable_id="TEST|TEST|TEST", consumable_type="TEST|TEST|TEST", path="TEST/TEST/TEST")
# self.run = models.Run.create(pk=RUN_PK, library_id='x', name=RUN_NAME, flowcell=flow, path="TEST/TEST/TEST")
# self.file = models.File.create(run=self.run, path='TEST/TEST/TEST/TEST', last_modified=datetime.now() - timedelta(hours=2))
# self.tag = models.Tag.create(name='TEST')
# models.TagJunction.create(tag=tag, run=self.run)
# @skip('no test')
@settings(deadline=500, suppress_health_check=(HealthCheck.all()))
@given(ss=Message.Samplesheets())
@with_database
def test_attach_sheet_run_no_run(self, ss):
ut = rpc.PoreRefinerDispatchServer()
strm = AsyncMock()
strm.recv_message.return_value = messages.RunAttachRequest(sheet=ss)
_run(ut.AttachSheetToRun(strm))
strm.send_message.assert_called_once()
@settings(deadline=500, suppress_health_check=(HealthCheck.all()))
@given(ss=Message.Samplesheets())
@with_database
def test_attach_sheet_to_run(self, ss):
# self.flow = flow = models.Flowcell.create(consumable_id="TEST|TEST|TEST", consumable_type="TEST|TEST|TEST", path="TEST/TEST/TEST")
self.run = models.Run.create(pk=RUN_PK, library_id='x', name=RUN_NAME, path="TEST/TEST/TEST")
self.file = models.File.create(run=self.run, path='TEST/TEST/TEST/TEST', last_modified=datetime.now() - timedelta(hours=2))
ut = rpc.PoreRefinerDispatchServer()
strm = AsyncMock()
strm.recv_message.return_value = messages.RunAttachRequest(sheet=ss, id=RUN_PK)
_run(ut.AttachSheetToRun(strm))
strm.send_message.assert_called_once()
def test_test(r):
print(r)
dfa = regex_to_DFA(r)
@settings(suppress_health_check=HealthCheck.all())
@given(s=positive(dfa, max_size=30))
def test_positive(s):
t = ''.join(s)
if print_it: print(' ', t)
# Check if t recognized by r.
assert dfa.accepts(t), \
'Generated string \'{}\' does not match regexp {}'.format(t, str(r))
assert re.match(r"^"+str(r)+"$", t), \
"Generated string \"{}\" does not match regexp {}".format(t, str(r))
@settings(suppress_health_check=HealthCheck.all())
@given(s=negative(dfa, max_size=30))
def test_negative(s):
t = ''.join(s)
if print_it: print(' ', t)
# Check if t not recognized by r.
#re.match does not work here (tries to match first instance)
assert not dfa.accepts(t), \
'Generated string \'{}\' does matches regexp {}'.format(t, str(r))
#assert not re.match(r"^("+str(r)+")$", t), \
# "Generated string \"{}\" does matches regexp {}".format(t, str(r))
test_positive()
test_negative()
def test_min_size_equal_to_max_size():
# you can't fit two codons in 3 bases
with pytest.raises(ValueError):
minimal(
cds(
min_size=3,
max_size=3,
include_start_codon=True,
include_stop_codon=True,
))
# but you can fit one
assert (minimal(
cds(
min_size=3,
max_size=3,
include_start_codon=True,
include_stop_codon=False,
)) == "ATA")
assert (minimal(
cds(
min_size=3,
max_size=3,
include_start_codon=False,
include_stop_codon=True,
)) == "AGA")
# non mod 3 specific sizes won't work
with pytest.raises(Unsatisfiable):
@given(cds(min_size=7, max_size=7))
@settings(suppress_health_check=HealthCheck.all())
def inner(x):
pass
inner()
with pytest.raises(Unsatisfiable):
@given(cds(min_size=8, max_size=8))
@settings(suppress_health_check=HealthCheck.all())
def inner(x):
pass
inner()
# if start and stop codons are included, there is no other codon in a 6-mer
assert minimal(cds(min_size=6, max_size=6)) == "ATAAGA"
def test_database_clears_secondary_key():
key = b"key"
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) == 10:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=1,
buffer_size=1024,
database=database,
suppress_health_check=HealthCheck.all(),
),
database_key=key,
)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.cached_test_function([10])
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 0
def test_contains_the_test_function_name_in_the_exception_string():
look_for_one = settings(max_examples=1,
suppress_health_check=HealthCheck.all())
@given(st.integers())
@look_for_one
def this_has_a_totally_unique_name(x):
reject()
with pytest.raises(Unsatisfiable,
match=re.escape(
this_has_a_totally_unique_name.__name__)):
this_has_a_totally_unique_name()
class Foo:
@given(st.integers())
@look_for_one
def this_has_a_unique_name_and_lives_on_a_class(self, x):
reject()
with pytest.raises(
Unsatisfiable,
match=re.escape(
Foo.this_has_a_unique_name_and_lives_on_a_class.__name__),
):
Foo().this_has_a_unique_name_and_lives_on_a_class()
class TestRun(TestCase):
@skip('broken')
@given(pk=sql_ints(),
name=strat.text(),
library_id=strat.text(),
alt_name=strat.text(),
run_id=strat.text(),
started=strat.datetimes().filter(lambda d: d < datetime.now()),
ended=strat.datetimes().filter(lambda d: d > datetime.now()),
path=paths(),
basecalling_model=strat.one_of(
*[strat.just(val) for val, _ in models.Run.basecallers]))
@with_database
def test_run(self, **kwargs):
self.flow = models.Flowcell.create(consumable_id='TEST',
consumable_type='TEST',
path='TEST/TEST')
assert models.Run.create(flowcell=self.flow, **kwargs).run_duration
@settings(deadline=None, suppress_health_check=(HealthCheck.all()))
@given(run=Model.Runs(), job=_jobs())
@with_database
def test_job_spawn(self, run, job):
# run.flowcell.save()
run.save()
self.assertIsNotNone(run.pk)
jobb = run.spawn(job)
self.assertIs(job, jobb.job_state)
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=5000,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug,
),
)
runner.cached_test_function(buf)
runner.run()
out, _ = capsys.readouterr()
assert re.match(u"\\d+ bytes \\[.*\\] -> ", out)
assert "INTERESTING" in out
def test_down_samples_the_pareto_front():
with deterministic_PRNG():
def test(data):
data.draw_bits(8)
data.draw_bits(8)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=1000,
database=db,
suppress_health_check=HealthCheck.all(),
phases=[Phase.reuse],
),
database_key=b"stuff",
)
for i in range(10000):
db.save(runner.pareto_key, int_to_bytes(i, 2))
runner.reuse_existing_examples()
assert 0 < runner.valid_examples <= 100
def test_stops_loading_pareto_front_if_interesting():
with deterministic_PRNG():
def test(data):
data.draw_bits(8)
data.draw_bits(8)
data.mark_interesting()
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=1000,
database=db,
suppress_health_check=HealthCheck.all(),
phases=[Phase.reuse],
),
database_key=b"stuff",
)
for i in range(10000):
db.save(runner.pareto_key, int_to_bytes(i, 2))
runner.reuse_existing_examples()
assert runner.call_count == 1
def test_database_clears_secondary_key():
key = b"key"
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) == 10:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=1,
buffer_size=1024,
database=database,
suppress_health_check=HealthCheck.all(),
),
database_key=key,
)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.cached_test_function([10])
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 0
def test_failure_sequence_inducing(building, testing, rnd):
buildit = iter(building)
testit = iter(testing)
def build(x):
try:
assume(not next(buildit))
except StopIteration:
pass
return x
@given(integers().map(build))
@settings(verbosity=Verbosity.quiet,
database=None,
suppress_health_check=HealthCheck.all(),
phases=no_shrink)
def test(x):
try:
i = next(testit)
except StopIteration:
return
if i == 1:
return
elif i == 2:
reject()
else:
raise Nope()
try:
test()
except (Nope, Unsatisfiable, Flaky):
pass
except UnsatisfiedAssumption:
raise SatisfyMe()
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(
f,
random=Random(0),
settings=settings(
suppress_health_check=HealthCheck.all(),
phases=(Phase.shrink, ),
database=None,
verbosity=Verbosity.debug,
),
database_key=None,
)
runner.cached_test_function(blob)
assert runner.interesting_examples
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.fixate_shrink_passes(["minimize_individual_blocks"])
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed])
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
def test_returning_non_none_does_not_fail_if_health_check_disabled():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all())
def a(x):
return 1
a()
def test_failure_sequence_inducing(building, testing, rnd):
buildit = iter(building)
testit = iter(testing)
def build(x):
try:
assume(not next(buildit))
except StopIteration:
pass
return x
@given(integers().map(build))
@settings(
verbosity=Verbosity.quiet, database=None,
suppress_health_check=HealthCheck.all(), phases=no_shrink
)
def test(x):
try:
i = next(testit)
except StopIteration:
return
if i == 1:
return
elif i == 2:
reject()
else:
raise Nope()
try:
test()
except (Nope, Unsatisfiable, Flaky):
pass
except UnsatisfiedAssumption:
raise SatisfyMe()
def test_the_slow_test_health_only_runs_if_health_checks_are_on():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all(), deadline=None)
def a(x):
time.sleep(1000)
a()
def test_clears_defunct_pareto_front():
with deterministic_PRNG():
def test(data):
data.draw_bits(8)
data.draw_bits(8)
db = InMemoryExampleDatabase()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=10000,
database=db,
suppress_health_check=HealthCheck.all(),
phases=[Phase.reuse],
),
database_key=b"stuff",
)
for i in range(256):
db.save(runner.pareto_key, bytes([i, 0]))
runner.run()
assert len(list(db.fetch(runner.pareto_key))) == 1
def test_debug_data(capsys):
buf = [0, 1, 2]
def f(data):
for x in hbytes(buf):
if data.draw_bits(8) != x:
data.mark_invalid()
data.start_example(1)
data.stop_example()
data.mark_interesting()
runner = ConjectureRunner(
f,
settings=settings(
max_examples=5000,
buffer_size=1024,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.debug,
),
)
runner.cached_test_function(buf)
runner.run()
out, _ = capsys.readouterr()
assert re.match(u"\\d+ bytes \\[.*\\] -> ", out)
assert "INTERESTING" in out
def run_language_test_for(root, data, seed):
random.seed(seed)
def test(local_data):
node = root
while not isinstance(node, Terminal):
if isinstance(node, Write):
local_data.write(hbytes(node.value))
node = node.child
else:
assert isinstance(node, Branch)
c = local_data.draw_bits(node.bits)
try:
node = node.children[c]
except KeyError:
if data is None:
return
node = node.children.setdefault(c, data.draw(nodes))
assert isinstance(node, Terminal)
if node.status == Status.INTERESTING:
local_data.mark_interesting(node.payload)
elif node.status == Status.INVALID:
local_data.mark_invalid()
runner = ConjectureRunner(test, settings=settings(
max_examples=1, max_shrinks=100, buffer_size=512,
database=None, suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.quiet, phases=list(Phase),
))
try:
runner.run()
finally:
if data is not None:
note(root)
assume(runner.interesting_examples)
def test_contains_the_test_function_name_in_the_exception_string():
look_for_one = settings(
max_examples=1, suppress_health_check=HealthCheck.all())
@given(integers())
@look_for_one
def this_has_a_totally_unique_name(x):
reject()
with raises(Unsatisfiable) as e:
this_has_a_totally_unique_name()
assert this_has_a_totally_unique_name.__name__ in e.value.args[0]
class Foo(object):
@given(integers())
@look_for_one
def this_has_a_unique_name_and_lives_on_a_class(self, x):
reject()
with raises(Unsatisfiable) as e:
Foo().this_has_a_unique_name_and_lives_on_a_class()
assert (
Foo.this_has_a_unique_name_and_lives_on_a_class.__name__
) in e.value.args[0]
def test_the_slow_test_health_only_runs_if_health_checks_are_on():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all(), deadline=None)
def a(x):
time.sleep(1000)
a()
def test_database_uses_values_from_secondary_key():
key = b'key'
database = InMemoryExampleDatabase()
def f(data):
if data.draw_bits(8) >= 5:
data.mark_interesting()
else:
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, buffer_size=1024,
database=database, suppress_health_check=HealthCheck.all(),
), database_key=key)
for i in range(10):
database.save(runner.secondary_key, hbytes([i]))
runner.test_function(ConjectureData.for_buffer(hbytes([10])))
assert runner.interesting_examples
assert len(set(database.fetch(key))) == 1
assert len(set(database.fetch(runner.secondary_key))) == 10
runner.clear_secondary_key()
assert len(set(database.fetch(key))) == 1
assert set(
map(int_from_bytes, database.fetch(runner.secondary_key))
) == set(range(6, 11))
v, = runner.interesting_examples.values()
assert list(v.buffer) == [5]
def test_returning_non_none_does_not_fail_if_health_check_disabled():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all())
def a(x):
return 1
a()
class OasisLookupFactoryWriteJsonFiles(TestCase):
@settings(deadline=None, suppress_health_check=HealthCheck.all())
@given(
successes=keys(from_statuses=just(OASIS_KEYS_STATUS['success']['id']),
size=5),
nonsuccesses=keys(from_statuses=sampled_from([
OASIS_KEYS_STATUS['fail']['id'], OASIS_KEYS_STATUS['nomatch']['id']
]),
size=5))
def test_records_are_given___records_are_written_to_json_keys_files_correctly(
self, successes, nonsuccesses):
with TemporaryDirectory() as d:
keys_file_path = os.path.join(d, 'keys.json')
keys_errors_file_path = os.path.join(d, 'keys-errors.json')
result = pd.DataFrame(successes + nonsuccesses)
key_server = BasicKeyServer(
{'builtin_lookup_type': 'deterministic'})
_, successes_count, _, nonsuccesses_count = key_server.write_keys_file(
[result],
successes_fp=keys_file_path,
errors_fp=keys_errors_file_path,
output_format='json',
keys_success_msg=False)
with io.open(keys_file_path, 'r', encoding='utf-8') as f1, io.open(
keys_errors_file_path, 'r', encoding='utf-8') as f2:
written_successes = json.load(f1)
written_nonsuccesses = json.load(f2)
self.assertEqual(successes_count, len(successes))
self.assertEqual(written_successes, successes)
self.assertEqual(nonsuccesses_count, len(nonsuccesses))
self.assertEqual(written_nonsuccesses, nonsuccesses)
def test_raises_unsatisfiable_if_all_false():
@given(integers())
@settings(max_examples=50, suppress_health_check=HealthCheck.all())
def test_assume_false(x):
reject()
with pytest.raises(Unsatisfiable):
test_assume_false()
def test_notes_hard_to_satisfy():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all())
def test(i):
assume(i == 0)
stats = call_for_statistics(test)
assert 'satisfied assumptions' in stats.exit_reason
def test_notes_hard_to_satisfy():
@given(st.integers())
@settings(suppress_health_check=HealthCheck.all())
def test(i):
assume(i == 13)
stats = call_for_statistics(test)
assert "satisfied assumptions" in stats.exit_reason
class LocalTest(TestCase):
@given(integers().map(break_the_db))
@settings(suppress_health_check=HealthCheck.all())
def test_does_not_break_other_things(self, unused):
pass
def test_normal_test_1(self):
Company.objects.create(name=u"MickeyCo")
def run_to_buffer(f):
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return hbytes(last_data.buffer)
def test_default_health_check_can_weaken_specific():
import random
@settings(suppress_health_check=HealthCheck.all())
@given(st.lists(st.integers(), min_size=1))
def test(x):
random.choice(x)
test()
def test_default_health_check_can_weaken_specific():
import random
@given(st.lists(st.integers(), min_size=1))
def test(x):
random.choice(x)
with settings(suppress_health_check=HealthCheck.all()):
test()
def test_default_health_check_can_weaken_specific():
import random
@settings(HEALTH_CHECK_SETTINGS, suppress_health_check=HealthCheck.all())
@given(st.lists(st.integers(), min_size=1))
def test(x):
random.choice(x)
test()
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
def f(data):
k = data.draw(st.integers())
data.output = repr(k)
if data.draw_bits(8) == stop and k >= n:
data.mark_interesting()
runner = ConjectureRunner(f, random=Random(0), settings=settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited,
phases=(Phase.shrink,), database=None, verbosity=Verbosity.debug
), database_key=None)
runner.test_function(ConjectureData.for_buffer(blob))
assert runner.interesting_examples
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(v, lambda x: x.status == Status.INTERESTING)
shrinker.clear_passes()
shrinker.add_new_pass('minimize_individual_blocks')
shrinker.shrink()
v = shrinker.shrink_target
m = ConjectureData.for_buffer(v.buffer).draw(st.integers())
assert m == n
# Upper bound on the length needed is calculated as follows:
# * We have an initial byte at the beginning to decide the length of the
# integer.
# * We have a terminal byte as the stop value.
# * The rest is the integer payload. This should be n. Including the sign
# bit, n needs (1 + n.bit_length()) / 8 bytes (rounded up). But we only
# have power of two sizes, so it may be up to a factor of two more than
# that.
bits_needed = 1 + n.bit_length()
actual_bits_needed = min(
[s for s in WideRangeIntStrategy.sizes if s >= bits_needed])
bytes_needed = actual_bits_needed // 8
# 3 extra bytes: two for the sampler, one for the capping value.
assert len(v.buffer) == 3 + bytes_needed
def run_to_buffer(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return hbytes(last_data.buffer)
def test(dfa, test_only=None):
all_positive = []
@settings(suppress_health_check=HealthCheck.all())
@given(s=positive(dfa))
#@settings(phases=[Phase.generate, Phase.shrink])
def test_positive(s):
all_positive.append("".join(s))
assert dfa.accepts(s)
all_negative = []
@settings(suppress_health_check=HealthCheck.all())
@given(s=negative(dfa))
#@settings(phases=[Phase.generate, Phase.shrink])
def test_negative(s):
all_negative.append("".join(s))
assert not dfa.accepts(s)
if test_only != "negative":
print('Some positive test-cases:')
test_positive()
c = Counter(all_positive)
for k in sorted(c):
print("{}: {} times".format(k, c[k]))
print(len(all_positive), "test-case(s) generated")
print(len(set(all_positive)), "unique test-case(s)")
print(max(map(len, all_positive)), "max length")
print(sum(map(len, all_positive)) / len(all_positive), "avg length")
print()
if test_only != "positive":
print('Some negative test-cases:')
test_negative()
c = Counter(all_negative)
for k in sorted(c):
print("{}: {} times".format(k, c[k]))
print(len(all_negative), "test-case(s) generated")
print(len(set(all_negative)), "unique test-case(s)")
print(max(map(len, all_negative)), "max length")
print(sum(map(len, all_negative)) / len(all_negative), "avg length")
print()
def accept(f):
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.test_function(ConjectureData.for_buffer(start))
assert runner.interesting_examples
last_data, = runner.interesting_examples.values()
return runner.new_shrinker(
last_data, lambda d: d.status == Status.INTERESTING
)
class TestPlistlib(unittest.TestCase):
@settings(suppress_health_check=HealthCheck.all())
@given(
payload=plistlib_data,
fmt=st.sampled_from([plistlib.FMT_XML, plistlib.FMT_BINARY]),
pass_format_arg=st.booleans(),
)
def test_dumps_loads(self, payload, fmt, pass_format_arg):
plist_dump = plistlib.dumps(payload, fmt=fmt)
plist_load = plistlib.loads(plist_dump,
fmt=fmt if pass_format_arg else None)
self.assertEqual(payload, plist_load)
def accept(test):
verbose = os.environ.get('VERBOSE') == 'true'
shrink = os.environ.get('SHRINK', 'true') == 'true'
provided_seed = int(os.environ['SEED'])
if verbose:
verbosity = Verbosity.debug
else:
verbosity = Verbosity.normal
seen_failure = False
evaluations = 0
original_test = test
@proxies(test)
def recording_test(*args, **kwargs):
nonlocal seen_failure, evaluations
if seen_failure:
evaluations += 1
try:
return original_test(*args, **kwargs)
except HypothesisException:
raise
except Exception:
if not seen_failure:
seen_failure = True
evaluations += 1
raise
test = seed(provided_seed)(settings(
database=None,
max_examples=10**6,
suppress_health_check=HealthCheck.all(),
verbosity=verbosity,
phases=[Phase.generate, Phase.shrink]
if shrink else [Phase.generate])(given(strat)(recording_test)))
try:
test()
except Exception:
if verbose:
traceback.print_exc()
print("EVALUATIONS:", evaluations)
print("TEST FAILED")
return
print("Expected test to fail with assertion error",
file=sys.stderr)
sys.exit(1)
def test_exit_because_max_iterations():
def f(data):
data.draw_bits(64)
data.mark_invalid()
runner = ConjectureRunner(f, settings=settings(
max_examples=1, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.call_count <= 1000
assert runner.exit_reason == ExitReason.max_iterations
def test_can_increase_number_of_bytes_drawn_in_tail():
# This is designed to trigger a case where the zero bound queue will end up
# increasing the size of data drawn because moving zeroes into the initial
# prefix will increase the amount drawn.
def f(data):
x = data.draw_bytes(5)
n = x.count(0)
b = data.draw_bytes(n + 1)
assert not any(b)
runner = ConjectureRunner(
f, settings=settings(
buffer_size=11, suppress_health_check=HealthCheck.all()))
runner.run()
def test_large_initial_write():
big = hbytes(b'\xff') * 512
def f(data):
data.write(big)
data.draw_bits(63)
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
max_examples=5000, buffer_size=1024,
database=None, suppress_health_check=HealthCheck.all(),
))
runner.run()
assert runner.exit_reason == ExitReason.finished
def test_can_increase_number_of_bytes_drawn_in_tail():
# This is designed to trigger a case where the zero bound queue will end up
# increasing the size of data drawn because moving zeroes into the initial
# prefix will increase the amount drawn.
def f(data):
x = data.draw_bytes(5)
n = x.count(0)
b = data.draw_bytes(n + 1)
assert not any(b)
runner = ConjectureRunner(
f, settings=settings(
max_examples=100,
buffer_size=11, suppress_health_check=HealthCheck.all()))
runner.run()
def test_will_reset_the_tree_as_it_goes(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
data.draw_bits(8)
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree[0]) > 1
step(2)
assert len(runner.tree[0]) == 1
def run_language_test_for(root, data, seed):
random.seed(seed)
def test(local_data):
node = root
while not isinstance(node, Terminal):
if isinstance(node, Write):
local_data.write(hbytes(node.value))
node = node.child
else:
assert isinstance(node, Branch)
c = local_data.draw_bits(node.bits)
try:
node = node.children[c]
except KeyError:
if data is None:
return
node = node.children.setdefault(c, data.draw(nodes))
assert isinstance(node, Terminal)
if node.status == Status.INTERESTING:
local_data.mark_interesting(node.payload)
elif node.status == Status.INVALID:
local_data.mark_invalid()
runner = ConjectureRunner(
test,
settings=settings(
max_examples=1,
buffer_size=512,
database=None,
suppress_health_check=HealthCheck.all(),
verbosity=Verbosity.quiet,
phases=list(Phase),
),
)
try:
runner.run()
finally:
if data is not None:
note(root)
assume(runner.interesting_examples)
def test_will_not_reset_the_tree_after_interesting_example(monkeypatch):
monkeypatch.setattr(engine_module, 'CACHE_RESET_FREQUENCY', 3)
def f(data):
if data.draw_bits(8) == 7:
data.mark_interesting()
with deterministic_PRNG():
runner = ConjectureRunner(f, settings=settings(
database=None, suppress_health_check=HealthCheck.all(),
))
def step(n):
runner.test_function(ConjectureData.for_buffer([n]))
step(0)
step(1)
assert len(runner.tree) > 1
step(7)
assert len(runner.tree) > 1
t = len(runner.tree)
runner.shrink_interesting_examples()
assert len(runner.tree) > t
buf.pop()
try:
d = ConjectureData.for_buffer(buf)
k = d.draw(st.integers())
stop = d.draw_bits(8)
if stop > 0 and k > 0:
return (draw(st.integers(0, k - 1)), hbytes(d.buffer))
except (StopTest, IndexError):
pass
@example((2, b'\x00\x00\n\x01'))
@example((1, b'\x00\x00\x06\x01'))
@example(problem=(32768, b'\x03\x01\x00\x00\x00\x00\x00\x01\x00\x02\x01'))
@settings(
suppress_health_check=HealthCheck.all(), timeout=unlimited, deadline=None,
max_examples=10, verbosity=Verbosity.normal
)
@given(problems())
def test_always_reduces_integers_to_smallest_suitable_sizes(problem):
n, blob = problem
blob = hbytes(blob)
try:
d = ConjectureData.for_buffer(blob)
k = d.draw(st.integers())
stop = blob[len(d.buffer)]
except (StopTest, IndexError):
reject()
assume(k > n)
assume(stop > 0)
# We draw n as two separate calls so that it doesn't show up as a
# single block. If it did, the heuristics that allow us to move
# blocks around would fire and it would move right, which would
# then allow us to shrink it more easily.
n = (data.draw_bits(16) << 16) | data.draw_bits(16)
if n == MAX_INT:
return (POISON,)
else:
return (None,)
LOTS = 10 ** 6
TEST_SETTINGS = settings(
database=None, suppress_health_check=HealthCheck.all(), max_examples=LOTS,
deadline=None, timeout=unlimited
)
@pytest.mark.parametrize('size', [2, 5, 10])
@pytest.mark.parametrize('seed', [0, 15993493061449915028])
def test_can_reduce_poison_from_any_subtree(size, seed):
"""This test validates that we can minimize to any leaf node of a binary
tree, regardless of where in the tree the leaf is."""
random = Random(seed)
# Initially we create the minimal tree of size n, regardless of whether it
# is poisoned (which it won't be - the poison event essentially never
# happens when drawing uniformly at random).
def weights(draw):
parts = draw(st.lists(st.integers()))
parts.reverse()
base = Fraction(1, 1)
for p in parts:
base = Fraction(1) / (1 + base)
return base
@example([Fraction(1, 3), Fraction(1, 3), Fraction(1, 3)])
@example([Fraction(1, 1), Fraction(1, 2)])
@example([Fraction(1, 2), Fraction(4, 10)])
@example([Fraction(1, 1), Fraction(3, 5), Fraction(1, 1)])
@example([Fraction(2, 257), Fraction(2, 5), Fraction(1, 11)])
@settings(
deadline=None, suppress_health_check=HealthCheck.all(),
max_examples=0 if IN_COVERAGE_TESTS else settings.default.max_examples,
)
@given(st.lists(weights(), min_size=1))
def test_sampler_distribution(weights):
total = sum(weights)
n = len(weights)
assume(total > 0)
probabilities = [w / total for w in weights]
sampler = cu.Sampler(weights)
calculated = [Fraction(0)] * n
for base, alternate, p_alternate in sampler.table:
ls = minimal(
st.lists(st.recursive(
st.booleans(),
lambda x: st.lists(
x, min_size=2 * (target - 1), max_size=2 * target
).map(tuple),
max_leaves=2 * target - 1)),
lambda x: len(set(x)) >= target,
timeout_after=None
)
assert len(ls) == target
@given(st.randoms())
@settings(
max_examples=50, phases=no_shrink, suppress_health_check=HealthCheck.all(),
deadline=None
)
def test_can_use_recursive_data_in_sets(rnd):
nested_sets = st.recursive(st.booleans(), st.frozensets, max_leaves=3)
find_any(nested_sets, random=rnd)
def flatten(x):
if isinstance(x, bool):
return frozenset((x,))
else:
result = frozenset()
for t in x:
result |= flatten(t)
if len(result) == 2:
break
def shrink(filename, sizes):
with open(filename, 'rb') as i:
initial_data = ConjectureData.for_buffer(i.read())
printing_to_stdout = False
def data_info(data):
name = hashlib.sha1(data.buffer).hexdigest()[:8]
input_length = len(data.buffer)
output_length = data.extra_information.generated_length
return {
"name": name, "input_length": input_length, "output_length": output_length,
"interesting": data.status == Status.INTERESTING
}
if sizes is not None:
if sizes == "-":
writer = sys.stdout
printing_to_stdout = True
else:
writer = open(sizes, 'w')
def log_data(data):
if data.status >= Status.VALID and hasattr(data.extra_information, 'generated_length'):
writer.write(json.dumps(data_info(data)) + "\n")
writer.flush()
else:
writer = None
def log_data(data):
pass
initial = mktemp(suffix='.c')
gen(initial_data, initial)
errtype, gcc, opt = interesting_reason(initial, printing=not printing_to_stdout)
if errtype == 'error':
def test_function(data):
with tempfile.TemporaryDirectory() as d:
prog = os.path.join(d, 'test.c')
try:
gen(data, prog)
data.extra_information.generated_length = os.stat(prog).st_size
run_gcc(gcc, opt, prog)
except subprocess.TimeoutExpired:
return
except subprocess.SubprocessError:
name = hashlib.sha1(data.buffer).hexdigest()[:8]
data.unique_name = name
data.mark_interesting()
finally:
log_data(data)
else:
assert errtype == "differs"
def test_function(data):
with tempfile.TemporaryDirectory() as d:
prog = os.path.join(d, 'test.c')
try:
gen(data, prog)
data.generated_length = os.stat(prog).st_size
if run_gcc(gcc, opt, prog) != run_gcc(
'/opt/compiler-explorer/gcc-8.2.0/bin/gcc', '-O0', prog
):
name = hashlib.sha1(data.buffer).hexdigest()[:8]
data.unique_name = name
data.mark_interesting()
except subprocess.SubprocessError:
pass
finally:
log_data(data)
eng.MAX_SHRINKS = 10 ** 6
with open(filename, 'rb') as i:
buffer = i.read()
# tracemalloc.start()
runner = eng.ConjectureRunner(test_function, settings=settings(
database=None,
max_examples=1000,
timeout=unlimited, suppress_health_check=HealthCheck.all(),
deadline=None,
verbosity=Verbosity.quiet if printing_to_stdout else Verbosity.debug,
buffer_size=BUFFER_SIZE
), random=Random(int.from_bytes(hashlib.sha1(buffer).digest(), 'big')))
class FakeTree(object):
def add(self, data):
pass
def rewrite(self, buffer):
return (buffer, None)
def generate_novel_prefix(self, random):
return b''
is_exhausted = False
def uncached_test_function(buffer):
data = ConjectureData.for_buffer(buffer)
runner.test_function(data)
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.statistics('lineno')
print("[ Top 10 ]")
for stat in top_stats[:10]:
print(stat)
return data.as_result()
# runner.tree = FakeTree()
# runner.cached_test_function = uncached_test_function
# runner.target_selector.add = lambda x: None
runner.test_function(ConjectureData.for_buffer(buffer))
assert runner.interesting_examples
runner.debug_data = lambda data: runner.debug(
f"DATA({getattr(data, 'unique_name', None)}): {len(data.buffer)} bytes, {data.status.name}, {data.interesting_origin}"
)
v, = runner.interesting_examples.values()
shrinker = runner.new_shrinker(
v, lambda d: d.status == Status.INTERESTING and d.interesting_origin == v.interesting_origin
)
initial_calls = runner.call_count
initial_valid = runner.valid_examples
start = time.monotonic()
shrinker.shrink()
end = time.monotonic()
final = data_info(shrinker.shrink_target)
final["bytes"] = encode_failure(shrinker.shrink_target.buffer).decode('ascii')
if writer is not None:
writer.write(json.dumps({
"runtime": end - start,
"calls": runner.call_count - initial_calls,
"valid": runner.valid_examples - initial_valid,
"final": final,
}) + "\n")
result = runner.interesting_examples[v.interesting_origin]
runner.debug_data(result)
gen(ConjectureData.for_buffer(result.buffer), 'shrunk.c')
if writer is not None:
writer.close()
# obtain one at https://mozilla.org/MPL/2.0/.
#
# END HEADER
from __future__ import absolute_import, division, print_function
import pytest
import hypothesis.strategies as st
from hypothesis import HealthCheck, Verbosity, find, given, reject, settings
from hypothesis.errors import NoSuchExample
from tests.common.utils import no_shrink
@pytest.mark.parametrize("strat", [st.text(min_size=5)])
@settings(phases=no_shrink, deadline=None, suppress_health_check=HealthCheck.all())
@given(st.data())
def test_explore_arbitrary_function(strat, data):
cache = {}
def predicate(x):
try:
return cache[x]
except KeyError:
return cache.setdefault(x, data.draw(st.booleans(), label=repr(x)))
try:
find(
strat,
predicate,
settings=settings(
from __future__ import absolute_import, division, print_function
from random import Random
import pytest
from hypothesis import HealthCheck, settings
from hypothesis.errors import Flaky
from hypothesis.internal.compat import hbytes, hrange
from hypothesis.internal.conjecture.data import ConjectureData, Status, StopTest
from hypothesis.internal.conjecture.datatree import DataTree
from hypothesis.internal.conjecture.engine import ConjectureRunner
TEST_SETTINGS = settings(
max_examples=5000, database=None, suppress_health_check=HealthCheck.all()
)
def runner_for(*examples):
if len(examples) == 1 and isinstance(examples[0], list):
examples = examples[0]
def accept(tf):
runner = ConjectureRunner(tf, settings=TEST_SETTINGS, random=Random(0))
runner.exit_with = lambda reason: None
ran_examples = []
for e in examples:
e = hbytes(e)
data = runner.cached_test_function(e)
ran_examples.append((e, data))
from hypothesis.internal.conjecture.engine import ConjectureRunner
@st.composite
def problem(draw):
b = hbytes(draw(st.binary(min_size=1, max_size=8)))
m = int_from_bytes(b) * 256
assume(m > 0)
marker = draw(st.binary(max_size=8))
bound = draw(st.integers(0, m - 1))
return (b, marker, bound)
base_settings = settings(
database=None,
deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10,
verbosity=Verbosity.normal, timeout=unlimited,
phases=(
Phase.explicit,
Phase.generate
)
)
@example((b'\x10\x00\x00\x00\x00\x00', b'', 2861143707951135))
@example((b'\x05Cn', b'%\x1b\xa0\xfa', 12394667))
@example((b'\x179 f', b'\xf5|', 24300326997))
@example((b'\x05*\xf5\xe5\nh', b'', 1076887621690235))
@example((b'=', b'', 2508))
@example((b'\x01\x00', b'', 20048))
@example((b'\x01', b'', 0))
from __future__ import division, print_function, absolute_import
import pytest
import hypothesis.strategies as st
from hypothesis import Verbosity, HealthCheck, find, given, reject, \
settings, unlimited
from hypothesis.errors import NoSuchExample
from tests.common.utils import no_shrink
@pytest.mark.parametrize('strat', [st.text(min_size=5)])
@settings(
phases=no_shrink, deadline=None,
suppress_health_check=HealthCheck.all()
)
@given(st.data())
def test_explore_arbitrary_function(strat, data):
cache = {}
def predicate(x):
try:
return cache[x]
except KeyError:
return cache.setdefault(x, data.draw(st.booleans(), label=repr(x)))
try:
find(
strat, predicate,
settings=settings(
# See issue https://github.com/HypothesisWorks/hypothesis-python/issues/38
# for details
pass
Litter = namedtuple('Litter', ('kitten1', 'kitten2'))
@given(builds(Litter, integers(), integers()))
def test_named_tuples_are_of_right_type(litter):
assert isinstance(litter, Litter)
@fails_with(AttributeError)
@given(integers().map(lambda x: x.nope))
@settings(suppress_health_check=HealthCheck.all())
def test_fails_in_reify(x):
pass
@given(text(u'a'))
def test_a_text(x):
assert set(x).issubset(set(u'a'))
@given(text(u''))
def test_empty_text(x):
assert not x
@given(text(u'abcdefg'))
