def test_given_can_infer_on_py2():
# Editing annotations before decorating is hilariously awkward, but works!
def inner(a):
pass
inner.__annotations__ = {"a": int}
given(a=infer)(inner)()
def test_given_can_infer_from_manual_annotations(infer_token):
# Editing annotations before decorating is hilariously awkward, but works!
def inner(a):
assert isinstance(a, int)
inner.__annotations__ = {"a": int}
given(a=infer_token)(inner)()
def test_given_can_infer_on_py2():
# Editing annotations before decorating is hilariously awkward, but works!
def inner(a):
pass
inner.__annotations__ = {'a': int}
given(a=infer)(inner)()
def test_can_generate_specified_version(function):
try:
given(st.integers(), st.integers())(settings(database=None)(function))()
except MultipleFailures:
pass
else:
raise AssertionError("Expected MultipleFailures")
def test_given_can_infer_from_manual_annotations():
# Editing annotations before decorating is hilariously awkward, but works!
def inner(a):
pass
inner.__annotations__ = {"a": int}
given(a=infer)(inner)()
def accept(f):
# Workaround for https://github.com/DRMacIver/hypothesis/issues/206
# Fixed in version 1.13 (released 2015 october 29th)
f.__module__ = '__anon__'
try:
given(*args, settings=Settings(max_examples=2000), **kwargs)(f)()
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def accept(f):
# Workaround for https://github.com/DRMacIver/hypothesis/issues/206
# Fixed in version 1.13 (released 2015 october 29th)
f.__module__ = '__anon__'
try:
given(*args, settings=Settings(max_examples=2000), **kwargs)(f)()
except Exception:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def test_inline_given_handles_self():
# Regression test for https://github.com/HypothesisWorks/hypothesis/issues/961
class Cls:
def method(self, **kwargs):
assert isinstance(self, Cls)
assert kwargs["k"] is sentinel
sentinel = object()
given(k=st.just(sentinel))(Cls().method)()
def valid(self=None):
nonlocal func, args, kwonlyargs
# infer can only be applied to keywords, so convert the positionals to kws
newargs = {arg: infer for arg in args if arg != "self"}
kwargs = {kw: infer for kw in kwonlyargs}
args = {**newargs, **kwargs}
if self:
return given(**args)(func)(self)
else:
return given(**args)(func)()
def wrapper(f):
hyp_func = hy.given(*given_args, **given_kwargs)(f)
fixed_seed_func = hy.seed(0)(hy.settings(max_examples=1)(hy.given(
*given_args, **given_kwargs)(f)))
def func(self, *args, **kwargs):
self.should_serialize = True
fixed_seed_func(self, *args, **kwargs)
self.should_serialize = False
hyp_func(self, *args, **kwargs)
return func
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
if cls.args_strategy is not None:
cls._args_strategy = st.tuples(
st.tuples(*cls.args_strategy).map(lambda args: cls.args(*args)),
st.tuples(*(cls.n - cls.h)*(cls.extrargs_strategy,))
).map(partial(sum, start=()))
cls.test_stress = given(cls._args_strategy, *cls.bounds_strategies)(StressfulERTest.test_stress)
cls.test_stress_m = given(
st.tuples(*cls.n*(cls.m_strategy,)),
cls._args_strategy, *cls.bounds_strategies
)(StressfulERTest.test_stress)
def infer(object) -> TestCase:
"""Use the hypothesis inference systems to create automated types from the annotations or ghetto typing.
>>> def f(int, b:int): ...
>>> test = infer(f)
"""
from unittest import FunctionTestCase
from inspect import getfullargspec
from hypothesis import given, strategies as st, find
spec = getfullargspec(object)
annotations = dict(**spec.annotations)
returns = annotations.pop('return', None)
if not spec.args: return FunctionTestCase(object)
if not (spec.defaults or spec.kwonlydefaults):
annotations = {
str: st.from_type(object) if isinstance(object, type)
and getattr(object, '__name__', '') != 'object' else
object if isinstance(object, st.SearchStrategy) else st.one_of(
list(map(st.just, object)))
for str, object in annotations.items()
}
if annotations:
return FunctionTestCase(given(**annotations)(object))
if (returns is not None) and callable(returns) ^ isinstance(
returns, type) and len(annotations) is 1:
return FunctionTestCase(
partial(find, *annotations.values(), lambda x: returns(object(x))))
def test_incircle(poly):
# The incircle should be centered for regular polygons.
assert sphere_isclose(poly.incircle, poly.maximal_centered_bounded_circle)
def check_rotation_invariance(quat):
rotated_poly = ConvexPolygon(rowan.rotate(quat, poly.vertices))
assert sphere_isclose(poly.incircle, rotated_poly.incircle)
# The incircle of a regular polygon should be rotation invariant.
given(Random2DRotationStrategy)(check_rotation_invariance)()
# The calculation should also be robust to out-of-plane rotations. Note
# that currently this test relies on the fact that circles are not
# orientable, otherwise they would need to be rotated back into the plane
# for the comparison.
given(Random3DRotationStrategy)(check_rotation_invariance)()
def prf_given():
"""
A wrapper for :py:func:`hypothesis.given` that establishes
parameters common to all Pseudo-Random Function tests.
:return: The same opaque type returned by
:py:func:`hypothesis.given`
"""
_prf_given = given(secret=st.binary(max_size=4096),
label=ascii_bytes(min_size=1, max_size=1024),
# OpenSSL does not use seeds longer than 1024 bytes
seed=st.binary(max_size=1024),
hash_cls=st.sampled_from([
hashes.SHA1,
hashes.SHA224,
hashes.SHA256,
hashes.SHA384,
hashes.SHA512,
hashes.RIPEMD160,
hashes.Whirlpool,
hashes.MD5,
]),
output_length=st.integers(min_value=0, max_value=1024))
def _ignore_slow_and_large_prf_given(function):
"""
Suppress data generation and size speed checks.
"""
ignore_slow = settings(suppress_health_check=[
HealthCheck.data_too_large,
HealthCheck.too_slow,
])
return ignore_slow(_prf_given(function))
return _ignore_slow_and_large_prf_given
def test_can_be_used_with_none_module():
def test_is_cool(i):
pass
test_is_cool.__module__ = None
test_is_cool = given(integers())(test_is_cool)
test_is_cool()
def inner_backend_decorator(func):
try:
use_imps = {
'all':
_picklable_vector_space_types + _other_vector_space_types,
'picklable': _picklable_vector_space_types
}[which]
except KeyError:
use_imps = which
first_args = {}
if index_strategy:
arr_ind_strategy = index_strategy(
vector_arrays(count=count,
dtype=dtype,
length=length,
compatible=compatible,
space_types=use_imps))
first_args['vectors_and_indices'] = arr_ind_strategy
else:
arr_strategy = vector_arrays(count=count,
dtype=dtype,
length=length,
compatible=compatible,
space_types=use_imps)
if count > 1:
first_args['vector_arrays'] = arr_strategy
else:
first_args['vector_array'] = arr_strategy
return given(**first_args, **kwargs)(func)
def create_test(
*,
operation: APIOperation,
test: Callable,
settings: Optional[hypothesis.settings] = None,
seed: Optional[int] = None,
data_generation_method: DataGenerationMethod = DataGenerationMethod.
default(),
_given_args: Tuple[GivenInput, ...] = (),
_given_kwargs: Optional[Dict[str, GivenInput]] = None,
) -> Callable:
"""Create a Hypothesis test."""
hook_dispatcher = getattr(test, "_schemathesis_hooks", None)
strategy = operation.as_strategy(
hooks=hook_dispatcher, data_generation_method=data_generation_method)
_given_kwargs = (_given_kwargs or {}).copy()
_given_kwargs.setdefault("case", strategy)
wrapped_test = hypothesis.given(*_given_args, **_given_kwargs)(test)
if seed is not None:
wrapped_test = hypothesis.seed(seed)(wrapped_test)
if asyncio.iscoroutinefunction(test):
wrapped_test.hypothesis.inner_test = make_async_test(
test) # type: ignore
setup_default_deadline(wrapped_test)
if settings is not None:
wrapped_test = settings(wrapped_test)
existing_settings = getattr(wrapped_test,
"_hypothesis_internal_use_settings", None)
if existing_settings and Phase.explicit in existing_settings.phases:
wrapped_test = add_examples(wrapped_test,
operation,
hook_dispatcher=hook_dispatcher)
return wrapped_test
def result(cls: type) -> type:
if not issubclass(cls, GenericTests):
raise TypeError(
"should operate on classes inheriting from GenericTests")
for name, method in inspect.getmembers(cls):
if name.startswith(testMethodPrefix) and callable(method):
parameters = inspect.signature(method).parameters
args = {
arg: param
for arg, param in parameters.items() if arg != "self"
}
if len(args) > 0:
given_args = dict()
for arg, param in args.items():
annotation = cls.relabel(
None if param.annotation ==
inspect.Parameter.empty else param.annotation)
if annotation in strategy_dict:
strat = strategy_dict[annotation]
elif arg == data_arg:
strat = st.data()
elif isinstance(annotation, st.SearchStrategy):
strat = annotation
else:
raise TypeError(
f"Cannot bind {cls.__name__}.{name}.{arg} with annotation {annotation} to strategy"
)
given_args[arg] = strat
setattr(cls, name, given(**given_args)(method))
return cls
def general_group_test(
f: Callable[[Type[jaxlie.MatrixLieGroup]], None],
max_examples: int = 100
) -> Callable[[Type[jaxlie.MatrixLieGroup], Any], None]:
"""Decorator for defining tests that run on all group types."""
# Disregard unused argument
def f_wrapped(Group: Type[jaxlie.MatrixLieGroup], _random_module) -> None:
f(Group)
# Disable timing check (first run requires JIT tracing and will be slower)
f_wrapped = settings(deadline=None)(f_wrapped)
# Add _random_module parameter
f_wrapped = given(_random_module=st.random_module())(f_wrapped)
# Parametrize tests with each group type
f_wrapped = pytest.mark.parametrize(
"Group",
[
jaxlie.SO2,
jaxlie.SE2,
jaxlie.SO3,
jaxlie.SE3,
],
)(f_wrapped)
return f_wrapped
def test_can_be_used_with_none_module():
def test_is_cool(i):
pass
test_is_cool.__module__ = None
test_is_cool = given(integers())(test_is_cool)
test_is_cool()
def _create_law_test_case(
container_type: Type[Lawful],
interface: Type[Lawful],
law: Law,
*,
settings: _Settings,
) -> None:
test_function = given(st.data())(hypothesis_settings(
**settings.settings_kwargs)(_run_law(container_type,
law,
settings=settings), ), )
called_from = inspect.stack()[2]
module = inspect.getmodule(called_from[0])
template = 'test_{container}_{interface}_{name}'
test_function.__name__ = template.format( # noqa: WPS125
container=container_type.__qualname__.lower(),
interface=interface.__qualname__.lower(),
name=law.name,
)
setattr(
module,
test_function.__name__,
# We mark all tests with `returns_lawful` marker,
# so users can easily skip them if needed.
pytest.mark.returns_lawful(test_function),
)
def create_test(
*,
endpoint: Endpoint,
test: Callable,
settings: Optional[hypothesis.settings] = None,
seed: Optional[int] = None,
_given_args: Tuple[GivenInput, ...] = (),
_given_kwargs: Optional[Dict[str, GivenInput]] = None,
) -> Callable:
"""Create a Hypothesis test."""
hook_dispatcher = getattr(test, "_schemathesis_hooks", None)
feedback: Optional[Feedback]
if endpoint.schema.stateful == Stateful.links:
feedback = Feedback(endpoint.schema.stateful, endpoint)
else:
feedback = None
strategy = endpoint.as_strategy(hooks=hook_dispatcher, feedback=feedback)
_given_kwargs = (_given_kwargs or {}).copy()
_given_kwargs.setdefault("case", strategy)
wrapped_test = hypothesis.given(*_given_args, **_given_kwargs)(test)
if seed is not None:
wrapped_test = hypothesis.seed(seed)(wrapped_test)
if asyncio.iscoroutinefunction(test):
wrapped_test.hypothesis.inner_test = make_async_test(test) # type: ignore
setup_default_deadline(wrapped_test)
if settings is not None:
wrapped_test = settings(wrapped_test)
wrapped_test._schemathesis_feedback = feedback # type: ignore
return add_examples(wrapped_test, endpoint, hook_dispatcher=hook_dispatcher)
def run_test(f1, f2, min_, max_):
g = given(st.integers(min_value=min_, max_value=max_),
st.integers(min_value=min_, max_value=max_), st.integers())
def w(n_up, n_down, seed) -> Callable:
cmp_imp(f1, f2, n_up, n_down, seed)
g(w)()
def run_and_statis(self,
max_examples=100,
opset_version=[7, 9, 15],
reproduce=None,
min_success_num=25,
max_duration=-1):
if os.getenv('HYPOTHESIS_TEST_PROFILE', 'ci') == "dev":
max_examples *= 10
min_success_num *= 10
# while at ce phase, there's no limit on time
max_duration = -1
start_time = time.time()
settings.register_profile(
"ci",
max_examples=max_examples,
suppress_health_check=hypothesis.HealthCheck.all(),
deadline=None,
print_blob=True,
derandomize=True,
report_multiple_bugs=False,
)
settings.load_profile("ci")
def sample_convert_generator(draw):
return self.sample_convert_config(draw)
def run_test(configs):
return self.run_test(configs=configs)
generator = st.composite(sample_convert_generator)
loop_func = given(generator())(run_test)
if reproduce is not None:
loop_func = reproduce(loop_func)
logging.info("Start to running test of {}".format(type(self)))
paddle.disable_static()
loop_func()
logging.info(
"===================Statistical Information===================")
logging.info("Number of Generated Programs: {}".format(
self.num_ran_models))
successful_ran_programs = int(self.num_ran_models)
if successful_ran_programs < min_success_num:
logging.warning("satisfied_programs = ran_programs")
logging.error(
"At least {} programs need to ran successfully, but now only about {} programs satisfied."
.format(min_success_num, successful_ran_programs))
assert False
used_time = time.time() - start_time
logging.info("Used time: {} s".format(round(used_time, 2)))
if max_duration > 0 and used_time > max_duration:
logging.error(
"The duration exceeds {} seconds, if this is neccessary, try to set a larger number for parameter `max_duration`."
.format(max_duration))
assert False
class TestTripleDES(unittest.TestCase):
_given = given(binary(min_size=24, max_size=24), # key
binary(min_size=8, max_size=8), # iv
binary(min_size=13*8, max_size=13*8), # plaintext
(tuples(integers(0, 13), integers(0, 13)) # split points
.filter(lambda split_pts: split_pts[0] <= split_pts[1])
.map(lambda lengths: [i * 8 for i in lengths])))
def split_test(self, key, iv, plaintext, split_points, make_impl=py_3des):
i, j = split_points
ciphertext = make_impl(key, iv).encrypt(plaintext)
self.assertEqual(make_impl(key, iv).decrypt(ciphertext), plaintext)
impl = make_impl(key, iv)
pl1, pl2, pl3 = plaintext[:i], plaintext[i:j], plaintext[j:]
ci1, ci2, ci3 = impl.encrypt(pl1), impl.encrypt(pl2), impl.encrypt(pl3)
self.assertEqual(ci1 + ci2 + ci3, ciphertext)
impl = make_impl(key, iv)
pl1, pl2, pl3 = impl.decrypt(ci1), impl.decrypt(ci2), impl.decrypt(ci3)
self.assertEqual(pl1 + pl2 + pl3, plaintext)
return ciphertext
@_given
@settings(**HYP_SETTINGS)
def test_python(self, key, iv, plaintext, split_points):
self.split_test(key, iv, plaintext, split_points)
@unittest.skipIf(not cryptomath.m2cryptoLoaded, "requires M2Crypto")
@_given
@settings(**HYP_SETTINGS)
def test_python_vs_mcrypto(self, key, iv, plaintext, split_points):
import tlslite.utils.openssl_tripledes
m2_3des = lambda k, iv: tlslite.utils.openssl_tripledes.new(k, 2, iv)
py_res = self.split_test(key, iv, plaintext, split_points, py_3des)
m2_res = self.split_test(key, iv, plaintext, split_points, m2_3des)
self.assertEqual(py_res, m2_res)
@unittest.skipIf(not cryptomath.pycryptoLoaded, "requires pycrypto")
@_given
@settings(**HYP_SETTINGS)
def test_python_vs_pycrypto(self, key, iv, plaintext, split_points):
import tlslite.utils.pycrypto_tripledes
pc_3des = lambda k, iv: tlslite.utils.pycrypto_tripledes.new(k, 2, iv)
try:
py_res = self.split_test(key, iv, plaintext, split_points, py_3des)
pc_res = self.split_test(key, iv, plaintext, split_points, pc_3des)
self.assertEqual(py_res, pc_res)
except ValueError as e:
# pycrypto deliberately rejects weak 3DES keys, skip such keys
assume(e.args != ('Triple DES key degenerates to single DES',))
raise
class TestTransactionHandler:
input_provider = given(
ticker=st.text(min_size=1, max_size=6),
amount=st.integers(),
price=st.floats(min_value=0, allow_infinity=False, allow_nan=False),
account_id=st.integers(),
timestamp=st.floats(),
)
# TODO I suppose we could merge the buy and sell tests and rely on
# hypothesis to discover which one(s) is broken.
@input_provider
def test_buy(self, ticker, amount, price, account_id, timestamp):
effs = handle_transaction(
StateSentinel(),
ticker,
amount,
price,
account_id,
timestamp,
E.Buy,
)
assert isinstance(effs, types.GeneratorType)
effs = list(effs)
assert len(effs) == 2
ticker_eff, cash_eff = effs
assert ticker_eff == Eff(ticker, amount, account_id, timestamp)
assert cash_eff == Eff('CASH', -amount * price, account_id, timestamp)
@input_provider
def test_sell(self, ticker, amount, price, account_id, timestamp):
effs = handle_transaction(
StateSentinel(),
ticker,
amount,
price,
account_id,
timestamp,
E.Sell,
)
assert isinstance(effs, types.GeneratorType)
effs = list(effs)
assert len(effs) == 2
ticker_eff, cash_eff = effs
assert ticker_eff == Eff(ticker, -amount, account_id, timestamp)
assert cash_eff == Eff('CASH', amount * price, account_id, timestamp)
def _run_tests():
from hypothesis import given
from hypothesis.strategies import integers
MAX_VALUE = 8
decor = given(n_rows = integers(min_value=1, max_value=MAX_VALUE),
n_cols = integers(min_value=1, max_value=MAX_VALUE),
seed = integers(min_value=0))
decor(test_random_coo)()
def _run_tests():
from hypothesis import given
from hypothesis.strategies import integers
MAX_VALUE = 8
hessenberg_decor = given(n_eqs = integers(min_value=1, max_value=MAX_VALUE),
n_vars = integers(min_value=0, max_value=MAX_VALUE),
seed = integers(min_value=0))
#hessenberg_decor(test_proxy)()
#quit()
hessenberg_decor(test_weighted_bipart)()
#
hessenberg_decor(test_to_hessenberg_none_forbidden)()
hessenberg_decor(test_to_hessenberg_some_forbidden)()
hessenberg_decor(test_to_hessenberg_all_forbidden)()
#-----
hessenberg_decor = given(n = integers(min_value=1, max_value=MAX_VALUE),
seed = integers(min_value=0))
# Checking whether the row order is retained (hashing could mess it up)
hessenberg_decor(test_to_hessenberg_none_forbidden_diagonal)()
hessenberg_decor(test_to_hessenberg_all_forbidden_diagonal)()
#-----
spiked_decor = given(n = integers(min_value=1, max_value=MAX_VALUE),
seed = integers(min_value=0))
spiked_decor(test_nonsingular_spiked)()
spiked_decor(test_to_spiked_none_forbidden)()
spiked_decor(test_to_spiked_some_forbidden)()
spiked_decor(test_to_spiked_all_forbidden)()
# Checking whether the row order is retained (hashing could mess it up)
spiked_decor(test_to_spiked_none_forbidden_diagonal)()
spiked_decor(test_to_spiked_all_forbidden_diagonal)()
class TestAES(unittest.TestCase):
_given = given(
binary(min_size=24, max_size=24), # key
binary(min_size=16, max_size=16), # iv
binary(min_size=13 * 16, max_size=13 * 16), # plaintext
(
tuples(integers(0, 13), integers(0, 13)) # split points
.filter(lambda split_pts: split_pts[0] <= split_pts[1]).map(
lambda lengths: [i * 16 for i in lengths])))
def split_test(self, key, iv, plaintext, split_points, make_impl=py_aes):
i, j = split_points
ciphertext = make_impl(key, iv).encrypt(plaintext)
self.assertEqual(make_impl(key, iv).decrypt(ciphertext), plaintext)
impl = make_impl(key, iv)
pl1, pl2, pl3 = plaintext[:i], plaintext[i:j], plaintext[j:]
ci1, ci2, ci3 = impl.encrypt(pl1), impl.encrypt(pl2), impl.encrypt(pl3)
self.assertEqual(ci1 + ci2 + ci3, ciphertext)
impl = make_impl(key, iv)
pl1, pl2, pl3 = impl.decrypt(ci1), impl.decrypt(ci2), impl.decrypt(ci3)
self.assertEqual(pl1 + pl2 + pl3, plaintext)
return ciphertext
@_given
@settings(**HYP_SETTINGS)
def test_python(self, key, iv, plaintext, split_points):
self.split_test(key, iv, plaintext, split_points)
@unittest.skipIf(not cryptomath.m2cryptoLoaded, "requires M2Crypto")
@_given
@settings(**HYP_SETTINGS)
def test_python_vs_mcrypto(self, key, iv, plaintext, split_points):
import tlslite.utils.openssl_aes
m2_aes = lambda k, iv: tlslite.utils.openssl_aes.new(k, 2, iv)
py_res = self.split_test(key, iv, plaintext, split_points, py_aes)
m2_res = self.split_test(key, iv, plaintext, split_points, m2_aes)
self.assertEqual(py_res, m2_res)
@unittest.skipIf(not cryptomath.pycryptoLoaded, "requires pycrypto")
@_given
@settings(**HYP_SETTINGS)
def test_python_vs_pycrypto(self, key, iv, plaintext, split_points):
import tlslite.utils.pycrypto_aes
pc_aes = lambda k, iv: tlslite.utils.pycrypto_aes.new(k, 2, iv)
py_res = self.split_test(key, iv, plaintext, split_points, py_aes)
pc_res = self.split_test(key, iv, plaintext, split_points, pc_aes)
self.assertEqual(py_res, pc_res)
def outer_wrapper(test):
def isolation_wrapper(*args, **kwargs):
network.rpc.snapshot()
test(*args, **kwargs)
network.rpc.revert()
# hypothesis.given must wrap the target test to correctly
# impersonate the call signature for pytest
hy_given = hypothesis.given(*given_args, **given_kwargs)
hy_wrapped = hy_given(test)
if hasattr(hy_wrapped, "hypothesis"):
hy_wrapped.hypothesis.inner_test = isolation_wrapper
return hy_wrapped
def generate_func_from_single_st(function, strategy):
"""generate function from a single strategy"""
function = given(strategy)(function)
# configure hypothesis
function = settings(
max_examples=1,
suppress_health_check=[
HealthCheck.large_base_example,
HealthCheck.too_slow,
HealthCheck.data_too_large,
HealthCheck.filter_too_much,
],
)(function)
return function
def accept(test):
verbose = os.environ.get('VERBOSE') == 'true'
shrink = os.environ.get('SHRINK', 'true') == 'true'
provided_seed = int(os.environ['SEED'])
if verbose:
verbosity = Verbosity.debug
else:
verbosity = Verbosity.normal
seen_failure = False
evaluations = 0
original_test = test
@proxies(test)
def recording_test(*args, **kwargs):
nonlocal seen_failure, evaluations
if seen_failure:
evaluations += 1
try:
return original_test(*args, **kwargs)
except HypothesisException:
raise
except Exception:
if not seen_failure:
seen_failure = True
evaluations += 1
raise
test = seed(provided_seed)(settings(
database=None,
max_examples=10**6,
suppress_health_check=HealthCheck.all(),
verbosity=verbosity,
phases=[Phase.generate, Phase.shrink]
if shrink else [Phase.generate])(given(strat)(recording_test)))
try:
test()
except Exception:
if verbose:
traceback.print_exc()
print("EVALUATIONS:", evaluations)
print("TEST FAILED")
return
print("Expected test to fail with assertion error",
file=sys.stderr)
sys.exit(1)
def _do(self, **parents):
if self.dist is not None:
k = self.dist(1)[0]
if k == 0:
k = 1
else:
k = self.n
recs = []
@settings(max_examples=k, deadline=self.deadline)
def gen(**kwargs):
rels = {}
for rv in self.relative_values:
rels.update({name: xform(**kwargs, **parents, **self.fixtures, **rels, **self.extras) for name, xform in rv.items()})
recs.append(self.create(self.model, **kwargs, **parents, **self.fixtures, **rels))
if self.strategy:
given(**self.strategy)(gen)()
else:
gen()
self.db.session.commit()
return recs
def specifier_test(test):
return given(templates_for(specifier), randoms())(settings(test))
def test_given_warns_when_mixing_positional_with_keyword(recwarn):
given(booleans(), y=booleans(), settings=Settings(strict=False))
assert recwarn.pop(DeprecationWarning) is not None
def test_bare_given_errors():
with pytest.raises(InvalidArgument):
given()
pass
def has_a_default(x, y, z=1):
pass
def has_varargs(*args):
pass
def has_kwargs(**kwargs):
pass
basic_test_cases = [
(has_one_arg, given(integers())),
(has_one_arg, given(hello=integers())),
(has_two_args, given(integers())),
(has_two_args, given(integers(), booleans())),
(has_a_default, given(integers(), integers())),
(has_a_default, given(integers(), integers(), integers())),
]
@pytest.mark.parametrize((u'f', u'g'), basic_test_cases)
def test_argspec_lines_up(f, g):
af = getargspec(f)
ag = getargspec(g(f))
assert af.args == ag.args
assert af.keywords == ag.keywords
assert af.varargs == ag.varargs
policy_name_strings = partial(nice_strings,
bad_chars=['\n', '/', '?', '#', '%'])
# Useful parameterisations:
params_namespaces = pytest.mark.parametrize('namespace', [ROOT_URL + "/ceph",
ROOT_URL + "/netapp"])
params_vol_presence = pytest.mark.parametrize('vol_exists',
["vol_present", "vol_absent"])
params_policy_presence = pytest.mark.parametrize(
'policy_status',
["policy_present", "policy_absent"])
params_auth_status = pytest.mark.parametrize('auth',
["authorised", "not_authorised"])
params_volume_names = compose_decorators(given(volume_name=name_strings()))
params_vol_ns_auth = compose_decorators(params_volume_names,
params_namespaces,
params_auth_status,
params_vol_presence)
params_ns_auth = compose_decorators(params_namespaces,
params_auth_status)
@composite
def patch_arguments(draw):
keys = draw(lists(elements=sampled_from(["autosize_enabled",
"autosize_increment",
"max_autosize"])))
d = dict()
def strategy_test_suite(
specifier,
max_examples=100, random=None,
):
settings = Settings(
database=None,
max_examples=max_examples,
min_satisfying_examples=2,
average_list_length=2.0,
)
random = random or Random()
strat = strategy(specifier, settings)
specifier_test = given(
TemplatesFor(specifier), Random, settings=settings
)
class ValidationSuite(TestCase):
def __repr__(self):
return 'strategy_test_suite(%s)' % (
show(specifier),
)
@given(specifier, settings=settings)
def test_does_not_error(self, value):
pass
def test_can_give_example(self):
strat.example()
def test_can_give_list_of_examples(self):
strategy([strat]).example()
def test_will_give_unsatisfiable_if_all_rejected(self):
@given(specifier, settings=settings)
def nope(x):
assume(False)
with self.assertRaises(Unsatisfiable):
nope()
def test_will_find_a_constant_failure(self):
@given(specifier, settings=settings)
def nope(x):
raise Rejected()
with self.assertRaises(Rejected):
nope()
def test_will_find_a_failure_from_the_database(self):
db = ExampleDatabase()
@given(specifier, settings=Settings(max_examples=10, database=db))
def nope(x):
raise Rejected()
try:
for i in hrange(3):
with self.assertRaises(Rejected):
nope() # pragma: no branch
finally:
db.close()
@given(
TemplatesFor(specifier), TemplatesFor(specifier),
settings=settings
)
def test_simplicity_is_asymmetric(self, x, y):
assert not (
strat.strictly_simpler(x, y) and
strat.strictly_simpler(y, x)
)
def test_will_handle_a_really_weird_failure(self):
db = ExampleDatabase()
@given(
specifier,
settings=Settings(
database=db,
max_examples=max_examples,
min_satisfying_examples=2,
average_list_length=2.0,
)
)
def nope(x):
s = hashlib.sha1(show(x).encode('utf-8')).digest()
if Random(s).randint(0, 1):
raise Rejected()
try:
try:
nope()
except Rejected:
pass
try:
nope()
except Rejected:
pass
finally:
db.close()
@specifier_test
def test_is_basic(self, value, rnd):
def is_basic(v):
if v is None or isinstance(v, text_type):
return True
if isinstance(v, integer_types):
return not (abs(v) >> 64)
if isinstance(v, list):
return all(is_basic(w) for w in v)
return False
supposedly_basic = strat.to_basic(value)
self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic))
@specifier_test
def test_only_raises_bad_data_in_from_basic(self, value, rnd):
basic = strat.to_basic(value)
messed_basic = mutate_basic(basic, rnd)
try:
strat.from_basic(messed_basic)
except BadData:
pass
@specifier_test
def test_can_round_trip_through_the_database(self, template, rnd):
empty_db = ExampleDatabase(
backend=SQLiteBackend(':memory:'),
)
try:
storage = empty_db.storage_for(specifier)
storage.save(template)
values = list(storage.fetch())
assert len(values) == 1
assert strat.to_basic(template) == strat.to_basic(values[0])
finally:
empty_db.close()
@specifier_test
def test_template_is_hashable(self, template, rnd):
hash(template)
# It can be easy to forget to convert a list...
hash(strat.from_basic(strat.to_basic(template)))
@given(
TemplatesFor(specifier), Random,
[[int]],
settings=settings
)
def test_apply_all_simplifiers(self, template, rnd, path):
path = list(filter(None, path))
assume(path)
current_template = template
for local_route in path:
simplifiers = list(strat.simplifiers(random, current_template))
if not simplifiers:
break
for i in local_route:
simplify = simplifiers[abs(i) % len(simplifiers)]
simpler = list(simplify(
rnd, current_template
))
if simpler:
current_template = random.choice(simpler)
@specifier_test
def test_can_minimize_to_empty(self, template, rnd):
simplest = template
try:
while True:
simplest = next(strat.full_simplify(rnd, simplest))
except StopIteration:
pass
assert list(strat.full_simplify(rnd, simplest)) == []
@specifier_test
def test_full_simplify_completes(self, template, rnd):
# Cut off at 1000 for the occasional case where we get
# really very large templates which have too many simplifies.
for x in islice(strat.full_simplify(rnd, template), 1000):
pass
@specifier_test
def test_does_not_increase_complexity(self, template, rnd):
for s in islice(strat.full_simplify(rnd, template), 100):
assert not strat.strictly_simpler(template, s)
@given(Random, settings=Settings(max_examples=1000))
def test_can_create_templates(self, random):
parameter = strat.draw_parameter(random)
strat.draw_template(BuildContext(random), parameter)
return ValidationSuite
def vec_reduce(test_func, strat, arith_func, type):
return pytest.mark.parametrize('strat,func,type', [
(strat, arith_func, type)
])(given(data=st.data())(test_func))
def _vec_float_binary(test_func, func, type):
return pytest.mark.parametrize('func,type', [
(func, type)
])(given(data=st.data())(test_func))
def vec_int_unary(test_func, unary_func, type):
return pytest.mark.parametrize('func,type', [
(unary_func, type)
])(given(data=st.data())(test_func))
pass
def has_a_default(x, y, z=1):
pass
def has_varargs(*args):
pass
def has_kwargs(**kwargs):
pass
basic_test_cases = [
(has_one_arg, given(int)),
(has_one_arg, given(hello=int)),
(has_two_args, given(int)),
(has_two_args, given(int, bool)),
(has_a_default, given(int, int)),
(has_a_default, given(int, int, int)),
]
@pytest.mark.parametrize(('f', 'g'), basic_test_cases)
def test_argspec_lines_up(f, g):
af = inspect.getargspec(f)
ag = inspect.getargspec(g(f))
assert af.args == ag.args
assert af.keywords == ag.keywords
assert af.varargs == ag.varargs
def test_given_edits_annotations(nargs):
spec_given = getfullargspec(given(*(nargs * [st.none()]))(pointless_composite))
assert spec_given.annotations.pop("return") is None
assert len(spec_given.annotations) == 3 - nargs
def test_bare_given_errors():
with pytest.raises(TypeError):
given()
def strategy_test_suite(
specifier,
max_examples=20, random=None,
):
settings = Settings(
database=None,
max_examples=max_examples,
min_satisfying_examples=2,
average_list_length=2.0,
)
random = random or Random()
strat = strategy(specifier, settings)
specifier_test = given(
templates_for(specifier), randoms(), settings=settings
)
class ValidationSuite(TestCase):
def __repr__(self):
return u'strategy_test_suite(%s)' % (
repr(specifier),
)
@given(specifier, settings=settings)
def test_does_not_error(self, value):
pass
def test_can_give_example(self):
strat.example()
def test_can_give_list_of_examples(self):
strategy(lists(strat)).example()
def test_will_give_unsatisfiable_if_all_rejected(self):
@given(specifier, settings=settings)
def nope(x):
assume(False)
self.assertRaises(Unsatisfiable, nope)
def test_will_find_a_constant_failure(self):
@given(specifier, settings=settings)
def nope(x):
raise Rejected()
self.assertRaises(Rejected, nope)
def test_will_find_a_failure_from_the_database(self):
db = ExampleDatabase()
@given(specifier, settings=Settings(max_examples=10, database=db))
def nope(x):
raise Rejected()
try:
for i in hrange(3):
self.assertRaises(Rejected, nope) # pragma: no cover
finally:
db.close()
@given(
templates_for(specifier), templates_for(specifier),
settings=settings
)
def test_simplicity_is_asymmetric(self, x, y):
assert not (
strat.strictly_simpler(x, y) and
strat.strictly_simpler(y, x)
)
@given(integers(), settings=settings)
def test_templates_generated_from_same_random_are_equal(self, i):
try:
t1 = strat.draw_and_produce(Random(i))
t2 = strat.draw_and_produce(Random(i))
except BadTemplateDraw:
assume(False)
if t1 is not t2:
assert t1 == t2
assert hash(t1) == hash(t2)
@given(integers(), settings=settings)
def test_templates_generated_from_same_random_are_equal_after_reify(
self, i
):
try:
t1 = strat.draw_and_produce(Random(i))
t2 = strat.draw_and_produce(Random(i))
except BadTemplateDraw:
assume(False)
if t1 is not t2:
with BuildContext():
strat.reify(t1)
with BuildContext():
strat.reify(t2)
assert t1 == t2
assert hash(t1) == hash(t2)
@given(randoms(), settings=settings)
def test_will_handle_a_really_weird_failure(self, rnd):
db = ExampleDatabase()
@given(
specifier,
settings=Settings(
database=db,
max_examples=max_examples,
min_satisfying_examples=2,
average_list_length=2.0,
), random=rnd
)
def nope(x):
s = hashlib.sha1(repr(x).encode(u'utf-8')).digest()
assert Random(s).randint(0, 1) == Random(s).randint(0, 1)
if Random(s).randint(0, 1):
raise Rejected(u'%r with digest %r' % (
x, s
))
try:
try:
nope()
except Rejected:
pass
try:
nope()
except Rejected:
pass
finally:
db.close()
@specifier_test
def test_is_basic(self, value, rnd):
def is_basic(v):
if v is None or isinstance(v, text_type):
return True
if isinstance(v, integer_types):
return not (abs(v) >> 64)
if isinstance(v, list):
return all(is_basic(w) for w in v)
return False
supposedly_basic = strat.to_basic(value)
self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic))
@specifier_test
def test_only_raises_bad_data_in_from_basic(self, value, rnd):
basic = strat.to_basic(value)
messed_basic = mutate_basic(basic, rnd)
try:
strat.from_basic(messed_basic)
except BadData:
pass
@specifier_test
def test_can_round_trip_through_the_database(self, template, rnd):
empty_db = ExampleDatabase(
backend=SQLiteBackend(u':memory:'),
)
try:
storage = empty_db.storage(u'round trip')
storage.save(template, strat)
values = list(storage.fetch(strat))
assert len(values) == 1
assert strat.to_basic(template) == strat.to_basic(values[0])
finally:
empty_db.close()
@specifier_test
def test_template_is_hashable(self, template, rnd):
hash(template)
# It can be easy to forget to convert a list...
hash(strat.from_basic(strat.to_basic(template)))
@specifier_test
def test_can_minimize_to_empty(self, template, rnd):
simplest = template
tracker = Tracker()
while True:
for t in strat.full_simplify(rnd, simplest):
if tracker.track(t) == 1:
simplest = t
break
else:
break
assert list(strat.full_simplify(rnd, simplest)) == []
@specifier_test
def test_full_simplify_completes(self, template, rnd):
# Cut off at 1000 for the occasional case where we get
# really very large templates which have too many simplifies.
for x in islice(strat.full_simplify(rnd, template), 1000):
pass
@specifier_test
def test_does_not_increase_complexity(self, template, rnd):
for s in islice(strat.full_simplify(rnd, template), 100):
assert not strat.strictly_simpler(template, s)
@given(randoms(), settings=Settings(max_examples=1000))
def test_can_create_templates(self, random):
parameter = strat.draw_parameter(random)
try:
strat.draw_template(random, parameter)
except BadTemplateDraw:
assume(False)
return ValidationSuite
# contribution.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.
#
# END HEADER
from __future__ import division, print_function, absolute_import
import pytest
from hypothesis import given
from hypothesis import strategies as st
given_booleans = given(st.booleans())
@given_booleans
def test_has_an_arg_named_x(x):
pass
@given_booleans
def test_has_an_arg_named_y(y):
pass
given_named_booleans = given(z=st.text())
def strategy_test_suite(
specifier,
max_examples=100, random=None
):
settings = Settings(
database=None,
max_examples=max_examples,
average_list_length=2.0,
)
random = random or Random()
verifier = Verifier(
settings=settings,
random=random
)
strat = strategy(specifier, settings)
specifier_test = given(
TemplatesFor(specifier), Random, settings=settings
)
class ValidationSuite(TestCase):
def __repr__(self):
return 'strategy_test_suite(%s)' % (
show(specifier),
)
@given(specifier, settings=settings)
def test_does_not_error(self, value):
pass
def test_can_give_example(self):
strat.example()
@specifier_test
def test_is_basic(self, value, rnd):
def is_basic(v):
return isinstance(
v, integer_types + (list, type(None), text_type)
) and (
not isinstance(v, list) or
all(is_basic(w) for w in v)
)
supposedly_basic = strat.to_basic(value)
self.assertTrue(is_basic(supposedly_basic), repr(supposedly_basic))
@specifier_test
def test_can_round_trip_through_the_database(self, template, rnd):
empty_db = ExampleDatabase(
backend=SQLiteBackend(':memory:'),
)
try:
storage = empty_db.storage_for(specifier)
storage.save(template)
values = list(storage.fetch())
assert len(values) == 1
assert strat.to_basic(template) == strat.to_basic(values[0])
finally:
empty_db.close()
@specifier_test
def test_template_is_hashable(self, template, rnd):
hash(template)
@specifier_test
def test_can_minimize_to_empty(self, template, rnd):
simplest = list(strat.simplify_such_that(
rnd,
template, lambda x: True
))[-1]
assert list(strat.full_simplify(rnd, simplest)) == []
@specifier_test
def test_can_complete_falsify_loop(self, template, rnd):
for _ in strat.full_simplify(rnd, template):
pass
@given(Random, settings=Settings(max_examples=1000))
def test_can_create_templates(self, random):
parameter = strat.draw_parameter(random)
strat.draw_template(BuildContext(random), parameter)
@given(Random, verifier=verifier)
def test_can_perform_all_basic_operations(self, rnd):
parameter = strat.draw_parameter(random)
template = strat.draw_template(BuildContext(rnd), parameter)
minimal_template = list(strat.simplify_such_that(
rnd,
template,
lambda x: True
))[-1]
strat.reify(minimal_template)
assert (
strat.to_basic(minimal_template) ==
strat.to_basic(
strat.from_basic(strat.to_basic(minimal_template)))
)
return ValidationSuite
