def _execute_end_to_end_test(self, function_name):
with open(f'{self._folder_expected_output}{function_name}.txt') as file:
expected_output = textwrap.dedent(file.read())
function = getattr(integers_functions, function_name)
strategy_factory = StrategyFactory(function)
actual_output = textwrap.dedent(strategy_factory.generate_composite_strategy())
self.assertEqual(expected_output, actual_output)
def _execute_failing_end_to_end_test(self, function_name):
function = getattr(sets_functions, function_name)
strategy_factory = StrategyFactory(function)
self.assertRaises(NotImplementedError, strategy_factory.generate_composite_strategy)
def test_module(module: Any) -> pd.DataFrame:
assert (inspect.ismodule(module))
if module.__name__ not in sys.modules:
importlib.import_module(module.__name__)
functions = [
f[1].__name__ for f in inspect.getmembers(module)
if inspect.isfunction(f[1])
]
global COUNTER
print(f"MUT: {module.__name__}")
modules = []
function_names = []
runs = []
assume_free = []
filter_free = []
nb_samples = []
failed_health_check = []
violation_error = []
exceptions = []
for function_name in functions:
modules.append(module.__name__)
function_names.append(function_name)
print(f"FUT: {function_name}")
COUNTER = 0
# add the local classes to the 'globals()' namespace
local_members = {name: obj for name, obj in inspect.getmembers(module)}
for cls_name, cls in local_members.items():
globals()[cls_name] = cls
function = prepare_function(getattr(module, function_name))
globals().update({function_name: function})
strategy_str = ""
try:
add_run = False
if USE_OLD_VERSION_IC:
strategy_str = str(
icontract_hypothesis.infer_strategy(function))
icontract_hypothesis.test_with_inferred_strategy(
function, localns=locals(), globalns=globals())
add_run = True
else:
sf = StrategyFactory(function,
localns=locals(),
globalns=globals())
strategy_str = sf.generate_composite_strategy()
if strategy_str:
# an empty strategy means that there was nothing we could generate
# for example when no type hints were given
# these cases are not taken into account
# check if it runs correctly
if len(sf._get_type_hints()) > 1:
test_function(function_name,
strategy_str,
multiple_arguments=True)
else:
test_function(function_name,
strategy_str,
multiple_arguments=False)
add_run = True
else:
modules.pop()
function_names.pop()
if add_run:
runs.append(True)
# check if it uses 'assume'
uses_assume = any(
RE_ASSUME.match(line) for line in strategy_str.split("\n"))
assume_free.append(not uses_assume)
# check if it uses 'filter'
uses_filter = True if RE_FILTER.match(strategy_str) else False
filter_free.append(not uses_filter)
nb_samples.append(COUNTER)
failed_health_check.append(False)
violation_error.append(False)
exceptions.append(None)
except hypothesis.errors.MultipleFailures as e:
runs.append(True)
if strategy_str:
# check if it uses 'assume'
uses_assume = any(
RE_ASSUME.match(line) for line in strategy_str.split("\n"))
assume_free.append(not uses_assume)
# check if it uses 'filter'
uses_filter = True if RE_FILTER.match(strategy_str) else False
filter_free.append(not uses_filter)
else:
assume_free.append(False)
filter_free.append(False)
nb_samples.append(COUNTER)
failed_health_check.append(True)
violation_error.append(False)
exceptions.append(None)
except hypothesis.errors.FailedHealthCheck as e:
runs.append(False)
# check if it uses 'assume'
uses_assume = any(
RE_ASSUME.match(line) for line in strategy_str.split("\n"))
assume_free.append(not uses_assume)
# check if it uses 'filter'
uses_filter = True if RE_FILTER.match(strategy_str) else False
filter_free.append(not uses_filter)
nb_samples.append(COUNTER)
failed_health_check.append(True)
violation_error.append(False)
exceptions.append(e)
except icontract.errors.ViolationError as e:
runs.append(False)
# check if it uses 'assume'
uses_assume = any(
RE_ASSUME.match(line) for line in strategy_str.split("\n"))
assume_free.append(not uses_assume)
# check if it uses 'filter'
uses_filter = True if RE_FILTER.match(strategy_str) else False
filter_free.append(not uses_filter)
nb_samples.append(COUNTER)
failed_health_check.append(False)
violation_error.append(True)
exceptions.append(e)
except Exception as e:
runs.append(False)
# check if it uses 'assume'
uses_assume = any(
RE_ASSUME.match(line) for line in strategy_str.split("\n"))
assume_free.append(not uses_assume)
# check if it uses 'filter'
uses_filter = True if RE_FILTER.match(strategy_str) else False
filter_free.append(not uses_filter)
nb_samples.append(COUNTER)
failed_health_check.append(False)
violation_error.append(False)
exceptions.append(f"{type(e)} : {e}")
df = pd.DataFrame(
data={
'module': modules,
'function_name': function_names,
'runs': runs,
'assume_free': assume_free,
'filter_free': filter_free,
'nb_samples': nb_samples,
'failed_health_check': failed_health_check,
'violation_error': violation_error,
'exceptions': exceptions
})
return df