def check_test_not_42():
has_failed = False
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestRandomNot42)
test_result = run_test_case_tests(test_case_class)
if not test_result.wasSuccessful():
failed("at least one test failed")
has_failed = True
if not check_used_only_assertions(test_case_class, ["assertNotEqual"]):
has_failed = True
mock_random_not_42 = unittest.mock.Mock(return_value=42)
with unittest.mock.patch('tested_code.random_not_42', mock_random_not_42):
task_module = import_task_file()
test_result = run_test_case_tests(task_module.TestRandomNot42)
if test_result.wasSuccessful():
failed("tests passed with broken implementation")
has_failed = True
if not has_failed:
passed()
def check_test_random_float_between_inclusive():
has_failed = False
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestRandomFloatBetweenInclusive)
test_result = run_test_case_tests(test_case_class)
if not test_result.wasSuccessful():
failed("at least one test failed")
has_failed = True
if not check_used_only_assertions(
test_case_class,
["assertGreaterEqual", "assertLessEqual"],
):
has_failed = True
for broken_func in [
lambda a, b: a - 1,
lambda a, b: b + 1,
]:
with unittest.mock.patch('tested_code.random_float_between_inclusive', broken_func):
task_module = import_task_file()
test_result = run_test_case_tests(task_module.TestRandomFloatBetweenInclusive)
if test_result.wasSuccessful():
failed("tests passed with broken implementation")
has_failed = True
if not has_failed:
passed()
def check_test_random_float_between_inclusive():
has_failed = False
task_module = import_task_file()
test_case_class = inspect_assertions(
task_module.TestRandomFloatBetweenInclusive)
test_result = run_test_case_tests(test_case_class)
if not test_result.wasSuccessful():
failed("at least one test failed")
has_failed = True
if not check_used_only_assertions(
test_case_class,
["assertGreaterEqual", "assertLessEqual"],
):
has_failed = True
for broken_func in [
lambda a, b: a - 1,
lambda a, b: b + 1,
]:
with unittest.mock.patch('tested_code.random_float_between_inclusive',
broken_func):
task_module = import_task_file()
test_result = run_test_case_tests(
task_module.TestRandomFloatBetweenInclusive)
if test_result.wasSuccessful():
failed("tests passed with broken implementation")
has_failed = True
if not has_failed:
passed()
def check_test_find_foo():
has_failed = False
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestFindFoo)
test_result = run_test_case_tests(test_case_class)
if not test_result.wasSuccessful():
failed("at least one test failed")
has_failed = True
if not check_used_only_assertions(
test_case_class,
["assertIsNone", "assertIsNotNone"],
):
has_failed = True
# check that all of the substrings were tested
mock_find_foo = unittest.mock.Mock(wraps=find_foo)
with unittest.mock.patch('tested_code.find_foo', mock_find_foo):
task_module = import_task_file()
test_case_class = task_module.TestFindFoo
run_test_case_tests(test_case_class)
normalized_call_args = [
normalize_call_args(call_args, func=find_foo)
for call_args in mock_find_foo.call_args_list
]
for substring in itertools.chain(
["foo"],
test_case_class.strings_with_foo,
test_case_class.strings_without_foo
):
if ((substring,), {}) not in normalized_call_args:
failed("substring \"{}\" not tested".format(substring))
has_failed = True
# check with broken find_foo()
def find_fo(s):
return re.search(r"fo", s)
for broken_find_fo in [
find_fo,
lambda s: None,
lambda s: 0,
]:
with unittest.mock.patch('tested_code.find_foo', broken_find_fo):
task_module = import_task_file()
test_result = run_test_case_tests(task_module.TestFindFoo)
if test_result.wasSuccessful():
failed("tests passed with broken implementation")
has_failed = True
if not has_failed:
passed()
def check_test_find_foo():
has_failed = False
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestFindFoo)
test_result = run_test_case_tests(test_case_class)
if not test_result.wasSuccessful():
failed("at least one test failed")
has_failed = True
if not check_used_only_assertions(
test_case_class,
["assertIsNone", "assertIsNotNone"],
):
has_failed = True
# check that all of the substrings were tested
mock_find_foo = unittest.mock.Mock(wraps=TestFindFoo)
with unittest.mock.patch('tested_code.find_foo', mock_find_foo):
task_module = import_task_file()
test_case_class = task_module.TestFindFoo
run_test_case_tests(test_case_class)
normalized_call_args = [
normalize_call_args(call_args, func=TestFindFoo)
for call_args in mock_find_foo.call_args_list
]
for substring in itertools.chain(["foo"],
test_case_class.strings_with_foo,
test_case_class.strings_without_foo):
if ((substring, ), {}) not in normalized_call_args:
failed("substring \"{}\" not tested".format(substring))
has_failed = True
# check with broken find_foo()
def find_fo(s):
return re.search(r"fo", s)
for broken_find_fo in [
find_fo,
lambda s: None,
lambda s: 0,
]:
with unittest.mock.patch('tested_code.find_foo', broken_find_fo):
task_module = import_task_file()
test_result = run_test_case_tests(task_module.TestFindFoo)
if test_result.wasSuccessful():
failed("tests passed with broken implementation")
has_failed = True
if not has_failed:
passed()
def check_container_truthiness_tests():
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestContainerTruthiness)
test_result = run_test_case_tests(test_case_class)
conditional_passed_or_failed(test_result.wasSuccessful(), "TestContainerTruthiness tests pass")
for (test_method_name, empty_container) in [
("test_list", []),
("test_tuple", ()),
("test_set", set()),
("test_dict", {}),
]:
calls = test_case_class.per_method_counters[test_method_name]["assertFalse"].call_args_list
normalized_calls = [
normalize_call_args(c, func=functools.partial(test_case_class.assertFalse, None))
for c in calls
]
conditional_passed_or_failed(
condition=any(args[0] == empty_container for args, kwargs in normalized_calls),
name="{} properly tested empty container".format(test_method_name),
)
calls = test_case_class.per_method_counters[test_method_name]["assertTrue"].call_args_list
normalized_calls = [
normalize_call_args(c, func=functools.partial(test_case_class.assertTrue, None))
for c in calls
]
conditional_passed_or_failed(
condition=any(
type(args[0]) == type(empty_container) and args[0] != empty_container
for args, kwargs in normalized_calls
),
name="{} properly tested non-empty container".format(test_method_name),
)
def check_integer_truthiness_tests():
task_module = import_task_file()
TestIntegerTruthiness = inspect_assertions(
task_module.TestIntegerTruthiness)
test_result = run_test_case_tests(TestIntegerTruthiness)
if not test_result.wasSuccessful():
failed("Some of the TestIntegerTruthiness tests failed!")
placeholder_windows = get_answer_placeholders()[:3]
test_zero_window, test_one_window, test_other_value_window = placeholder_windows
# check test_zero
n_assertions = TestIntegerTruthiness.get_test_method_total_call_count(
"test_zero")
test_name = "TestIntegerTruthiness.test_zero"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif unittest.mock.call(0) in TestIntegerTruthiness.per_method_counters[
"test_zero"]["assertFalse"].call_args_list:
passed(name=test_name)
else:
failed(name=test_name)
# check test_one
n_assertions = TestIntegerTruthiness.get_test_method_total_call_count(
"test_one")
test_name = "TestIntegerTruthiness.test_one"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif unittest.mock.call(1) in TestIntegerTruthiness.per_method_counters[
"test_one"]["assertTrue"].call_args_list:
passed(name=test_name)
else:
failed(name=test_name)
# check test_other_value
wrapped_assertTrue = TestIntegerTruthiness.per_method_counters[
"test_other_value"]["assertTrue"]
n_assertions = TestIntegerTruthiness.get_test_method_total_call_count(
"test_other_value")
test_name = "TestIntegerTruthiness.test_other_value"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif ("self.assertTrue" in test_other_value_window
and wrapped_assertTrue.call_count > 0
and isinstance(wrapped_assertTrue.call_args[0][0], int)
and wrapped_assertTrue.call_args[0][0] not in {0, 1}):
passed(name=test_name)
else:
failed(name=test_name)
def check_integer_truthiness_tests():
task_module = import_task_file()
TestIntegerTruthiness = inspect_assertions(task_module.TestIntegerTruthiness)
test_result = run_test_case_tests(TestIntegerTruthiness)
if not test_result.wasSuccessful():
failed("Some of the TestIntegerTruthiness tests failed!")
placeholder_windows = get_answer_placeholders()[0:3]
test_zero_window, test_one_window, test_other_value_window = placeholder_windows
# check test_zero
n_assertions = TestIntegerTruthiness.get_test_method_total_call_count("test_zero")
test_name = "TestIntegerTruthiness.test_zero"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif unittest.mock.call(0) in TestIntegerTruthiness.per_method_counters["test_zero"]["assertFalse"].call_args_list:
passed(name=test_name)
else:
failed(name=test_name)
# check test_one
n_assertions = TestIntegerTruthiness.get_test_method_total_call_count("test_one")
test_name = "TestIntegerTruthiness.test_one"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif unittest.mock.call(1) in TestIntegerTruthiness.per_method_counters["test_one"]["assertTrue"].call_args_list:
passed(name=test_name)
else:
failed(name=test_name)
# check test_other_value
wrapped_assertTrue = TestIntegerTruthiness.per_method_counters["test_other_value"]["assertTrue"]
n_assertions = TestIntegerTruthiness.get_test_method_total_call_count("test_other_value")
test_name = "TestIntegerTruthiness.test_other_value"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif (
"self.assertTrue" in test_other_value_window and
wrapped_assertTrue.call_count > 0 and
isinstance(wrapped_assertTrue.call_args[0][0], int) and
wrapped_assertTrue.call_args[0][0] not in {0, 1}
):
passed(name=test_name)
else:
failed(name=test_name)
def check_test_not_42():
has_failed = False
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestRandomNot42)
test_result = run_test_case_tests(test_case_class)
if not test_result.wasSuccessful():
failed("at least one test failed")
has_failed = True
if not check_used_only_assertions(test_case_class, ["assertNotEqual"]):
has_failed = True
mock_random_not_42 = unittest.mock.Mock(return_value=42)
with unittest.mock.patch('tested_code.random_not_42', mock_random_not_42):
task_module = import_task_file()
test_result = run_test_case_tests(task_module.TestRandomNot42)
if test_result.wasSuccessful():
failed("tests passed with broken implementation")
has_failed = True
if not has_failed:
passed()
def check_container_truthiness_tests():
task_module = import_task_file()
test_case_class = inspect_assertions(task_module.TestContainerTruthiness)
test_result = run_test_case_tests(test_case_class)
conditional_passed_or_failed(test_result.wasSuccessful(),
"TestContainerTruthiness tests pass")
for (test_method_name, empty_container) in [
("test_list", []),
("test_tuple", ()),
("test_set", set()),
("test_dict", {}),
]:
calls = test_case_class.per_method_counters[test_method_name][
"assertFalse"].call_args_list
normalized_calls = [
normalize_call_args(c,
func=functools.partial(
test_case_class.assertFalse, None))
for c in calls
]
conditional_passed_or_failed(
condition=any(args[0] == empty_container
for args, kwargs in normalized_calls),
name="{} properly tested empty container".format(test_method_name),
)
calls = test_case_class.per_method_counters[test_method_name][
"assertTrue"].call_args_list
normalized_calls = [
normalize_call_args(c,
func=functools.partial(
test_case_class.assertTrue, None))
for c in calls
]
conditional_passed_or_failed(
condition=any(
type(args[0]) == type(empty_container)
and args[0] != empty_container
for args, kwargs in normalized_calls),
name="{} properly tested non-empty container".format(
test_method_name),
)
def check_none_truthiness_tests():
task_module = import_task_file()
TestNoneTruthiness = inspect_assertions(task_module.TestNoneTruthiness)
test_result = run_test_case_tests(TestNoneTruthiness)
if not test_result.wasSuccessful():
failed("Some of the TestIntegerTruthiness tests failed!")
wrapped_assertFalse = TestNoneTruthiness.per_method_counters["test_none"]["assertFalse"]
n_assertions = TestNoneTruthiness.get_test_method_total_call_count("test_none")
test_name = "TestNoneTruthiness.test_none"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif unittest.mock.call(None) in TestNoneTruthiness.per_method_counters["test_none"]["assertFalse"].call_args_list:
passed(name=test_name)
else:
failed(name=test_name)
def check_none_truthiness_tests():
task_module = import_task_file()
TestNoneTruthiness = inspect_assertions(task_module.TestNoneTruthiness)
test_result = run_test_case_tests(TestNoneTruthiness)
if not test_result.wasSuccessful():
failed("Some of the TestIntegerTruthiness tests failed!")
wrapped_assertFalse = TestNoneTruthiness.per_method_counters["test_none"][
"assertFalse"]
n_assertions = TestNoneTruthiness.get_test_method_total_call_count(
"test_none")
test_name = "TestNoneTruthiness.test_none"
if n_assertions > 1:
failed(name=test_name, message="must use only one assertion")
elif n_assertions == 0:
failed(name=test_name, message="must use an assertion")
elif unittest.mock.call(None) in TestNoneTruthiness.per_method_counters[
"test_none"]["assertFalse"].call_args_list:
passed(name=test_name)
else:
failed(name=test_name)
def check_tests_fail_on_broken_implementations():
expected_results = {
'broken_shuffle_1': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': False,
'test_input_not_mutated': True,
},
'broken_shuffle_2': {
'test_empty': True,
'test_same_length': False,
'test_same_values': False,
'test_values_are_shuffled': None,
'test_input_not_mutated': True,
},
'broken_shuffle_3': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': False,
'test_input_not_mutated': True,
},
'broken_shuffle_4': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': True,
'test_input_not_mutated': False,
},
'broken_shuffle_5': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': None,
'test_input_not_mutated': False,
},
'broken_shuffle_6': {
'test_empty': True,
'test_same_length': True,
'test_same_values': False,
'test_values_are_shuffled': None,
'test_input_not_mutated': True,
},
'broken_shuffle_7': {
'test_empty': False,
'test_same_length': None,
'test_same_values': None,
'test_values_are_shuffled': None,
'test_input_not_mutated': False,
},
}
task_module = import_task_file()
orig_random_shuffle = task_module.random_shuffle
for broken_func_name in sorted(expected_results):
task_module.random_shuffle = getattr(task_module, broken_func_name)
test_result = run_test_case_tests(task_module.TestRandomShuffle)
test_method_name2failure = {
test_case.id().rsplit('.', 1)[1]: test_case
for (test_case, msg) in test_result.failures
}
test_method_name2error = {
test_case.id().rsplit('.', 1)[1]: test_case
for (test_case, msg) in test_result.errors
}
test_method_name2failure_or_error = {}
test_method_name2failure_or_error.update(test_method_name2failure)
test_method_name2failure_or_error.update(test_method_name2error)
has_failed = False
for test_method_name in get_test_method_names(task_module.TestRandomShuffle):
expected_result = expected_results[broken_func_name][test_method_name]
if expected_result is None:
# ignore
continue
is_error_or_failure = test_method_name in test_method_name2failure_or_error
if expected_result is True and is_error_or_failure:
failed("{func_name} failed test {test_name}, which it should pass".format(
func_name=broken_func_name,
test_name="TestRandomShuffle."+test_method_name,
), name="test {}".format(broken_func_name))
has_failed = True
message = [x[1] for x in test_result.failures + test_result.errors if x[0].id().rsplit('.', 1)[1] == test_method_name][0]
print(message)
elif expected_result is False and not is_error_or_failure:
failed("{func_name} passed test {test_name}, which it should fail".format(
func_name=broken_func_name,
test_name="TestRandomShuffle."+test_method_name,
), name="test {}".format(broken_func_name))
has_failed = True
if not has_failed:
passed(name="test {}".format(broken_func_name))
task_module.random_shuffle = orig_random_shuffle
def check_tests_fail_on_broken_implementations():
expected_results = {
'broken_shuffle_1': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': False,
'test_input_not_mutated': True,
},
'broken_shuffle_2': {
'test_empty': True,
'test_same_length': False,
'test_same_values': False,
'test_values_are_shuffled': None,
'test_input_not_mutated': True,
},
'broken_shuffle_3': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': False,
'test_input_not_mutated': True,
},
'broken_shuffle_4': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': True,
'test_input_not_mutated': False,
},
'broken_shuffle_5': {
'test_empty': True,
'test_same_length': True,
'test_same_values': True,
'test_values_are_shuffled': None,
'test_input_not_mutated': False,
},
'broken_shuffle_6': {
'test_empty': True,
'test_same_length': True,
'test_same_values': False,
'test_values_are_shuffled': None,
'test_input_not_mutated': True,
},
'broken_shuffle_7': {
'test_empty': False,
'test_same_length': None,
'test_same_values': None,
'test_values_are_shuffled': None,
'test_input_not_mutated': False,
},
}
task_module = import_task_file()
orig_random_shuffle = task_module.random_shuffle
for broken_func_name in sorted(expected_results):
task_module.random_shuffle = getattr(task_module, broken_func_name)
test_result = run_test_case_tests(task_module.TestRandomShuffle)
test_method_name2failure = {
test_case.id().rsplit('.', 1)[1]: test_case
for (test_case, msg) in test_result.failures
}
test_method_name2error = {
test_case.id().rsplit('.', 1)[1]: test_case
for (test_case, msg) in test_result.errors
}
test_method_name2failure_or_error = {}
test_method_name2failure_or_error.update(test_method_name2failure)
test_method_name2failure_or_error.update(test_method_name2error)
has_failed = False
for test_method_name in get_test_method_names(
task_module.TestRandomShuffle):
expected_result = expected_results[broken_func_name][
test_method_name]
if expected_result is None:
# ignore
continue
is_error_or_failure = test_method_name in test_method_name2failure_or_error
if expected_result is True and is_error_or_failure:
failed(
"{func_name} failed test {test_name}, which it should pass"
.format(
func_name=broken_func_name,
test_name="TestRandomShuffle." + test_method_name,
),
name="test {}".format(broken_func_name))
has_failed = True
message = [
x[1] for x in test_result.failures + test_result.errors
if x[0].id().rsplit('.', 1)[1] == test_method_name
][0]
print(message)
elif expected_result is False and not is_error_or_failure:
failed(
"{func_name} passed test {test_name}, which it should fail"
.format(
func_name=broken_func_name,
test_name="TestRandomShuffle." + test_method_name,
),
name="test {}".format(broken_func_name))
has_failed = True
if not has_failed:
passed(name="test {}".format(broken_func_name))
task_module.random_shuffle = orig_random_shuffle
