def check_test_not_42(): has_failed = False task_module = import_task_file() test_case_class = inspect_assertions(task_module.TestRandomNot42) test_result = run_test_case_tests(test_case_class) if not test_result.wasSuccessful(): failed("at least one test failed") has_failed = True if not check_used_only_assertions(test_case_class, ["assertNotEqual"]): has_failed = True mock_random_not_42 = unittest.mock.Mock(return_value=42) with unittest.mock.patch('tested_code.random_not_42', mock_random_not_42): task_module = import_task_file() test_result = run_test_case_tests(task_module.TestRandomNot42) if test_result.wasSuccessful(): failed("tests passed with broken implementation") has_failed = True if not has_failed: passed()
def check_test_random_float_between_inclusive(): has_failed = False task_module = import_task_file() test_case_class = inspect_assertions(task_module.TestRandomFloatBetweenInclusive) test_result = run_test_case_tests(test_case_class) if not test_result.wasSuccessful(): failed("at least one test failed") has_failed = True if not check_used_only_assertions( test_case_class, ["assertGreaterEqual", "assertLessEqual"], ): has_failed = True for broken_func in [ lambda a, b: a - 1, lambda a, b: b + 1, ]: with unittest.mock.patch('tested_code.random_float_between_inclusive', broken_func): task_module = import_task_file() test_result = run_test_case_tests(task_module.TestRandomFloatBetweenInclusive) if test_result.wasSuccessful(): failed("tests passed with broken implementation") has_failed = True if not has_failed: passed()
def check_test_random_float_between_inclusive(): has_failed = False task_module = import_task_file() test_case_class = inspect_assertions( task_module.TestRandomFloatBetweenInclusive) test_result = run_test_case_tests(test_case_class) if not test_result.wasSuccessful(): failed("at least one test failed") has_failed = True if not check_used_only_assertions( test_case_class, ["assertGreaterEqual", "assertLessEqual"], ): has_failed = True for broken_func in [ lambda a, b: a - 1, lambda a, b: b + 1, ]: with unittest.mock.patch('tested_code.random_float_between_inclusive', broken_func): task_module = import_task_file() test_result = run_test_case_tests( task_module.TestRandomFloatBetweenInclusive) if test_result.wasSuccessful(): failed("tests passed with broken implementation") has_failed = True if not has_failed: passed()
def check_test_find_foo(): has_failed = False task_module = import_task_file() test_case_class = inspect_assertions(task_module.TestFindFoo) test_result = run_test_case_tests(test_case_class) if not test_result.wasSuccessful(): failed("at least one test failed") has_failed = True if not check_used_only_assertions( test_case_class, ["assertIsNone", "assertIsNotNone"], ): has_failed = True # check that all of the substrings were tested mock_find_foo = unittest.mock.Mock(wraps=find_foo) with unittest.mock.patch('tested_code.find_foo', mock_find_foo): task_module = import_task_file() test_case_class = task_module.TestFindFoo run_test_case_tests(test_case_class) normalized_call_args = [ normalize_call_args(call_args, func=find_foo) for call_args in mock_find_foo.call_args_list ] for substring in itertools.chain( ["foo"], test_case_class.strings_with_foo, test_case_class.strings_without_foo ): if ((substring,), {}) not in normalized_call_args: failed("substring \"{}\" not tested".format(substring)) has_failed = True # check with broken find_foo() def find_fo(s): return re.search(r"fo", s) for broken_find_fo in [ find_fo, lambda s: None, lambda s: 0, ]: with unittest.mock.patch('tested_code.find_foo', broken_find_fo): task_module = import_task_file() test_result = run_test_case_tests(task_module.TestFindFoo) if test_result.wasSuccessful(): failed("tests passed with broken implementation") has_failed = True if not has_failed: passed()
def check_test_find_foo(): has_failed = False task_module = import_task_file() test_case_class = inspect_assertions(task_module.TestFindFoo) test_result = run_test_case_tests(test_case_class) if not test_result.wasSuccessful(): failed("at least one test failed") has_failed = True if not check_used_only_assertions( test_case_class, ["assertIsNone", "assertIsNotNone"], ): has_failed = True # check that all of the substrings were tested mock_find_foo = unittest.mock.Mock(wraps=TestFindFoo) with unittest.mock.patch('tested_code.find_foo', mock_find_foo): task_module = import_task_file() test_case_class = task_module.TestFindFoo run_test_case_tests(test_case_class) normalized_call_args = [ normalize_call_args(call_args, func=TestFindFoo) for call_args in mock_find_foo.call_args_list ] for substring in itertools.chain(["foo"], test_case_class.strings_with_foo, test_case_class.strings_without_foo): if ((substring, ), {}) not in normalized_call_args: failed("substring \"{}\" not tested".format(substring)) has_failed = True # check with broken find_foo() def find_fo(s): return re.search(r"fo", s) for broken_find_fo in [ find_fo, lambda s: None, lambda s: 0, ]: with unittest.mock.patch('tested_code.find_foo', broken_find_fo): task_module = import_task_file() test_result = run_test_case_tests(task_module.TestFindFoo) if test_result.wasSuccessful(): failed("tests passed with broken implementation") has_failed = True if not has_failed: passed()
def check_container_truthiness_tests(): task_module = import_task_file() test_case_class = inspect_assertions(task_module.TestContainerTruthiness) test_result = run_test_case_tests(test_case_class) conditional_passed_or_failed(test_result.wasSuccessful(), "TestContainerTruthiness tests pass") for (test_method_name, empty_container) in [ ("test_list", []), ("test_tuple", ()), ("test_set", set()), ("test_dict", {}), ]: calls = test_case_class.per_method_counters[test_method_name]["assertFalse"].call_args_list normalized_calls = [ normalize_call_args(c, func=functools.partial(test_case_class.assertFalse, None)) for c in calls ] conditional_passed_or_failed( condition=any(args[0] == empty_container for args, kwargs in normalized_calls), name="{} properly tested empty container".format(test_method_name), ) calls = test_case_class.per_method_counters[test_method_name]["assertTrue"].call_args_list normalized_calls = [ normalize_call_args(c, func=functools.partial(test_case_class.assertTrue, None)) for c in calls ] conditional_passed_or_failed( condition=any( type(args[0]) == type(empty_container) and args[0] != empty_container for args, kwargs in normalized_calls ), name="{} properly tested non-empty container".format(test_method_name), )
def check_integer_truthiness_tests(): task_module = import_task_file() TestIntegerTruthiness = inspect_assertions( task_module.TestIntegerTruthiness) test_result = run_test_case_tests(TestIntegerTruthiness) if not test_result.wasSuccessful(): failed("Some of the TestIntegerTruthiness tests failed!") placeholder_windows = get_answer_placeholders()[:3] test_zero_window, test_one_window, test_other_value_window = placeholder_windows # check test_zero n_assertions = TestIntegerTruthiness.get_test_method_total_call_count( "test_zero") test_name = "TestIntegerTruthiness.test_zero" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif unittest.mock.call(0) in TestIntegerTruthiness.per_method_counters[ "test_zero"]["assertFalse"].call_args_list: passed(name=test_name) else: failed(name=test_name) # check test_one n_assertions = TestIntegerTruthiness.get_test_method_total_call_count( "test_one") test_name = "TestIntegerTruthiness.test_one" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif unittest.mock.call(1) in TestIntegerTruthiness.per_method_counters[ "test_one"]["assertTrue"].call_args_list: passed(name=test_name) else: failed(name=test_name) # check test_other_value wrapped_assertTrue = TestIntegerTruthiness.per_method_counters[ "test_other_value"]["assertTrue"] n_assertions = TestIntegerTruthiness.get_test_method_total_call_count( "test_other_value") test_name = "TestIntegerTruthiness.test_other_value" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif ("self.assertTrue" in test_other_value_window and wrapped_assertTrue.call_count > 0 and isinstance(wrapped_assertTrue.call_args[0][0], int) and wrapped_assertTrue.call_args[0][0] not in {0, 1}): passed(name=test_name) else: failed(name=test_name)
def check_integer_truthiness_tests(): task_module = import_task_file() TestIntegerTruthiness = inspect_assertions(task_module.TestIntegerTruthiness) test_result = run_test_case_tests(TestIntegerTruthiness) if not test_result.wasSuccessful(): failed("Some of the TestIntegerTruthiness tests failed!") placeholder_windows = get_answer_placeholders()[0:3] test_zero_window, test_one_window, test_other_value_window = placeholder_windows # check test_zero n_assertions = TestIntegerTruthiness.get_test_method_total_call_count("test_zero") test_name = "TestIntegerTruthiness.test_zero" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif unittest.mock.call(0) in TestIntegerTruthiness.per_method_counters["test_zero"]["assertFalse"].call_args_list: passed(name=test_name) else: failed(name=test_name) # check test_one n_assertions = TestIntegerTruthiness.get_test_method_total_call_count("test_one") test_name = "TestIntegerTruthiness.test_one" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif unittest.mock.call(1) in TestIntegerTruthiness.per_method_counters["test_one"]["assertTrue"].call_args_list: passed(name=test_name) else: failed(name=test_name) # check test_other_value wrapped_assertTrue = TestIntegerTruthiness.per_method_counters["test_other_value"]["assertTrue"] n_assertions = TestIntegerTruthiness.get_test_method_total_call_count("test_other_value") test_name = "TestIntegerTruthiness.test_other_value" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif ( "self.assertTrue" in test_other_value_window and wrapped_assertTrue.call_count > 0 and isinstance(wrapped_assertTrue.call_args[0][0], int) and wrapped_assertTrue.call_args[0][0] not in {0, 1} ): passed(name=test_name) else: failed(name=test_name)
def check_container_truthiness_tests(): task_module = import_task_file() test_case_class = inspect_assertions(task_module.TestContainerTruthiness) test_result = run_test_case_tests(test_case_class) conditional_passed_or_failed(test_result.wasSuccessful(), "TestContainerTruthiness tests pass") for (test_method_name, empty_container) in [ ("test_list", []), ("test_tuple", ()), ("test_set", set()), ("test_dict", {}), ]: calls = test_case_class.per_method_counters[test_method_name][ "assertFalse"].call_args_list normalized_calls = [ normalize_call_args(c, func=functools.partial( test_case_class.assertFalse, None)) for c in calls ] conditional_passed_or_failed( condition=any(args[0] == empty_container for args, kwargs in normalized_calls), name="{} properly tested empty container".format(test_method_name), ) calls = test_case_class.per_method_counters[test_method_name][ "assertTrue"].call_args_list normalized_calls = [ normalize_call_args(c, func=functools.partial( test_case_class.assertTrue, None)) for c in calls ] conditional_passed_or_failed( condition=any( type(args[0]) == type(empty_container) and args[0] != empty_container for args, kwargs in normalized_calls), name="{} properly tested non-empty container".format( test_method_name), )
def check_none_truthiness_tests(): task_module = import_task_file() TestNoneTruthiness = inspect_assertions(task_module.TestNoneTruthiness) test_result = run_test_case_tests(TestNoneTruthiness) if not test_result.wasSuccessful(): failed("Some of the TestIntegerTruthiness tests failed!") wrapped_assertFalse = TestNoneTruthiness.per_method_counters["test_none"]["assertFalse"] n_assertions = TestNoneTruthiness.get_test_method_total_call_count("test_none") test_name = "TestNoneTruthiness.test_none" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif unittest.mock.call(None) in TestNoneTruthiness.per_method_counters["test_none"]["assertFalse"].call_args_list: passed(name=test_name) else: failed(name=test_name)
def check_none_truthiness_tests(): task_module = import_task_file() TestNoneTruthiness = inspect_assertions(task_module.TestNoneTruthiness) test_result = run_test_case_tests(TestNoneTruthiness) if not test_result.wasSuccessful(): failed("Some of the TestIntegerTruthiness tests failed!") wrapped_assertFalse = TestNoneTruthiness.per_method_counters["test_none"][ "assertFalse"] n_assertions = TestNoneTruthiness.get_test_method_total_call_count( "test_none") test_name = "TestNoneTruthiness.test_none" if n_assertions > 1: failed(name=test_name, message="must use only one assertion") elif n_assertions == 0: failed(name=test_name, message="must use an assertion") elif unittest.mock.call(None) in TestNoneTruthiness.per_method_counters[ "test_none"]["assertFalse"].call_args_list: passed(name=test_name) else: failed(name=test_name)
def check_tests_fail_on_broken_implementations(): expected_results = { 'broken_shuffle_1': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': False, 'test_input_not_mutated': True, }, 'broken_shuffle_2': { 'test_empty': True, 'test_same_length': False, 'test_same_values': False, 'test_values_are_shuffled': None, 'test_input_not_mutated': True, }, 'broken_shuffle_3': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': False, 'test_input_not_mutated': True, }, 'broken_shuffle_4': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': True, 'test_input_not_mutated': False, }, 'broken_shuffle_5': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': None, 'test_input_not_mutated': False, }, 'broken_shuffle_6': { 'test_empty': True, 'test_same_length': True, 'test_same_values': False, 'test_values_are_shuffled': None, 'test_input_not_mutated': True, }, 'broken_shuffle_7': { 'test_empty': False, 'test_same_length': None, 'test_same_values': None, 'test_values_are_shuffled': None, 'test_input_not_mutated': False, }, } task_module = import_task_file() orig_random_shuffle = task_module.random_shuffle for broken_func_name in sorted(expected_results): task_module.random_shuffle = getattr(task_module, broken_func_name) test_result = run_test_case_tests(task_module.TestRandomShuffle) test_method_name2failure = { test_case.id().rsplit('.', 1)[1]: test_case for (test_case, msg) in test_result.failures } test_method_name2error = { test_case.id().rsplit('.', 1)[1]: test_case for (test_case, msg) in test_result.errors } test_method_name2failure_or_error = {} test_method_name2failure_or_error.update(test_method_name2failure) test_method_name2failure_or_error.update(test_method_name2error) has_failed = False for test_method_name in get_test_method_names(task_module.TestRandomShuffle): expected_result = expected_results[broken_func_name][test_method_name] if expected_result is None: # ignore continue is_error_or_failure = test_method_name in test_method_name2failure_or_error if expected_result is True and is_error_or_failure: failed("{func_name} failed test {test_name}, which it should pass".format( func_name=broken_func_name, test_name="TestRandomShuffle."+test_method_name, ), name="test {}".format(broken_func_name)) has_failed = True message = [x[1] for x in test_result.failures + test_result.errors if x[0].id().rsplit('.', 1)[1] == test_method_name][0] print(message) elif expected_result is False and not is_error_or_failure: failed("{func_name} passed test {test_name}, which it should fail".format( func_name=broken_func_name, test_name="TestRandomShuffle."+test_method_name, ), name="test {}".format(broken_func_name)) has_failed = True if not has_failed: passed(name="test {}".format(broken_func_name)) task_module.random_shuffle = orig_random_shuffle
def check_tests_fail_on_broken_implementations(): expected_results = { 'broken_shuffle_1': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': False, 'test_input_not_mutated': True, }, 'broken_shuffle_2': { 'test_empty': True, 'test_same_length': False, 'test_same_values': False, 'test_values_are_shuffled': None, 'test_input_not_mutated': True, }, 'broken_shuffle_3': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': False, 'test_input_not_mutated': True, }, 'broken_shuffle_4': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': True, 'test_input_not_mutated': False, }, 'broken_shuffle_5': { 'test_empty': True, 'test_same_length': True, 'test_same_values': True, 'test_values_are_shuffled': None, 'test_input_not_mutated': False, }, 'broken_shuffle_6': { 'test_empty': True, 'test_same_length': True, 'test_same_values': False, 'test_values_are_shuffled': None, 'test_input_not_mutated': True, }, 'broken_shuffle_7': { 'test_empty': False, 'test_same_length': None, 'test_same_values': None, 'test_values_are_shuffled': None, 'test_input_not_mutated': False, }, } task_module = import_task_file() orig_random_shuffle = task_module.random_shuffle for broken_func_name in sorted(expected_results): task_module.random_shuffle = getattr(task_module, broken_func_name) test_result = run_test_case_tests(task_module.TestRandomShuffle) test_method_name2failure = { test_case.id().rsplit('.', 1)[1]: test_case for (test_case, msg) in test_result.failures } test_method_name2error = { test_case.id().rsplit('.', 1)[1]: test_case for (test_case, msg) in test_result.errors } test_method_name2failure_or_error = {} test_method_name2failure_or_error.update(test_method_name2failure) test_method_name2failure_or_error.update(test_method_name2error) has_failed = False for test_method_name in get_test_method_names( task_module.TestRandomShuffle): expected_result = expected_results[broken_func_name][ test_method_name] if expected_result is None: # ignore continue is_error_or_failure = test_method_name in test_method_name2failure_or_error if expected_result is True and is_error_or_failure: failed( "{func_name} failed test {test_name}, which it should pass" .format( func_name=broken_func_name, test_name="TestRandomShuffle." + test_method_name, ), name="test {}".format(broken_func_name)) has_failed = True message = [ x[1] for x in test_result.failures + test_result.errors if x[0].id().rsplit('.', 1)[1] == test_method_name ][0] print(message) elif expected_result is False and not is_error_or_failure: failed( "{func_name} passed test {test_name}, which it should fail" .format( func_name=broken_func_name, test_name="TestRandomShuffle." + test_method_name, ), name="test {}".format(broken_func_name)) has_failed = True if not has_failed: passed(name="test {}".format(broken_func_name)) task_module.random_shuffle = orig_random_shuffle