def test_module_argument_not_verifiable_syserror(chosen_arguments, capsys):
"""Check that not valid arguments will not verify correctly"""
with pytest.raises(SystemExit):
arguments.parse(chosen_arguments)
standard_out, standard_err = capsys.readouterr()
assert standard_out is EMPTY_STRING
assert ERROR in standard_err
def test_check_function_verification_list(commandline_arguments):
"""Ensure that check verification works for standard functions."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
parsed_arguments, remaining_arguments = arguments.parse(
commandline_arguments)
args_verified = arguments.verify(parsed_arguments)
assert args_verified is True
external_checker_directory = checkers.get_checker_dir(parsed_arguments)
checker_source = checkers.get_source([external_checker_directory])
check_name = checkers.get_chosen_check(parsed_arguments)
check_file = checkers.transform_check(check_name)
check_exists = checkers.verify_check_existence(check_file,
checker_source)
assert check_exists is True
# create the check
check = checker_source.load_plugin(check_file)
# verify that the check has the functions, specified separately
assert (checkers.verify_check_functions(
check, ["act", "get_parser", "parse"]) is True)
# verify that the check has the functions, specified according to defaults
assert checkers.verify_check_functions(check) is True
# verify that the check does not have the provided functions, specified separately
assert (checkers.verify_check_functions(
check, ["actWRONG", "get_parser", "parse"]) is False)
assert (checkers.verify_check_functions(
check, ["act", "get_parserWRONG", "parse"]) is False)
assert (checkers.verify_check_functions(
check, ["actWRONG", "get_parser", "parseWRONG"]) is False)
def test_act_produces_output(commandline_arguments, expected_result,
load_checker):
"""Ensure that using the check produces output."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
parsed_arguments, remaining_arguments = arguments.parse(
commandline_arguments)
args_verified = arguments.verify(parsed_arguments)
assert args_verified is True
check_exists, checker_source, check_file = load_checker(
parsed_arguments)
assert check_exists is True
check = checker_source.load_plugin(check_file)
check_result = check.act(parsed_arguments, remaining_arguments)
# check the result
assert check_result is not None
assert len(check_result) == 1
assert check_result[0] is expected_result
# check the contents of the report
assert report.get_result() is not None
assert len(report.get_result()["check"]) > 1
assert report.get_result()["outcome"] is expected_result
if expected_result:
assert report.get_result()["diagnostic"] == ""
else:
assert report.get_result()["diagnostic"] != ""
def test_is_valid_file_valid(chosen_arguments):
"""Check that valid argument combinations do not verify correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
verified_arguments = arguments.is_valid_file_and_directory(parsed_arguments)
assert verified_arguments is True
verified_arguments = arguments.is_valid_file_or_directory(parsed_arguments)
assert verified_arguments is True
def test_check_extraction_from_commandline_arguments_has_help_single_checker_filtered(
):
"""Ensure that checker finding and help extraction works for a single filtered check."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
checker = "ListChecks"
commandline_arguments = [
"--checkerdir",
"./gator/checks",
"ListChecks",
"--namecontains",
"Exec",
]
gg_arguments, remaining_arguments = arguments.parse(
commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
found_check = checkers.get_chosen_check(gg_arguments)
assert found_check == checker
checker_source = checkers.get_source()
check_helps = checkers.get_checks_help(checker_source,
namecontains="Exec")
assert check_helps != ""
assert "Exec" in check_helps
counted_newlines = check_helps.count("\n")
assert counted_newlines > 0
def test_check_extraction_from_commandline_arguments_has_help_two_checkers_one_invalid(
tmpdir
):
"""Ensure that checker finding and help extraction works for a provided checker."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
checker_file = tmpdir.mkdir("internal_checkers").join("check_testing.py")
# this must be valid Python code because it will be loaded by pluginbase
checker_file.write('"' 'a checker"' "")
checker_directory = (
tmpdir.dirname + "/" + tmpdir.basename + "/" + "internal_checkers"
)
checker = "check_CountCommits"
assert len(tmpdir.listdir()) == 1
commandline_arguments = ["--checkerdir", checker_directory, checker]
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
found_check = checkers.get_chosen_check(gg_arguments)
assert found_check == checker
# ensure that get_checks_help does not try to create a help
# message for an invalid check that does not have a get_parser function
checker_source = checkers.get_source([checker_directory])
check_helps = checkers.get_checks_help(checker_source)
assert check_helps != ""
assert "CountCommits" in check_helps
counted_newlines = check_helps.count("\n")
assert counted_newlines > 0
def test_is_command_ancillary(chosen_arguments):
"""Check that file ancillary detection verifies correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
# note that this function is only checking for the presence
# of the ancillary arguments like fragment and count
verified_arguments = arguments.is_command_ancillary(parsed_arguments)
assert verified_arguments is True
def test_no_arguments_incorrect_system_exit_not_verified(capsys):
"""Ensure that no command-line arguments causes SystemExit crash and is not verified."""
with pytest.raises(SystemExit):
gg_arguments, remaining_arguments = arguments.parse([])
arguments_args_verified = arguments.verify(gg_arguments)
assert arguments_args_verified == NOT_VERIFIED
# capture the output so that test output is not polluted
_ = capsys.readouterr()
def test_description_not_specified_is_not_valid_arguments_verify(capsys):
"""Check that command-line argument without valid string verifies."""
commandline_arguments = ["--description", "--json", "check_FakeMessages"]
with pytest.raises(SystemExit):
gg_arguments, remaining_arguments = arguments.parse(
commandline_arguments)
_ = arguments.verify(gg_arguments)
captured = capsys.readouterr()
assert "--description: expected one argument" in captured.err
def test_empty_checkerdir_extraction_from_commandline_arguments(tmpdir):
"""Ensure that command-line argument extraction works in checker function if empty checkerdir."""
_ = tmpdir.mkdir("checks").join("check_messages.py")
assert len(tmpdir.listdir()) == 1
commandline_arguments = ["check_messages"]
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
found_checker_directory = checkers.get_checker_dir(gg_arguments)
assert found_checker_directory == ""
def test_description_is_valid_arguments_verify():
"""Check that command-line argument with valid string verifies."""
commandline_arguments = [
"--description",
"Do you have fake things?",
"check_FakeMessages",
]
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
def test_check_extraction_from_commandline_arguments(tmpdir):
"""Ensure that command-line argument extraction works in checker function."""
checker = "check_CountCommits"
_ = tmpdir.mkdir("checks").join(checker + ".py")
assert len(tmpdir.listdir()) == 1
checker_directory = tmpdir.dirname + "/" + tmpdir.basename + "/" + "checks"
commandline_arguments = ["--checkerdir", checker_directory, checker]
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
found_check = checkers.get_chosen_check(gg_arguments)
assert found_check == checker
def test_no_arguments_incorrect_system_exit(capsys):
"""No command-line arguments causes SystemExit crash of argparse with error output."""
with pytest.raises(SystemExit):
_ = arguments.parse([])
captured = capsys.readouterr()
# there is no standard output
counted_newlines = captured.out.count("\n")
assert counted_newlines == 0
# standard error has two lines from pytest
assert "usage:" in captured.err
counted_newlines = captured.err.count("\n")
assert counted_newlines == 2
def test_checkerdir_is_not_valid_arguments_verify(tmpdir):
"""Check that command-line argument with valid directory verifies."""
_ = tmpdir.mkdir("checks").join("check_FakeMessages.py")
assert len(tmpdir.listdir()) == 1
# this directory does not exist on the file system and verification should not work
checker_directory = tmpdir.dirname + "/" + tmpdir.basename + "/" + "checksWRONG"
commandline_arguments = [
"--checkerdir", checker_directory, "check_FakeMessages"
]
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is False
def test_act_produces_output(
commandline_arguments,
chosen_file,
containing_directory,
provided_count,
expected_result,
tmpdir,
load_checker,
):
"""Check that using the check produces output."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
new_file = tmpdir.mkdir(containing_directory).join(chosen_file)
new_file.write(
"paragraph one \n\n paragraph two two \n\n paragraph three three three"
)
assert (
new_file.read() ==
"paragraph one \n\n paragraph two two \n\n paragraph three three three"
)
assert len(tmpdir.listdir()) == 1
overall_directory = (tmpdir.dirname + "/" + tmpdir.basename + "/" +
containing_directory)
commandline_arguments = [
"CountParagraphWords",
"--file",
"file_to_find",
"--directory",
overall_directory,
"--count",
provided_count,
]
parsed_arguments, remaining_arguments = arguments.parse(
commandline_arguments)
args_verified = arguments.verify(parsed_arguments)
assert args_verified is True
check_exists, checker_source, check_file = load_checker(
parsed_arguments)
assert check_exists is True
check = checker_source.load_plugin(check_file)
check_result = check.act(parsed_arguments, remaining_arguments)
# check the result
assert check_result is not None
assert len(check_result) == 1
# assert check_result[0] is expected_result
# check the contents of the report
assert report.get_result() is not None
assert len(report.get_result()["check"]) > 1
assert report.get_result()["outcome"] is expected_result
if expected_result:
assert report.get_result()["diagnostic"] == ""
else:
assert report.get_result()["diagnostic"] != ""
def test_help_produces_output(capsys):
"""Ensure that when given a request for help, it is produced correctly."""
with pytest.raises(SystemExit):
_ = arguments.parse(["--help"])
captured = capsys.readouterr()
# there is no standard output
counted_newlines = captured.out.count("\n")
assert counted_newlines > 0
# standard error has lines produced by specialized print_help
assert "required argument:" in captured.out
assert "usage:" in captured.out
assert "all checks:" in captured.out
assert "internal checks:" in captured.out
def test_check_extraction_from_commandline_arguments_has_help_single_checker():
"""Ensure that checker finding and help extraction works for a provided checker."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
checker = "check_CountCommits"
commandline_arguments = [checker]
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
found_check = checkers.get_chosen_check(gg_arguments)
assert found_check == checker
checker_source = checkers.get_source()
check_helps = checkers.get_checks_help(checker_source)
assert check_helps != ""
assert "CountCommits" in check_helps
counted_newlines = check_helps.count("\n")
assert counted_newlines > 0
def test_check_function_verification_separate(commandline_arguments):
"""Ensure that check verification works for standard functions."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
parsed_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(parsed_arguments)
assert args_verified is True
external_checker_directory = checkers.get_checker_dir(parsed_arguments)
checker_source = checkers.get_source([external_checker_directory])
check_name = checkers.get_chosen_check(parsed_arguments)
check_file = checkers.transform_check(check_name)
check_exists = checkers.verify_check_existence(check_file, checker_source)
assert check_exists is True
check = checker_source.load_plugin(check_file)
assert checkers.verify_check_function(check, "act") is True
assert checkers.verify_check_function(check, "get_parser") is True
assert checkers.verify_check_function(check, "parse") is True
def test_reset_checker_source(load_checker):
"""Ensure that reset of the checker source works correctly."""
assert checkers.CHECKER_SOURCE is None
testargs = [os.getcwd()]
commandline_arguments = [
"CountCommandOutput",
"--command",
'echo "CorrectCommand"',
"--count",
"100",
]
with patch.object(sys, "argv", testargs):
parsed_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(parsed_arguments)
assert args_verified is True
check_exists, checker_source, check_file = load_checker(parsed_arguments)
assert check_exists is True
checkers.reset_source()
assert checkers.CHECKER_SOURCE is None
def check_arguments(system_arguments):
"""Check the arguments return the desired actions to perform"""
# parse and verify the arguments
actions = []
gg_arguments = arguments.parse(system_arguments)
# Action: display the welcome message
if gg_arguments.nowelcome is not True:
actions.append([DISPLAY, "welcome_message", VOID])
if gg_arguments.json is True:
# pylint: disable=global-statement
global OUTPUT_TYPE
OUTPUT_TYPE = getattr(REPORT, JSON)
did_verify_arguments = arguments.verify(gg_arguments)
# arguments are incorrect
if did_verify_arguments is False:
# Action: display incorrect arguments message
actions.append([DISPLAY, "incorrect_message", VOID])
# Action: exit the program
actions.append([RUN, "run_exit", [INCORRECT_ARGUMENTS]])
return gg_arguments, actions
def test_optional_commandline_arguments_can_verify(commandline_arguments):
"""Check that correct optional command-line arguments check correctly."""
gg_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(gg_arguments)
assert args_verified is True
def test_act_produces_output_with_exact(
commandline_arguments,
chosen_file,
containing_directory,
provided_count,
expected_result,
tmpdir,
load_checker,
):
"""Check that using the check produces output."""
test_contents = """
## What Do People Think about GatorGrader?
GatorGrader addresses many of the challenges that an instructor faces when
designing automated checks for the source code or technical writing that a
## Installing GatorGrader
Installing GatorGrader is not necessary if you intend to use it through its
[Gradle plugin](https://github.com/GatorEducator/gatorgradle). If you want to
## Running GatorGrader
Students and instructors normally use GatorGrader through its [Gradle
plugin](https://github.com/GatorEducator/gatorgradle), specifying the requested
## Testing GatorGrader
### Automated Testing
The developers use [Pytest](https://docs.pytest.org/en/latest/) for the testing
of GatorGrader. Depending on your goals, there are several different..."""
testargs = [os.getcwd()]
with patch.object(sys, "argv", testargs):
new_file = tmpdir.mkdir(containing_directory).join(chosen_file)
new_file.write(test_contents)
assert new_file.read() == test_contents
assert len(tmpdir.listdir()) == 1
overall_directory = (
tmpdir.dirname + "/" + tmpdir.basename + "/" + containing_directory
)
commandline_arguments = [
"CountMarkdownTags",
"--file",
"file_to_find",
"--directory",
overall_directory,
"--count",
provided_count,
"--tag",
"heading",
"--exact",
]
parsed_arguments, remaining_arguments = arguments.parse(commandline_arguments)
args_verified = arguments.verify(parsed_arguments)
assert args_verified is True
check_exists, checker_source, check_file = load_checker(parsed_arguments)
assert check_exists is True
check = checker_source.load_plugin(check_file)
check_result = check.act(parsed_arguments, remaining_arguments)
# check the result
assert check_result is not None
assert len(check_result) == 1
# assert check_result[0] is expected_result
# check the contents of the report
assert report.get_result() is not None
assert len(report.get_result()["check"]) > 1
assert report.get_result()["outcome"] is expected_result
if expected_result:
assert report.get_result()["diagnostic"] == ""
else:
assert report.get_result()["diagnostic"] != ""
def test_exact_count_check_valid(chosen_arguments):
"""Check that invalid argument combinations do verify correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
verified_arguments = arguments.is_valid_exact(parsed_arguments)
assert verified_arguments is True
def parse_arguments(system_arguments):
"""Parse and then return the parsed command-line arguments and those that remain."""
parsed_arguments, remaining_arguments = arguments.parse(system_arguments)
return parsed_arguments, remaining_arguments
def test_is_valid_count_with_file_or_command(chosen_arguments):
"""Check that invalid argument combinations do not verify correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
verified_arguments = arguments.is_valid_count(parsed_arguments)
assert verified_arguments is True
def test_is_valid_words_valid(chosen_arguments):
"""Check that valid argument combinations do not verify correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
verified_arguments = arguments.is_valid_words(parsed_arguments)
assert verified_arguments is True
def test_invalid_argument_combinations_accepted(chosen_arguments):
"""Check that not valid argument combinations do not verify correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
verified_arguments = arguments.verify(parsed_arguments)
assert verified_arguments is False
def test_is_not_valid_language_combinations_wrong(chosen_arguments):
"""Check that invalid argument combinations do not verify correctly"""
parsed_arguments = arguments.parse(chosen_arguments)
verified_arguments = arguments.is_valid_language(parsed_arguments)
assert verified_arguments is False
def test_arguments_verified(verifiable_gg_args):
"""Run arguments with verifiable arguments and it is verified"""
gg_arguments = arguments.parse(verifiable_gg_args)
gg_args_verified = arguments.verify(gg_arguments)
assert gg_args_verified == VERIFIED
def test_default_argument_values_correct(no_gg_args):
"""The default command-line arguments are correct"""
gg_arguments = arguments.parse(no_gg_args)
arguments_args_verified = arguments.verify(gg_arguments)
assert arguments_args_verified == NOT_VERIFIED
