def test_simple_json_output():
output = StringIO()
reporter = JSONReporter()
linter = PyLinter(reporter=reporter)
checkers.initialize(linter)
linter.config.persistent = 0
linter.reporter.set_output(output)
linter.open()
linter.set_current_module("0123")
linter.add_message("line-too-long", line=1, args=(1, 2))
# we call this method because we didn't actually run the checkers
reporter.display_messages(None)
expected_result = [[
("column", 0),
("line", 1),
("message", "Line too long (1/2)"),
("message-id", "C0301"),
("module", "0123"),
("obj", ""),
("path", "0123"),
("symbol", "line-too-long"),
("type", "convention"),
]]
report_result = json.loads(output.getvalue())
report_result = [
sorted(report_result[0].items(), key=lambda item: item[0])
]
assert report_result == expected_result
def test_json_report_when_file_has_syntax_error(self):
out = StringIO()
module = join(HERE, "regrtest_data", "syntax_error.py")
self._runtest([module], code=2, reporter=JSONReporter(out))
output = json.loads(out.getvalue())
assert isinstance(output, list)
assert len(output) == 1
assert isinstance(output[0], dict)
# So each version wants a different column number...
if platform.python_implementation() == "PyPy":
column = 9
elif sys.version_info >= (3, 8):
column = 9
else:
column = 15
expected = {
"obj": "",
"column": column,
"line": 1,
"type": "error",
"symbol": "syntax-error",
"module": "syntax_error",
}
message = output[0]
for key, value in expected.items():
assert key in message
assert message[key] == value
assert "invalid syntax" in message["message"].lower()
assert "<unknown>" in message["message"].lower()
def run_pylint(bundle: Bundle, submission: Path, remaining: float) \
-> Tuple[List[Message], List[AnnotateCode]]:
"""
Calls pylint to annotate submitted source code and adds resulting score and
annotations to tab.
"""
config = bundle.config
language_options = bundle.config.config_for()
if language_options.get("pylint_config", None):
config_path = config.resources / language_options.get('pylint_config')
else:
# Use the default file.
config_path = config.judge / "tested/languages/python/pylint_config.rc"
pylint_out = StringIO()
try:
args = [f"--rcfile={config_path}", str(submission)]
logger.debug("Running with template_args %s", args)
lint.Run(args, reporter=JSONReporter(output=pylint_out), do_exit=False)
except Exception as e:
logger.warning("Pylint crashed with", exc_info=e)
return [
"Pylint crashed",
ExtendedMessage(description=str(e),
format='code',
permission=Permission.STAFF)
], []
try:
messages = json.loads(pylint_out.getvalue())
except Exception as e:
logger.warning("Pylint produced bad output", exc_info=e)
return [
"Pylint produced bad output.",
ExtendedMessage(description=str(e),
format='code',
permission=Permission.STAFF)
], []
annotations = []
for message in messages:
category = message_categories.get(message["type"], Severity.WARNING)
logger.debug("Handling message %s", str(message))
annotations.append(
AnnotateCode(
row=max(int(message["line"]) - 1, 0),
column=max(int(message["column"]) - 1, 0),
text=f"{message['message']} ({message['message-id']})",
type=category))
# sort linting messages on line, column and code
annotations.sort(key=lambda a: (a.row, a.column, a.text))
# for now, reports are not processed
return [], annotations
def _lint_package(self, data: PackageToLint) -> list[dict[str, str | int]]:
# We want to test all the code we can
enables = ["--enable-all-extensions", "--enable=all"]
# Duplicate code takes too long and is relatively safe
# TODO: Find a way to allow cyclic-import and compare output correctly
disables = ["--disable=duplicate-code,cyclic-import"]
arguments = data.pylint_args + enables + disables
output = StringIO()
reporter = JSONReporter(output)
Run(arguments, reporter=reporter, exit=False)
return json.loads(output.getvalue())
def test_all(self):
"""Make pylint check itself."""
reporters = [
TextReporter(StringIO()),
ColorizedTextReporter(StringIO()),
JSONReporter(StringIO()),
]
self._runtest(
[join(HERE, "functional", "a", "arguments.py")],
reporter=MultiReporter(reporters),
code=2,
)
def test_json_report_does_not_escape_quotes(self):
out = StringIO()
module = join(HERE, "regrtest_data", "unused_variable.py")
self._runtest([module], code=4, reporter=JSONReporter(out))
output = json.loads(out.getvalue())
assert isinstance(output, list)
assert len(output) == 1
assert isinstance(output[0], dict)
expected = {
"symbol": "unused-variable",
"module": "unused_variable",
"column": 4,
"message": "Unused variable 'variable'",
"message-id": "W0612",
"line": 4,
"type": "warning",
}
message = output[0]
for key, value in expected.items():
assert key in message
assert message[key] == value
def test_json_report_when_file_is_missing(self):
out = StringIO()
module = join(HERE, "regrtest_data", "totally_missing.py")
self._runtest([module], code=1, reporter=JSONReporter(out))
output = json.loads(out.getvalue())
assert isinstance(output, list)
assert len(output) == 1
assert isinstance(output[0], dict)
expected = {
"obj": "",
"column": 0,
"line": 1,
"type": "fatal",
"symbol": "fatal",
"module": module,
}
message = output[0]
for key, value in expected.items():
assert key in message
assert message[key] == value
assert message["message"].startswith("No module named")
def test_json_report_when_file_has_syntax_error(self):
out = StringIO()
module = join(HERE, "regrtest_data", "syntax_error.py")
self._runtest([module], code=2, reporter=JSONReporter(out))
output = json.loads(out.getvalue())
assert isinstance(output, list)
assert len(output) == 1
assert isinstance(output[0], dict)
expected = {
"obj": "",
"column": 8 if platform.python_implementation() == "PyPy" else 15,
"line": 1,
"type": "error",
"symbol": "syntax-error",
"module": "syntax_error",
}
message = output[0]
for key, value in expected.items():
assert key in message
assert message[key] == value
assert "invalid syntax" in message["message"].lower()
def get_linter_result(score):
output = StringIO()
reporter = JSONReporter(output)
linter = PyLinter(reporter=reporter)
checkers.initialize(linter)
linter.config.persistent = 0
linter.config.score = score
linter.open()
linter.set_current_module("0123")
linter.add_message("line-too-long", line=1, args=(1, 2))
# we call those methods because we didn't actually run the checkers
if score:
reporter.display_reports(EvaluationSection(expected_score_message))
reporter.display_messages(None)
report_result = json.loads(output.getvalue())
return report_result
def _run_pylint(code: str, errors: list[str] | None) -> list[dict]:
"""
Runs pylint on the given code and returns a list of dictionaries
containing the resulting errors or warnings.
"""
# Stdin swapping is faster than running a subprocess. Although this
# likely can't be used with multiprocessing.
pylint_output = StringIO()
# This is why you don't diddle with other people's IO without asking.
sys.stdin = TextIOWrapper(BytesIO(code.encode()))
commands = []
if errors is not None:
commands += ('--disable', 'all', '--enable', *errors)
commands += ('--from-stdin', '_pylint_runner')
Run(
commands,
reporter=JSONReporter(pylint_output),
do_exit=False,
)
sys.stdin = sys.__stdin__
result = pylint_output.getvalue()
return json.loads(result)
def get_linter_result(score: bool, message: Dict[str,
Any]) -> List[Dict[str, Any]]:
output = StringIO()
reporter = JSONReporter(output)
linter = PyLinter(reporter=reporter)
checkers.initialize(linter)
linter.namespace.persistent = 0
linter.namespace.score = score
linter.open()
linter.set_current_module("0123")
linter.add_message(
message["msg"],
line=message["line"],
args=message["args"],
end_lineno=message["end_line"],
end_col_offset=message["end_column"],
)
# we call those methods because we didn't actually run the checkers
if score:
reporter.display_reports(EvaluationSection(expected_score_message))
reporter.display_messages(None)
report_result = json.loads(output.getvalue())
return report_result
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.console = text.ColorizedTextReporter()
self.json = JSONReporter(output=open('pylint.json', 'w'))
self._start = None
def run_pylint(bundle: Bundle, submission: Path, remaining: float) \
-> Tuple[List[Message], List[AnnotateCode]]:
"""
Calls pylint to annotate submitted source code and adds resulting score and
annotations to tab.
"""
config = bundle.config
language_options = bundle.config.config_for()
if language_options.get("pylint_config", None):
config_path = config.resources / language_options.get('pylint_config')
else:
# Use the default file.
config_path = config.judge / "tested/languages/python/pylint_config.rc"
pylint_out = StringIO()
try:
args = [f"--rcfile={config_path}", str(submission)]
logger.debug("Running with template_args %s", args)
lint.Run(args, reporter=JSONReporter(output=pylint_out), do_exit=False)
except Exception as e:
logger.warning("Pylint crashed with", exc_info=e)
return [get_i18n_string("languages.python.linter.crashed"), ExtendedMessage(
description=str(e),
format='code',
permission=Permission.STAFF
)], []
try:
messages = json.loads(pylint_out.getvalue())
except Exception as e:
logger.warning("Pylint produced bad output", exc_info=e)
return [get_i18n_string("languages.python.linter.output"), ExtendedMessage(
description=str(e),
format='code',
permission=Permission.STAFF
)], []
annotations = []
for message in messages:
category = message_categories.get(message.get("type", "warning"),
Severity.WARNING)
logger.debug("Handling message %s", str(message))
message_id = message.get('message-id', None)
message_text = message.get('message', None)
more_info = get_i18n_string("languages.linter.more-info")
if not message_id and not message_text:
continue
elif not message_id:
text = message_text
elif not message_text:
text = f'({message_id}, <a href="https://pylint.pycqa.org/en/latest/' \
f'technical_reference/features.html#basic-checker-messages" ' \
f'target="_blank">{more_info}</a>)'
else:
text = f'{message_text} ({message_id},' \
f'<a href="https://pylint.pycqa.org/en/latest/' \
f'technical_reference/features.html#basic-checker-messages" ' \
f'target="_blank">{more_info}</a>)'
annotations.append(AnnotateCode(
row=max(int(message.get("line", "-1")) - 1, 0),
column=max(int(message.get("column", "-1")) - 1, 0),
text=text,
type=category
))
# sort linting messages on line, column and code
annotations.sort(key=lambda a: (a.row, a.column, a.text))
# for now, reports are not processed
return [], annotations