def stderr_diff(self) -> Optional[core_ut.DiffResult]:
if (self._cmd.expected_stderr_source == ExpectedOutputSource.none
or self._fdbk.stderr_fdbk_level != ValueFeedbackLevel.expected_and_actual):
return None
stderr_filename = self._ag_test_command_result.stderr_filename
diff_whitespace_kwargs = {
'ignore_blank_lines': self._cmd.ignore_blank_lines,
'ignore_case': self._cmd.ignore_case,
'ignore_whitespace': self._cmd.ignore_whitespace,
'ignore_whitespace_changes': self._cmd.ignore_whitespace_changes
}
if self._cmd.expected_stderr_source == ExpectedOutputSource.text:
with tempfile.NamedTemporaryFile('w') as expected_stderr:
expected_stderr.write(self._cmd.expected_stderr_text)
expected_stderr.flush()
return core_ut.get_diff(expected_stderr.name, stderr_filename,
**diff_whitespace_kwargs)
elif self._cmd.expected_stderr_source == ExpectedOutputSource.instructor_file:
return core_ut.get_diff(self._cmd.expected_stderr_instructor_file.abspath,
stderr_filename,
**diff_whitespace_kwargs)
else:
raise ValueError(
'Invalid expected stderr source: {}'.format(self._cmd.expected_stdout_source))
def test_ignore_blank_lines(self):
self._write_and_seek(self.file1, 'spam\n\n\negg\n')
self._write_and_seek(self.file2, 'spam\negg\n')
result = core_ut.get_diff(self.file1.name,
self.file2.name,
ignore_blank_lines=True)
self.assertTrue(result.diff_pass)
def test_ignore_case(self):
self._write_and_seek(self.file1, 'SPAM')
self._write_and_seek(self.file2, 'spam')
result = core_ut.get_diff(self.file1.name,
self.file2.name,
ignore_case=True)
self.assertTrue(result.diff_pass)
def test_ignore_whitespace_changes(self):
self._write_and_seek(self.file1, 'spam egg')
self._write_and_seek(self.file2, 'spam \tegg')
result = core_ut.get_diff(self.file1.name,
self.file2.name,
ignore_whitespace_changes=True)
self.assertTrue(result.diff_pass)
def test_weird_line_endings(self):
self._write_and_seek(self.file1, '''egg
\r
cheese''')
self._write_and_seek(self.file2, '''egg
cheese\r
''')
expected = [' egg\n', '- \r\n', '- cheese', '+ cheese\r\n']
diff = core_ut.get_diff(self.file1.name, self.file2.name)
self.assertEqual(expected, list(diff.diff_content))
def test_all_ignore_options(self):
self._write_and_seek(self.file1, 'spam sausage\n\n\negg\n')
self._write_and_seek(self.file2, 'SPAM \tsausage\negg\n')
result = core_ut.get_diff(self.file1.name,
self.file2.name,
ignore_case=True,
ignore_whitespace=True,
ignore_whitespace_changes=True,
ignore_blank_lines=True)
self.assertTrue(result.diff_pass)
def test_trailing_newline_missing(self):
self._write_and_seek(self.file1, '''egg
sausage''')
self._write_and_seek(self.file2, '''spam
egg
sausage
''')
expected = ['+ spam\n', ' egg\n', '- sausage', '+ sausage\n']
diff = core_ut.get_diff(self.file1.name, self.file2.name)
self.assertEqual(expected, list(diff.diff_content))
def test_non_utf_chars(self):
non_utf_bytes = b'\x80 and some other stuff just because\n'
self._write_and_seek(self.file1, b'some stuff')
self._write_and_seek(self.file2, non_utf_bytes)
expected_diff = [
'- some stuff',
'+ ' + non_utf_bytes.decode('utf-8', 'surrogateescape')
]
diff = core_ut.get_diff(self.file1.name, self.file2.name)
self.assertEqual(expected_diff, diff.diff_content)
def test_diff_delta_strs_in_files(self):
self._write_and_seek(self.file1, '''egg+ cheese
spam- sausage
''')
self._write_and_seek(self.file2, '''egg
cheese
sausage''')
expected = [
'- egg+ cheese\n', '- spam- sausage\n', '+ egg\n', '+ cheese\n',
'+ sausage'
]
diff = core_ut.get_diff(self.file1.name, self.file2.name)
self.assertEqual(expected, list(diff.diff_content))
def test_diff_content(self):
self._write_and_seek(self.file1, '\n'.join(
('q', 'a', 'b', 'x', 'c', 'd', 'e\n')))
self._write_and_seek(self.file2, '\n'.join(
('a', 'b', 'y', 'c', 'd', 'f', 'e\n')))
expected = [
'- q\n', ' a\n', ' b\n', '- x\n', '+ y\n', ' c\n', ' d\n',
'+ f\n', ' e\n'
]
diff = core_ut.get_diff(self.file1.name, self.file2.name)
self.assertEqual(expected, list(diff.diff_content))
def grade_ag_test_command_impl(sandbox: AutograderSandbox,
ag_test_cmd: ag_models.AGTestCommand,
case_result: ag_models.AGTestCaseResult):
with FileCloser() as file_closer:
run_result = run_ag_test_command(ag_test_cmd, sandbox,
case_result.ag_test_suite_result)
result_data = {
'return_code': run_result.return_code,
'timed_out': run_result.timed_out,
'stdout_truncated': run_result.stdout_truncated,
'stderr_truncated': run_result.stderr_truncated,
}
if ag_test_cmd.expected_return_code == ag_models.ExpectedReturnCode.zero:
result_data['return_code_correct'] = run_result.return_code == 0
elif ag_test_cmd.expected_return_code == ag_models.ExpectedReturnCode.nonzero:
result_data['return_code_correct'] = run_result.return_code != 0
expected_stdout, expected_stdout_filename = _get_expected_stdout_file_and_name(
ag_test_cmd)
file_closer.register_file(expected_stdout)
if expected_stdout_filename is not None:
diff = core_ut.get_diff(
expected_stdout_filename,
run_result.stdout.name,
ignore_case=ag_test_cmd.ignore_case,
ignore_whitespace=ag_test_cmd.ignore_whitespace,
ignore_whitespace_changes=ag_test_cmd.
ignore_whitespace_changes,
ignore_blank_lines=ag_test_cmd.ignore_blank_lines)
result_data['stdout_correct'] = diff.diff_pass
expected_stderr, expected_stderr_filename = _get_expected_stderr_file_and_name(
ag_test_cmd)
file_closer.register_file(expected_stderr)
if expected_stderr_filename is not None:
diff = core_ut.get_diff(
expected_stderr_filename,
run_result.stderr.name,
ignore_case=ag_test_cmd.ignore_case,
ignore_whitespace=ag_test_cmd.ignore_whitespace,
ignore_whitespace_changes=ag_test_cmd.
ignore_whitespace_changes,
ignore_blank_lines=ag_test_cmd.ignore_blank_lines)
result_data['stderr_correct'] = diff.diff_pass
print(result_data)
@retry_should_recover
def save_ag_test_cmd_result():
with transaction.atomic():
cmd_result = ag_models.AGTestCommandResult.objects.update_or_create(
defaults=result_data,
ag_test_command=ag_test_cmd,
ag_test_case_result=case_result)[
0] # type: ag_models.AGTestCommandResult
with open(cmd_result.stdout_filename, 'wb') as f:
shutil.copyfileobj(run_result.stdout, f)
with open(cmd_result.stderr_filename, 'wb') as f:
shutil.copyfileobj(run_result.stderr, f)
save_ag_test_cmd_result()
def _get_expected_diff(expected_text: str, actual_output_filename: str):
with tempfile.NamedTemporaryFile('w') as f:
f.write(expected_text)
f.flush()
return core_ut.get_diff(f.name, actual_output_filename)
