def test_stderr_correctness_show_expected_and_actual(self):
self.ag_test_command.validate_and_update(normal_fdbk_config={
'stderr_fdbk_level':
ag_models.ValueFeedbackLevel.expected_and_actual
})
correct_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal)
self.assertTrue(fdbk.stderr_correct)
diff = _get_expected_diff(self.ag_test_command.expected_stderr_text,
correct_result.stderr_filename)
self.assertEqual(diff.diff_content, fdbk.stderr_diff.diff_content)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points_possible)
correct_result.delete()
incorrect_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_result,
ag_models.FeedbackCategory.normal)
self.assertFalse(fdbk.stderr_correct)
diff = _get_expected_diff(self.ag_test_command.expected_stderr_text,
incorrect_result.stderr_filename)
self.assertEqual(diff.diff_content, fdbk.stderr_diff.diff_content)
self.assertEqual(self.ag_test_command.deduction_for_wrong_stderr,
fdbk.stderr_points)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points_possible)
def test_stderr_correctness_show_correct_or_incorrect(self):
self.ag_test_command.validate_and_update(normal_fdbk_config={
'stderr_fdbk_level':
ag_models.ValueFeedbackLevel.correct_or_incorrect
})
correct_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal)
self.assertTrue(fdbk.stderr_correct)
self.assertIsNone(fdbk.stderr_diff)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points_possible)
correct_result.delete()
incorrect_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_result,
ag_models.FeedbackCategory.normal)
self.assertFalse(fdbk.stderr_correct)
self.assertEqual(self.ag_test_command.deduction_for_wrong_stderr,
fdbk.stderr_points)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points_possible)
def test_stderr_correctness_show_diff_from_file(self):
instructor_file = obj_build.make_instructor_file(self.project)
self.ag_test_command.expected_stderr_source = (
ag_models.ExpectedOutputSource.instructor_file)
self.ag_test_command.expected_stderr_instructor_file = instructor_file
self.ag_test_command.save()
result = self.make_correct_result()
with instructor_file.open() as f:
expected_stderr = f.read()
result.stderr = expected_stderr
result.save()
diff = _get_expected_diff(expected_stderr, result.stderr_filename)
self.assertEqual(
diff.diff_content,
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.normal).stderr_diff.diff_content)
result.stderr = 'the wrong stderr'
result.save()
diff = _get_expected_diff(expected_stderr, result.stderr_filename)
self.assertEqual(
diff.diff_content,
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.normal).stderr_diff.diff_content)
def test_return_code_correctness_show_expected_and_actual(self):
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'return_code_fdbk_level':
ag_models.ValueFeedbackLevel.expected_and_actual
})
correct_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal)
self.assertTrue(fdbk.return_code_correct)
self.assertEqual(self.ag_test_command.expected_return_code,
fdbk.expected_return_code)
self.assertEqual(correct_result.return_code, fdbk.actual_return_code)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points_possible)
correct_result.delete()
incorrect_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_result,
ag_models.FeedbackCategory.normal)
self.assertFalse(fdbk.return_code_correct)
self.assertEqual(self.ag_test_command.expected_return_code,
fdbk.expected_return_code)
self.assertEqual(incorrect_result.return_code, fdbk.actual_return_code)
self.assertEqual(self.ag_test_command.deduction_for_wrong_return_code,
fdbk.return_code_points)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points_possible)
def test_stderr_not_checked(self):
self.ag_test_command.validate_and_update(
expected_stderr_source=ag_models.ExpectedOutputSource.none)
correct_cmd_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_cmd_result, ag_models.FeedbackCategory.max)
self.assertIsNone(fdbk.stderr_correct)
self.assertIsNone(fdbk.stderr_diff)
self.assertEqual(_stderr_text(correct_cmd_result), _stderr_text(fdbk))
self.assertEqual(0, fdbk.stderr_points)
self.assertEqual(0, fdbk.stderr_points_possible)
expected_total_pts = (
self.ag_test_command.points_for_correct_return_code +
self.ag_test_command.points_for_correct_stdout)
expected_total_pts_possible = expected_total_pts
self.assertEqual(expected_total_pts, fdbk.total_points)
self.assertEqual(expected_total_pts_possible,
fdbk.total_points_possible)
correct_cmd_result.delete()
incorrect_cmd_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_cmd_result,
ag_models.FeedbackCategory.max)
self.assertIsNone(fdbk.stderr_correct)
self.assertEqual(0, fdbk.stderr_points)
self.assertEqual(0, fdbk.stderr_points_possible)
expected_total_pts = (
self.ag_test_command.deduction_for_wrong_return_code +
self.ag_test_command.deduction_for_wrong_stdout)
self.assertEqual(expected_total_pts, fdbk.total_points)
self.assertEqual(expected_total_pts_possible,
fdbk.total_points_possible)
def test_return_code_not_checked(self):
self.ag_test_command.validate_and_update(
expected_return_code=ag_models.ExpectedReturnCode.none)
correct_cmd_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_cmd_result, ag_models.FeedbackCategory.max)
self.assertEqual(ag_models.ExpectedReturnCode.none,
fdbk.expected_return_code)
self.assertIsNone(fdbk.return_code_correct)
self.assertEqual(0, fdbk.return_code_points)
self.assertEqual(0, fdbk.return_code_points_possible)
expected_total_pts = (self.ag_test_command.points_for_correct_stdout +
self.ag_test_command.points_for_correct_stderr)
self.assertEqual(expected_total_pts, fdbk.total_points)
expected_total_pts_possible = expected_total_pts
self.assertEqual(expected_total_pts_possible,
fdbk.total_points_possible)
correct_cmd_result.delete()
incorrect_cmd_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_cmd_result,
ag_models.FeedbackCategory.max)
self.assertIsNone(fdbk.return_code_correct)
self.assertEqual(0, fdbk.return_code_points)
self.assertEqual(0, fdbk.return_code_points_possible)
expected_total_pts = (self.ag_test_command.deduction_for_wrong_stdout +
self.ag_test_command.deduction_for_wrong_stderr)
self.assertEqual(expected_total_pts, fdbk.total_points)
self.assertEqual(expected_total_pts_possible,
fdbk.total_points_possible)
def test_feedback_calculator_factory_method(self):
# check against the actual objects (their pks)
result = self.make_correct_result()
self.assertEqual(
self.ag_test_command.normal_fdbk_config.to_dict(),
get_cmd_fdbk(
result, ag_models.FeedbackCategory.normal).fdbk_conf.to_dict())
self.assertEqual(
self.ag_test_command.ultimate_submission_fdbk_config.to_dict(),
get_cmd_fdbk(result, ag_models.FeedbackCategory.ultimate_submission
).fdbk_conf.to_dict())
self.assertEqual(
self.ag_test_command.past_limit_submission_fdbk_config.to_dict(),
get_cmd_fdbk(result, ag_models.FeedbackCategory.
past_limit_submission).fdbk_conf.to_dict())
self.assertEqual(
self.ag_test_command.staff_viewer_fdbk_config.to_dict(),
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.staff_viewer).fdbk_conf.to_dict())
max_fdbk = get_cmd_fdbk(result,
ag_models.FeedbackCategory.max).fdbk_conf
self.assertEqual(ag_models.ValueFeedbackLevel.get_max(),
max_fdbk.return_code_fdbk_level)
self.assertEqual(ag_models.ValueFeedbackLevel.get_max(),
max_fdbk.stdout_fdbk_level)
self.assertEqual(ag_models.ValueFeedbackLevel.get_max(),
max_fdbk.stderr_fdbk_level)
self.assertTrue(max_fdbk.show_points)
self.assertTrue(max_fdbk.show_actual_return_code)
self.assertTrue(max_fdbk.show_actual_stdout)
self.assertTrue(max_fdbk.show_actual_stderr)
self.assertTrue(max_fdbk.show_whether_timed_out)
def test_show_timed_out(self):
result = self.make_correct_result()
self.assertFalse(
get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out)
result.delete()
result = self.make_incorrect_result()
self.assertTrue(
get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out)
def test_hide_timed_out(self):
self.ag_test_command.validate_and_update(
normal_fdbk_config={'show_whether_timed_out': False})
result = self.make_correct_result()
self.assertIsNone(
get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out)
result.delete()
result = self.make_incorrect_result()
self.assertIsNone(
get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out)
def test_fdbk_to_dict(self):
result = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result)
expected_keys = [
'pk',
'ag_test_command_name',
'ag_test_command_pk',
'fdbk_settings',
'timed_out',
'return_code_correct',
'expected_return_code',
'actual_return_code',
'return_code_points',
'return_code_points_possible',
'stdout_correct',
'stdout_points',
'stdout_points_possible',
'stderr_correct',
'stderr_points',
'stderr_points_possible',
'total_points',
'total_points_possible',
]
self.assertCountEqual(
expected_keys,
get_cmd_fdbk(result,
ag_models.FeedbackCategory.max).to_dict().keys())
def test_points_visibility(self):
self.ag_test_command.validate_and_update(
normal_fdbk_config={'show_points': False})
correct_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal)
self.assertEqual(0, fdbk.total_points)
self.assertEqual(0, fdbk.total_points_possible)
correct_result.delete()
incorrect_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_result,
ag_models.FeedbackCategory.normal)
self.assertEqual(0, fdbk.total_points)
self.assertEqual(0, fdbk.total_points_possible)
def test_timed_out_with_return_code_stdout_and_stderr_None_count_as_wrong(
self):
result = ag_models.AGTestCommandResult.objects.validate_and_create(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result,
timed_out=True)
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.max)
self.assertFalse(fdbk.return_code_correct)
self.assertFalse(fdbk.stdout_correct)
self.assertFalse(fdbk.stderr_correct)
def test_return_code_correctness_hidden(self):
self.ag_test_command.validate_and_update(normal_fdbk_config={
'return_code_fdbk_level':
ag_models.ValueFeedbackLevel.no_feedback
})
correct_result = self.make_correct_result()
fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal)
self.assertIsNone(fdbk.return_code_correct)
self.assertEqual(0, fdbk.return_code_points)
self.assertEqual(0, fdbk.return_code_points_possible)
correct_result.delete()
incorrect_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(incorrect_result,
ag_models.FeedbackCategory.normal)
self.assertIsNone(fdbk.return_code_correct)
self.assertEqual(0, fdbk.return_code_points)
self.assertEqual(0, fdbk.return_code_points_possible)
def test_some_ag_test_cases_not_visible(self):
self.ag_test_case2.validate_and_update(
ultimate_submission_fdbk_config={'visible': False})
expected_points = get_cmd_fdbk(
self.cmd_result1,
ag_models.FeedbackCategory.max).total_points_possible
fdbk = get_suite_fdbk(self.ag_test_suite_result,
ag_models.FeedbackCategory.ultimate_submission)
self.assertSequenceEqual([self.ag_test_case_result1.pk],
[res.pk for res in fdbk.ag_test_case_results])
self.assertEqual(expected_points, fdbk.total_points)
self.assertEqual(expected_points, fdbk.total_points_possible)
def test_stderr_show_actual(self):
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'stderr_fdbk_level': ag_models.ValueFeedbackLevel.no_feedback,
'show_actual_stderr': True
})
result = self.make_correct_result()
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal)
self.assertEqual(_stderr_text(result), _stderr_text(fdbk))
self.assertIsNone(fdbk.stderr_correct)
self.assertEqual(0, fdbk.stderr_points)
self.assertEqual(0, fdbk.stderr_points_possible)
def test_fdbk_to_dict(self):
self.maxDiff = None
self.ag_test_cmd1.validate_and_update(
points_for_correct_return_code=2,
points_for_correct_stdout=3,
points_for_correct_stderr=8
)
self.ag_test_cmd2.validate_and_update(
points_for_correct_return_code=4,
points_for_correct_stdout=5,
points_for_correct_stderr=7
)
result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_test_case_result)
result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result)
expected_keys = [
'pk',
'ag_test_case_name',
'ag_test_case_pk',
'fdbk_settings',
'total_points',
'total_points_possible',
'ag_test_command_results',
]
for fdbk_category in ag_models.FeedbackCategory:
result_dict = get_case_fdbk(self.ag_test_case_result, fdbk_category).to_dict()
self.assertCountEqual(expected_keys, result_dict.keys())
self.assertCountEqual(
[get_cmd_fdbk(result1, fdbk_category).to_dict(),
get_cmd_fdbk(result2, fdbk_category).to_dict()],
result_dict['ag_test_command_results'])
def test_stderr_show_expected_and_actual_overrides_show_actual(self):
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'stderr_fdbk_level':
ag_models.ValueFeedbackLevel.expected_and_actual,
'show_actual_stderr': False
})
result = self.make_correct_result()
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal)
self.assertIsNotNone(fdbk.stderr_diff)
self.assertIsNotNone(fdbk.stderr)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points)
def do_diff_options_test(self,
expected_stdout='',
actual_stdout='',
expected_stderr='',
actual_stderr='',
expect_stdout_correct=True,
expect_stderr_correct=True,
**diff_options):
self.ag_test_command.validate_and_update(
expected_stdout_text=expected_stdout,
expected_stderr_text=expected_stderr,
**diff_options)
result = self.make_correct_result()
with open(result.stdout_filename, 'w') as f:
f.write(actual_stdout)
actual_stdout_filename = f.name
with open(result.stderr_filename, 'w') as f:
f.write(actual_stderr)
actual_stderr_filename = f.name
mock_path = 'autograder.core.utils.get_diff'
with mock.patch(mock_path) as mock_differ_cls:
diff = get_cmd_fdbk(result,
ag_models.FeedbackCategory.max).stdout_diff
mock_differ_cls.assert_called_with(mock.ANY,
actual_stdout_filename,
**diff_options)
with mock.patch(mock_path) as mock_differ_cls:
diff = get_cmd_fdbk(result,
ag_models.FeedbackCategory.max).stderr_diff
mock_differ_cls.assert_called_with(mock.ANY,
actual_stderr_filename,
**diff_options)
if expect_stdout_correct:
diff = get_cmd_fdbk(result,
ag_models.FeedbackCategory.max).stdout_diff
self.assertTrue(diff.diff_pass, msg=diff.diff_content)
else:
self.assertFalse(
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.max).stdout_diff.diff_pass)
if expect_stderr_correct:
self.assertTrue(
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.max).stderr_diff.diff_pass)
else:
self.assertFalse(
get_cmd_fdbk(
result,
ag_models.FeedbackCategory.max).stderr_diff.diff_pass)
def test_return_code_show_actual(self):
# Show actual return code, but not expected return code or points.
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'return_code_fdbk_level':
ag_models.ValueFeedbackLevel.no_feedback,
'show_actual_return_code': True
})
result = self.make_correct_result()
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal)
self.assertEqual(result.return_code, fdbk.actual_return_code)
self.assertIsNone(fdbk.return_code_correct)
self.assertEqual(0, fdbk.return_code_points)
self.assertEqual(0, fdbk.return_code_points_possible)
def test_return_code_show_expected_and_actual_overrides_show_actual(self):
# Show expected and actual return code (overrides show_actual_return_code setting).
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'return_code_fdbk_level':
ag_models.ValueFeedbackLevel.expected_and_actual,
'show_actual_return_code': False
})
result = self.make_correct_result()
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal)
self.assertIsNotNone(fdbk.actual_return_code)
self.assertIsNotNone(fdbk.expected_return_code)
self.assertTrue(fdbk.return_code_correct)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points)
def test_stderr_show_correctness_but_not_actual(self):
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'stderr_fdbk_level':
ag_models.ValueFeedbackLevel.correct_or_incorrect,
'show_actual_stderr': False
})
result = self.make_correct_result()
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal)
self.assertIsNone(fdbk.stderr)
self.assertTrue(fdbk.stderr_correct)
self.assertIsNone(fdbk.stderr_diff)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points_possible)
def test_return_code_show_correctness_but_not_actual(self):
# Show return code correctness and points, but not actual return code.
self.ag_test_command.validate_and_update(
normal_fdbk_config={
'return_code_fdbk_level':
ag_models.ValueFeedbackLevel.correct_or_incorrect,
'show_actual_return_code': False
})
result = self.make_correct_result()
fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal)
self.assertIsNone(fdbk.actual_return_code)
self.assertIsNone(fdbk.expected_return_code)
self.assertTrue(fdbk.return_code_correct)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points_possible)
def test_points_everything_incorrect_max_fdbk(self):
cmd_result = self.make_incorrect_result()
fdbk = get_cmd_fdbk(cmd_result, ag_models.FeedbackCategory.max)
self.assertEqual(self.ag_test_command.deduction_for_wrong_return_code,
fdbk.return_code_points)
self.assertEqual(self.ag_test_command.points_for_correct_return_code,
fdbk.return_code_points_possible)
self.assertEqual(self.ag_test_command.deduction_for_wrong_stdout,
fdbk.stdout_points)
self.assertEqual(self.ag_test_command.points_for_correct_stdout,
fdbk.stdout_points_possible)
self.assertEqual(self.ag_test_command.deduction_for_wrong_stderr,
fdbk.stderr_points)
self.assertEqual(self.ag_test_command.points_for_correct_stderr,
fdbk.stderr_points_possible)
self.assertEqual(self.min_points_possible, fdbk.total_points)
self.assertEqual(self.max_points_possible, fdbk.total_points_possible)
