def test_non_default_docker_image(self, *args):
suite = obj_build.make_ag_test_suite(
self.project, docker_image_to_use=constants.SupportedImages.eecs490)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
cmd='racket --version',
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=3,
expected_return_code=ag_models.ExpectedReturnCode.zero)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
self.assertEqual(0, cmd_result.return_code)
self.assertEqual(
3,
get_submission_fdbk(
self.submission, ag_models.FeedbackCategory.max).total_points_possible)
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_two_best_dont_count_for_user(self):
self.project.validate_and_update(
ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.best)
suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='suite',
project=self.project,
buggy_impl_names=[f'bug{i}' for i in range(3)],
points_per_exposed_bug=1)
best_submission = obj_build.make_finished_submission(
self.group,
does_not_count_for=[self.does_not_count_for_user.username])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=suite,
submission=best_submission,
bugs_exposed=suite.buggy_impl_names)
self.assertEqual(
3,
get_submission_fdbk(best_submission,
ag_models.FeedbackCategory.max).total_points)
second_best_submission = obj_build.make_finished_submission(
self.group,
does_not_count_for=[self.does_not_count_for_user.username])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=suite,
submission=second_best_submission,
bugs_exposed=suite.buggy_impl_names[:-1])
self.assertEqual(
2,
get_submission_fdbk(second_best_submission,
ag_models.FeedbackCategory.max).total_points)
other_submission = obj_build.make_finished_submission(self.group)
self.assertEqual(
0,
get_submission_fdbk(other_submission,
ag_models.FeedbackCategory.max).total_points)
counts_for_user_ultimate_submission = get_ultimate_submission(
self.group, user=self.counts_for_user)
self.assertEqual(best_submission, counts_for_user_ultimate_submission)
does_not_count_for_user_ultimate_submission = get_ultimate_submission(
self.group, user=self.does_not_count_for_user)
self.assertEqual(other_submission,
does_not_count_for_user_ultimate_submission)
def test_all_suites_deferred(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, deferred=True)
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=1)
suite2 = obj_build.make_ag_test_suite(self.project, deferred=True)
case2 = obj_build.make_ag_test_case(suite2)
cmd2 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=2)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_one_ag_suite_deferred_one_student_suite_deferred(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project, deferred=False)
case1 = obj_build.make_ag_test_case(suite1)
cmd1 = obj_build.make_full_ag_test_command(
case1,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=1)
suite2 = obj_build.make_ag_test_suite(self.project, deferred=True)
case2 = obj_build.make_ag_test_case(suite2)
cmd2 = obj_build.make_full_ag_test_command(
case2,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=2)
deferred_student_suite = ag_models.StudentTestSuite.objects.validate_and_create(
name='deferryyyy',
project=self.project,
deferred=True)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
deferred_student_suite_result = ag_models.StudentTestSuiteResult.objects.get(
submission=self.submission,
student_test_suite=deferred_student_suite)
def test_one_suite_one_case_one_cmd(self, *args):
suite = obj_build.make_ag_test_suite(self.project)
case = obj_build.make_ag_test_case(suite)
print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'"
cmd = obj_build.make_full_ag_test_command(
case,
cmd=print_to_stdout_and_stderr,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=4,
points_for_correct_stdout=1,
points_for_correct_stderr=1,
expected_return_code=ag_models.ExpectedReturnCode.zero,
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text="hello",
expected_stderr_source=ag_models.ExpectedOutputSource.text,
expected_stderr_text="whoops")
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_result = ag_models.AGTestCommandResult.objects.get(
ag_test_command=cmd,
ag_test_case_result__ag_test_suite_result__submission=self.submission)
with open(cmd_result.stderr_filename) as f:
output = f.read()
print(output)
self.assertEqual(0, cmd_result.return_code, msg=output)
self.assertEqual('hello', open(cmd_result.stdout_filename).read())
self.assertEqual('whoops', open(cmd_result.stderr_filename).read())
self.assertTrue(cmd_result.stdout_correct)
self.assertTrue(cmd_result.stderr_correct)
self.assertEqual(
6,
get_submission_fdbk(
self.submission, ag_models.FeedbackCategory.max).total_points_possible)
self.assertEqual(
6,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_max_fdbk(self):
fdbk = get_submission_fdbk(self.submission,
ag_models.FeedbackCategory.max)
self.assertEqual(self.total_points, fdbk.total_points)
self.assertEqual(self.total_points, fdbk.total_points_possible)
self.assertSequenceEqual(
[self.ag_suite_result1.pk, self.ag_suite_result2.pk],
[res.pk for res in fdbk.ag_test_suite_results])
self.assertSequenceEqual(
[self.student_suite_result1, self.student_suite_result2],
fdbk.student_test_suite_results)
def test_multiple_suites_cases_and_cmds(self, *args):
suite1 = obj_build.make_ag_test_suite(self.project)
case1 = obj_build.make_ag_test_case(suite1)
case2 = obj_build.make_ag_test_case(suite1)
suite2 = obj_build.make_ag_test_suite(self.project)
case3 = obj_build.make_ag_test_case(suite2)
case4 = obj_build.make_ag_test_case(suite2)
print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'"
for case in case1, case2, case3, case4:
for i in range(2):
obj_build.make_full_ag_test_command(
ag_test_case=case,
cmd=print_to_stdout_and_stderr,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
points_for_correct_return_code=4,
points_for_correct_stdout=1,
points_for_correct_stderr=1,
expected_return_code=ag_models.ExpectedReturnCode.zero,
expected_stdout_source=ag_models.ExpectedOutputSource.text,
expected_stdout_text="hello",
expected_stderr_source=ag_models.ExpectedOutputSource.text,
expected_stderr_text="whoops")
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
cmd_results = ag_models.AGTestCommandResult.objects.filter(
ag_test_case_result__ag_test_suite_result__submission=self.submission)
for res in cmd_results:
self.assertEqual(0, res.return_code)
self.assertEqual('hello', open(res.stdout_filename).read())
self.assertEqual('whoops', open(res.stderr_filename).read())
self.assertTrue(res.stdout_correct)
self.assertTrue(res.stderr_correct)
self.assertEqual(
48,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
def test_deferred_retry_on_error(self, impl_mock, *args):
suite = obj_build.make_ag_test_suite(self.project, deferred=True)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(
case,
set_arbitrary_points=False,
set_arbitrary_expected_vals=False,
expected_return_code=ag_models.ExpectedReturnCode.zero,
points_for_correct_return_code=3)
tasks.grade_submission(self.submission.pk)
self.submission.refresh_from_db()
self.assertEqual(ag_models.Submission.GradingStatus.finished_grading,
self.submission.status)
res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd)
self.assertEqual(0, res.return_code)
self.assertTrue(res.return_code_correct)
self.assertEqual(
3,
get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)
