def test_non_default_docker_image(self, *args): suite = obj_build.make_ag_test_suite( self.project, docker_image_to_use=constants.SupportedImages.eecs490) case = obj_build.make_ag_test_case(suite) cmd = obj_build.make_full_ag_test_command( case, cmd='racket --version', set_arbitrary_points=False, set_arbitrary_expected_vals=False, points_for_correct_return_code=3, expected_return_code=ag_models.ExpectedReturnCode.zero) tasks.grade_submission(self.submission.pk) self.submission.refresh_from_db() cmd_result = ag_models.AGTestCommandResult.objects.get( ag_test_command=cmd, ag_test_case_result__ag_test_suite_result__submission=self.submission) self.assertEqual(0, cmd_result.return_code) self.assertEqual( 3, get_submission_fdbk( self.submission, ag_models.FeedbackCategory.max).total_points_possible) self.assertEqual( 3, get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points) self.assertEqual(ag_models.Submission.GradingStatus.finished_grading, self.submission.status)
def test_two_best_dont_count_for_user(self): self.project.validate_and_update( ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.best) suite = ag_models.StudentTestSuite.objects.validate_and_create( name='suite', project=self.project, buggy_impl_names=[f'bug{i}' for i in range(3)], points_per_exposed_bug=1) best_submission = obj_build.make_finished_submission( self.group, does_not_count_for=[self.does_not_count_for_user.username]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=suite, submission=best_submission, bugs_exposed=suite.buggy_impl_names) self.assertEqual( 3, get_submission_fdbk(best_submission, ag_models.FeedbackCategory.max).total_points) second_best_submission = obj_build.make_finished_submission( self.group, does_not_count_for=[self.does_not_count_for_user.username]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=suite, submission=second_best_submission, bugs_exposed=suite.buggy_impl_names[:-1]) self.assertEqual( 2, get_submission_fdbk(second_best_submission, ag_models.FeedbackCategory.max).total_points) other_submission = obj_build.make_finished_submission(self.group) self.assertEqual( 0, get_submission_fdbk(other_submission, ag_models.FeedbackCategory.max).total_points) counts_for_user_ultimate_submission = get_ultimate_submission( self.group, user=self.counts_for_user) self.assertEqual(best_submission, counts_for_user_ultimate_submission) does_not_count_for_user_ultimate_submission = get_ultimate_submission( self.group, user=self.does_not_count_for_user) self.assertEqual(other_submission, does_not_count_for_user_ultimate_submission)
def test_all_suites_deferred(self, *args): suite1 = obj_build.make_ag_test_suite(self.project, deferred=True) case1 = obj_build.make_ag_test_case(suite1) cmd1 = obj_build.make_full_ag_test_command( case1, set_arbitrary_points=False, set_arbitrary_expected_vals=False, expected_return_code=ag_models.ExpectedReturnCode.zero, points_for_correct_return_code=1) suite2 = obj_build.make_ag_test_suite(self.project, deferred=True) case2 = obj_build.make_ag_test_case(suite2) cmd2 = obj_build.make_full_ag_test_command( case2, set_arbitrary_points=False, set_arbitrary_expected_vals=False, expected_return_code=ag_models.ExpectedReturnCode.zero, points_for_correct_return_code=2) tasks.grade_submission(self.submission.pk) self.submission.refresh_from_db() self.assertEqual( 3, get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points) self.assertEqual(ag_models.Submission.GradingStatus.finished_grading, self.submission.status)
def test_one_ag_suite_deferred_one_student_suite_deferred(self, *args): suite1 = obj_build.make_ag_test_suite(self.project, deferred=False) case1 = obj_build.make_ag_test_case(suite1) cmd1 = obj_build.make_full_ag_test_command( case1, set_arbitrary_points=False, set_arbitrary_expected_vals=False, expected_return_code=ag_models.ExpectedReturnCode.zero, points_for_correct_return_code=1) suite2 = obj_build.make_ag_test_suite(self.project, deferred=True) case2 = obj_build.make_ag_test_case(suite2) cmd2 = obj_build.make_full_ag_test_command( case2, set_arbitrary_points=False, set_arbitrary_expected_vals=False, expected_return_code=ag_models.ExpectedReturnCode.zero, points_for_correct_return_code=2) deferred_student_suite = ag_models.StudentTestSuite.objects.validate_and_create( name='deferryyyy', project=self.project, deferred=True) tasks.grade_submission(self.submission.pk) self.submission.refresh_from_db() self.assertEqual( 3, get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points) self.assertEqual(ag_models.Submission.GradingStatus.finished_grading, self.submission.status) deferred_student_suite_result = ag_models.StudentTestSuiteResult.objects.get( submission=self.submission, student_test_suite=deferred_student_suite)
def test_one_suite_one_case_one_cmd(self, *args): suite = obj_build.make_ag_test_suite(self.project) case = obj_build.make_ag_test_case(suite) print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'" cmd = obj_build.make_full_ag_test_command( case, cmd=print_to_stdout_and_stderr, set_arbitrary_points=False, set_arbitrary_expected_vals=False, points_for_correct_return_code=4, points_for_correct_stdout=1, points_for_correct_stderr=1, expected_return_code=ag_models.ExpectedReturnCode.zero, expected_stdout_source=ag_models.ExpectedOutputSource.text, expected_stdout_text="hello", expected_stderr_source=ag_models.ExpectedOutputSource.text, expected_stderr_text="whoops") tasks.grade_submission(self.submission.pk) self.submission.refresh_from_db() cmd_result = ag_models.AGTestCommandResult.objects.get( ag_test_command=cmd, ag_test_case_result__ag_test_suite_result__submission=self.submission) with open(cmd_result.stderr_filename) as f: output = f.read() print(output) self.assertEqual(0, cmd_result.return_code, msg=output) self.assertEqual('hello', open(cmd_result.stdout_filename).read()) self.assertEqual('whoops', open(cmd_result.stderr_filename).read()) self.assertTrue(cmd_result.stdout_correct) self.assertTrue(cmd_result.stderr_correct) self.assertEqual( 6, get_submission_fdbk( self.submission, ag_models.FeedbackCategory.max).total_points_possible) self.assertEqual( 6, get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points) self.assertEqual(ag_models.Submission.GradingStatus.finished_grading, self.submission.status)
def test_max_fdbk(self): fdbk = get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max) self.assertEqual(self.total_points, fdbk.total_points) self.assertEqual(self.total_points, fdbk.total_points_possible) self.assertSequenceEqual( [self.ag_suite_result1.pk, self.ag_suite_result2.pk], [res.pk for res in fdbk.ag_test_suite_results]) self.assertSequenceEqual( [self.student_suite_result1, self.student_suite_result2], fdbk.student_test_suite_results)
def test_multiple_suites_cases_and_cmds(self, *args): suite1 = obj_build.make_ag_test_suite(self.project) case1 = obj_build.make_ag_test_case(suite1) case2 = obj_build.make_ag_test_case(suite1) suite2 = obj_build.make_ag_test_suite(self.project) case3 = obj_build.make_ag_test_case(suite2) case4 = obj_build.make_ag_test_case(suite2) print_to_stdout_and_stderr = "bash -c 'printf hello; printf whoops >&2'" for case in case1, case2, case3, case4: for i in range(2): obj_build.make_full_ag_test_command( ag_test_case=case, cmd=print_to_stdout_and_stderr, set_arbitrary_points=False, set_arbitrary_expected_vals=False, points_for_correct_return_code=4, points_for_correct_stdout=1, points_for_correct_stderr=1, expected_return_code=ag_models.ExpectedReturnCode.zero, expected_stdout_source=ag_models.ExpectedOutputSource.text, expected_stdout_text="hello", expected_stderr_source=ag_models.ExpectedOutputSource.text, expected_stderr_text="whoops") tasks.grade_submission(self.submission.pk) self.submission.refresh_from_db() cmd_results = ag_models.AGTestCommandResult.objects.filter( ag_test_case_result__ag_test_suite_result__submission=self.submission) for res in cmd_results: self.assertEqual(0, res.return_code) self.assertEqual('hello', open(res.stdout_filename).read()) self.assertEqual('whoops', open(res.stderr_filename).read()) self.assertTrue(res.stdout_correct) self.assertTrue(res.stderr_correct) self.assertEqual( 48, get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points) self.assertEqual(ag_models.Submission.GradingStatus.finished_grading, self.submission.status)
def test_deferred_retry_on_error(self, impl_mock, *args): suite = obj_build.make_ag_test_suite(self.project, deferred=True) case = obj_build.make_ag_test_case(suite) cmd = obj_build.make_full_ag_test_command( case, set_arbitrary_points=False, set_arbitrary_expected_vals=False, expected_return_code=ag_models.ExpectedReturnCode.zero, points_for_correct_return_code=3) tasks.grade_submission(self.submission.pk) self.submission.refresh_from_db() self.assertEqual(ag_models.Submission.GradingStatus.finished_grading, self.submission.status) res = ag_models.AGTestCommandResult.objects.get(ag_test_command=cmd) self.assertEqual(0, res.return_code) self.assertTrue(res.return_code_correct) self.assertEqual( 3, get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).total_points)