def grade_ag_test_suite_impl(ag_test_suite: ag_models.AGTestSuite, submission: ag_models.Submission, *ag_test_cases_to_run: ag_models.AGTestCase): @retry_should_recover def get_or_create_suite_result(): return ag_models.AGTestSuiteResult.objects.get_or_create( ag_test_suite=ag_test_suite, submission=submission)[0] suite_result = get_or_create_suite_result() sandbox = AutograderSandbox( name='submission{}-suite{}-{}'.format(submission.pk, ag_test_suite.pk, uuid.uuid4().hex), environment_variables={ 'usernames': ' '.join(submission.group.member_names) }, allow_network_access=ag_test_suite.allow_network_access, docker_image=constants.DOCKER_IMAGE_IDS_TO_URLS[ ag_test_suite.docker_image_to_use]) print(ag_test_suite.docker_image_to_use) print(sandbox.docker_image) with sandbox: add_files_to_sandbox(sandbox, ag_test_suite, submission) print('Running setup for', ag_test_suite.name) _run_suite_setup(sandbox, ag_test_suite, suite_result) if not ag_test_cases_to_run: ag_test_cases_to_run = ag_test_suite.ag_test_cases.all() for ag_test_case in ag_test_cases_to_run: print('Grading test case', ag_test_case.name) grade_ag_test_case_impl(sandbox, ag_test_case, suite_result) update_denormalized_ag_test_results(submission.pk)
def setUp(self): super().setUp() self.maxDiff = None self.client = APIClient() self.project = obj_build.make_project( ultimate_submission_policy=ag_models.UltimateSubmissionPolicy. most_recent, ) self.url = reverse('project-ultimate-submission-scores', kwargs={'pk': self.project.pk}) self.admin = obj_build.make_admin_user(self.project.course) self.ag_test_suite = obj_build.make_ag_test_suite(project=self.project) self.ag_test_case = obj_build.make_ag_test_case( ag_test_suite=self.ag_test_suite) self.ag_test_cmd = obj_build.make_full_ag_test_command( ag_test_case=self.ag_test_case) self.student_group = obj_build.make_group(project=self.project, num_members=2) self.student_submission = obj_build.make_finished_submission( self.student_group) self.student_result = obj_build.make_incorrect_ag_test_command_result( ag_test_command=self.ag_test_cmd, submission=self.student_submission) self.student_submission = update_denormalized_ag_test_results( self.student_submission.pk) self.student_result_fdbk = SubmissionResultFeedback( self.student_submission, ag_models.FeedbackCategory.max, AGTestPreLoader(self.project)) self.assertEqual(0, self.student_result_fdbk.total_points) self.assertNotEqual(0, self.student_result_fdbk.total_points_possible) self.staff_group = obj_build.make_group( project=self.project, members_role=obj_build.UserRole.admin) self.staff_submission = obj_build.make_finished_submission( self.staff_group) self.staff_result = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_cmd, submission=self.staff_submission) self.staff_submission = update_denormalized_ag_test_results( self.staff_submission.pk) self.staff_result_fdbk = SubmissionResultFeedback( self.staff_submission, ag_models.FeedbackCategory.max, AGTestPreLoader(self.project)) self.assertNotEqual(0, self.staff_result_fdbk.total_points) self.assertNotEqual(0, self.staff_result_fdbk.total_points_possible) # Make sure we use the right queryset other_project = obj_build.make_project(course=self.project.course) other_group = obj_build.make_group(project=other_project) other_submission = obj_build.make_finished_submission(other_group)
class _TestCase(UnitTestBase): class GroupAndSubmissionData: def __init__(self, group: ag_models.Group, best_submission: ag_models.Submission, most_recent_submission: ag_models.Submission): self.group = group self.best_submission = best_submission self.most_recent_submission = most_recent_submission def prepare_data(self, project: ag_models.Project, num_suites=1, cases_per_suite=1, cmds_per_suite=1, num_groups=1, num_other_submissions=0) -> List[GroupAndSubmissionData]: cmds = [] for i in range(num_suites): sys.stdout.write('\rBuilding suite {}'.format(i)) sys.stdout.flush() suite = obj_build.make_ag_test_suite(project) for j in range(cases_per_suite): case = obj_build.make_ag_test_case(suite) for k in range(cmds_per_suite): cmd = obj_build.make_full_ag_test_command(case) cmds.append(cmd) group_and_submission_data = [] for i in range(num_groups): sys.stdout.write('\rBuilding group {}'.format(i)) sys.stdout.flush() group = obj_build.make_group(project=project) best_sub = obj_build.make_finished_submission(group=group) for cmd in cmds: obj_build.make_correct_ag_test_command_result( cmd, submission=best_sub) best_sub = update_denormalized_ag_test_results(best_sub.pk) for j in range(num_other_submissions): sub = obj_build.make_finished_submission(group=group) for cmd in cmds: obj_build.make_incorrect_ag_test_command_result( cmd, submission=sub) sub = update_denormalized_ag_test_results(sub.pk) most_recent = obj_build.make_finished_submission(group=group) for cmd in cmds: obj_build.make_incorrect_ag_test_command_result( cmd, submission=most_recent) most_recent = update_denormalized_ag_test_results(most_recent.pk) group_and_submission_data.append( self.GroupAndSubmissionData(group, best_sub, most_recent)) return group_and_submission_data
def test_max_fdbk_some_incorrect(self): # Make something incorrect, re-check total points and total points # possible. self.ag_cmd_result1.return_code_correct = False self.ag_cmd_result1.stdout_correct = False self.ag_cmd_result1.stderr_correct = False self.ag_cmd_result1.save() self.submission = update_denormalized_ag_test_results( self.submission.pk) self.student_suite_result1.bugs_exposed = [] self.student_suite_result1.save() fdbk = get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max) self.assertEqual( self.total_points_per_ag_suite + self.total_points_per_student_suite, fdbk.total_points) self.assertEqual(self.total_points_possible, fdbk.total_points_possible) # Make sure that adjusting max_points for a student test suite propagates max_points = self.points_per_bug_exposed.to_integral_value() self.student_suite2.validate_and_update(max_points=max_points) fdbk = get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max) self.assertEqual(self.total_points_per_ag_suite + max_points, fdbk.total_points) self.assertEqual( self.total_points_per_ag_suite * 2 + self.total_points_per_student_suite + max_points, fdbk.total_points_possible)
def test_delete_suite_result_denormed_results_updated(self): suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_suite, submission=self.submission) self.assertNotEqual(suite_result.pk, self.ag_suite.pk) dont_delete_suite = obj_build.make_ag_test_suite(self.project) dont_delete_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=dont_delete_suite, submission=self.submission) self.assertNotEqual(dont_delete_suite.pk, dont_delete_suite_result.pk) submission2 = obj_build.make_submission(group=obj_build.make_group( project=self.project)) suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_suite, submission=submission2) self.assertNotEqual(suite_result2.pk, self.ag_suite.pk) self.submission = update_denormalized_ag_test_results( self.submission.pk) self.assertIn(str(dont_delete_suite.pk), self.submission.denormalized_ag_test_results) submission2 = update_denormalized_ag_test_results(submission2.pk) for submission, suite_res in ((self.submission, suite_result), (submission2, suite_result2)): submission.refresh_from_db() self.assertIn(str(suite_res.ag_test_suite_id), submission.denormalized_ag_test_results) self.ag_suite.delete() for submission, suite_res in ((self.submission, suite_result), (submission2, suite_result2)): submission.refresh_from_db() self.assertNotIn(str(suite_res.ag_test_suite_id), submission.denormalized_ag_test_results) self.submission.refresh_from_db() self.assertIn(str(dont_delete_suite.pk), self.submission.denormalized_ag_test_results)
def _add_results_to_submission( self, submission: ag_models.Submission, *, results_correct: bool) -> ag_models.Submission: if results_correct: obj_build.make_correct_ag_test_command_result( self.ag_test_cmd, submission=submission) else: obj_build.make_incorrect_ag_test_command_result( self.ag_test_cmd, submission=submission) ag_models.StudentTestSuiteResult.objects.validate_and_create( submission=submission, student_test_suite=self.student_test_suite) return update_denormalized_ag_test_results(submission.pk)
def set_denormalized_results(apps, schema_editor): Submission = apps.get_model('core', 'Submission') for submission in Submission.objects.all(): update_denormalized_ag_test_results(submission.pk)
def setUp(self): super().setUp() self.client = APIClient() max_group_size = 3 self.project = obj_build.make_project( visible_to_students=True, ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.best, max_group_size=max_group_size) self.suite1 = obj_build.make_ag_test_suite(self.project) self.suite1_case = obj_build.make_ag_test_case(self.suite1) self.suite1_cmd = obj_build.make_full_ag_test_command(self.suite1_case) self.suite2 = obj_build.make_ag_test_suite(self.project) self.suite2_case = obj_build.make_ag_test_case(self.suite2) self.suite2_cmd = obj_build.make_full_ag_test_command(self.suite2_case) self.student_suite1_bugs = ['suite1_bug{}'.format(i) for i in range(3)] self.student_suite1 = obj_build.make_student_test_suite( project=self.project, buggy_impl_names=self.student_suite1_bugs) self.student_suite2_bugs = ['suite2_bug{}'.format(i) for i in range(5)] self.student_suite2 = obj_build.make_student_test_suite( project=self.project, buggy_impl_names=self.student_suite2_bugs) self.student_group1 = obj_build.make_group(project=self.project) self.group1_submission1_best = obj_build.make_finished_submission( group=self.student_group1) obj_build.make_correct_ag_test_command_result( self.suite1_cmd, submission=self.group1_submission1_best) obj_build.make_correct_ag_test_command_result( self.suite2_cmd, submission=self.group1_submission1_best) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.group1_submission1_best, bugs_exposed=self.student_suite1_bugs) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.group1_submission1_best, bugs_exposed=self.student_suite2_bugs) self.group1_submission2 = obj_build.make_finished_submission( group=self.student_group1) obj_build.make_correct_ag_test_command_result( self.suite1_cmd, submission=self.group1_submission2) obj_build.make_incorrect_ag_test_command_result( self.suite2_cmd, submission=self.group1_submission2) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.group1_submission2, bugs_exposed=[]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.group1_submission2, bugs_exposed=[]) self.group1_submission1_best = update_denormalized_ag_test_results( self.group1_submission1_best.pk) self.group1_submission2 = update_denormalized_ag_test_results( self.group1_submission2.pk) self.student_group2 = obj_build.make_group(num_members=max_group_size, project=self.project) self.group2_only_submission = obj_build.make_finished_submission( group=self.student_group2) obj_build.make_incorrect_ag_test_command_result( self.suite1_cmd, submission=self.group2_only_submission) obj_build.make_incorrect_ag_test_command_result( self.suite2_cmd, submission=self.group2_only_submission) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.group2_only_submission, bugs_exposed=self.student_suite1_bugs[:-1]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.group2_only_submission, bugs_exposed=self.student_suite2_bugs[:-1]) self.group2_only_submission = update_denormalized_ag_test_results( self.group2_only_submission.pk) self.staff_group = obj_build.make_group( project=self.project, members_role=obj_build.UserRole.staff) self.staff_submission1 = obj_build.make_finished_submission( group=self.staff_group) obj_build.make_correct_ag_test_command_result( self.suite1_cmd, submission=self.staff_submission1) obj_build.make_correct_ag_test_command_result( self.suite2_cmd, submission=self.staff_submission1) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.staff_submission1, bugs_exposed=self.student_suite1_bugs[:-1]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.staff_submission1, bugs_exposed=self.student_suite2_bugs) self.staff_submission1 = update_denormalized_ag_test_results( self.staff_submission1.pk) self.no_submissions_group = obj_build.make_group(project=self.project) [self.admin] = obj_build.make_admin_users(self.project.course, 1) self.maxDiff = None
def setUp(self): super().setUp() self.maxDiff = None self.project = obj_build.make_project() self.course = self.project.course self.ag_test_suite1 = obj_build.make_ag_test_suite(self.project) self.ag_test_case1 = obj_build.make_ag_test_case(self.ag_test_suite1) self.ag_test_cmd1 = obj_build.make_full_ag_test_command( self.ag_test_case1, set_arbitrary_points=True) self.ag_test_suite2 = obj_build.make_ag_test_suite(self.project) self.ag_test_case2 = obj_build.make_ag_test_case(self.ag_test_suite2) self.ag_test_cmd2 = obj_build.make_full_ag_test_command( self.ag_test_case2, set_arbitrary_points=True) self.points_per_bug_exposed = Decimal('3.5') self.num_buggy_impls = 4 self.student_suite1 = ag_models.StudentTestSuite.objects.validate_and_create( name='suite1', project=self.project, buggy_impl_names=[ 'bug{}'.format(i) for i in range(self.num_buggy_impls) ], points_per_exposed_bug=self.points_per_bug_exposed ) # type: ag_models.StudentTestSuite self.student_suite2 = ag_models.StudentTestSuite.objects.validate_and_create( name='suite2', project=self.project, buggy_impl_names=[ 'bug{}'.format(i) for i in range(self.num_buggy_impls) ], points_per_exposed_bug=self.points_per_bug_exposed ) # type: ag_models.StudentTestSuite self.group = obj_build.make_group(num_members=1, project=self.project) self.submission = obj_build.make_submission(group=self.group) self.ag_suite_result1 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_test_suite1, submission=self.submission) # type: ag_models.AGTestSuiteResult self.ag_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_suite_result=self.ag_suite_result1, ag_test_case=self.ag_test_case1 ) # type: ag_models.AGTestCaseResult self.ag_cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_case_result1) self.ag_suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_test_suite2, submission=self.submission) # type: ag_models.AGTestSuiteResult self.ag_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_suite_result=self.ag_suite_result2, ag_test_case=self.ag_test_case2 ) # type: ag_models.AGTestCaseResult self.ag_cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_case_result2) self.num_student_tests = 6 self.student_tests = [ 'test{}'.format(i) for i in range(self.num_student_tests) ] self.student_suite_result1 = ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.submission, student_tests=self.student_tests, bugs_exposed=self.student_suite1.buggy_impl_names ) # type: ag_models.StudentTestSuiteResult self.student_suite_result2 = ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.submission, student_tests=self.student_tests, bugs_exposed=self.student_suite2.buggy_impl_names ) # type: ag_models.StudentTestSuiteResult self.total_points_per_ag_suite = get_suite_fdbk( self.ag_suite_result1, ag_models.FeedbackCategory.max).total_points self.total_points_per_student_suite = self.num_buggy_impls * self.points_per_bug_exposed self.total_points = (self.total_points_per_ag_suite * 2 + self.total_points_per_student_suite * 2) self.total_points_possible = self.total_points self.assertEqual( self.total_points_per_ag_suite, get_suite_fdbk(self.ag_suite_result2, ag_models.FeedbackCategory.max).total_points) self.assertEqual( self.total_points_per_student_suite, self.student_suite_result1.get_fdbk( ag_models.FeedbackCategory.max).total_points) print(self.total_points) self.assertNotEqual(0, self.total_points_per_ag_suite) self.assertNotEqual(0, self.total_points_per_student_suite) self.assertNotEqual(0, self.total_points) self.submission = update_denormalized_ag_test_results( self.submission.pk)