def grade_ag_test_suite_impl(ag_test_suite: ag_models.AGTestSuite,
submission: ag_models.Submission,
*ag_test_cases_to_run: ag_models.AGTestCase):
@retry_should_recover
def get_or_create_suite_result():
return ag_models.AGTestSuiteResult.objects.get_or_create(
ag_test_suite=ag_test_suite, submission=submission)[0]
suite_result = get_or_create_suite_result()
sandbox = AutograderSandbox(
name='submission{}-suite{}-{}'.format(submission.pk, ag_test_suite.pk,
uuid.uuid4().hex),
environment_variables={
'usernames': ' '.join(submission.group.member_names)
},
allow_network_access=ag_test_suite.allow_network_access,
docker_image=constants.DOCKER_IMAGE_IDS_TO_URLS[
ag_test_suite.docker_image_to_use])
print(ag_test_suite.docker_image_to_use)
print(sandbox.docker_image)
with sandbox:
add_files_to_sandbox(sandbox, ag_test_suite, submission)
print('Running setup for', ag_test_suite.name)
_run_suite_setup(sandbox, ag_test_suite, suite_result)
if not ag_test_cases_to_run:
ag_test_cases_to_run = ag_test_suite.ag_test_cases.all()
for ag_test_case in ag_test_cases_to_run:
print('Grading test case', ag_test_case.name)
grade_ag_test_case_impl(sandbox, ag_test_case, suite_result)
update_denormalized_ag_test_results(submission.pk)
def setUp(self):
super().setUp()
self.maxDiff = None
self.client = APIClient()
self.project = obj_build.make_project(
ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.
most_recent, )
self.url = reverse('project-ultimate-submission-scores',
kwargs={'pk': self.project.pk})
self.admin = obj_build.make_admin_user(self.project.course)
self.ag_test_suite = obj_build.make_ag_test_suite(project=self.project)
self.ag_test_case = obj_build.make_ag_test_case(
ag_test_suite=self.ag_test_suite)
self.ag_test_cmd = obj_build.make_full_ag_test_command(
ag_test_case=self.ag_test_case)
self.student_group = obj_build.make_group(project=self.project,
num_members=2)
self.student_submission = obj_build.make_finished_submission(
self.student_group)
self.student_result = obj_build.make_incorrect_ag_test_command_result(
ag_test_command=self.ag_test_cmd,
submission=self.student_submission)
self.student_submission = update_denormalized_ag_test_results(
self.student_submission.pk)
self.student_result_fdbk = SubmissionResultFeedback(
self.student_submission, ag_models.FeedbackCategory.max,
AGTestPreLoader(self.project))
self.assertEqual(0, self.student_result_fdbk.total_points)
self.assertNotEqual(0, self.student_result_fdbk.total_points_possible)
self.staff_group = obj_build.make_group(
project=self.project, members_role=obj_build.UserRole.admin)
self.staff_submission = obj_build.make_finished_submission(
self.staff_group)
self.staff_result = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_cmd, submission=self.staff_submission)
self.staff_submission = update_denormalized_ag_test_results(
self.staff_submission.pk)
self.staff_result_fdbk = SubmissionResultFeedback(
self.staff_submission, ag_models.FeedbackCategory.max,
AGTestPreLoader(self.project))
self.assertNotEqual(0, self.staff_result_fdbk.total_points)
self.assertNotEqual(0, self.staff_result_fdbk.total_points_possible)
# Make sure we use the right queryset
other_project = obj_build.make_project(course=self.project.course)
other_group = obj_build.make_group(project=other_project)
other_submission = obj_build.make_finished_submission(other_group)
class _TestCase(UnitTestBase):
class GroupAndSubmissionData:
def __init__(self, group: ag_models.Group,
best_submission: ag_models.Submission,
most_recent_submission: ag_models.Submission):
self.group = group
self.best_submission = best_submission
self.most_recent_submission = most_recent_submission
def prepare_data(self,
project: ag_models.Project,
num_suites=1,
cases_per_suite=1,
cmds_per_suite=1,
num_groups=1,
num_other_submissions=0) -> List[GroupAndSubmissionData]:
cmds = []
for i in range(num_suites):
sys.stdout.write('\rBuilding suite {}'.format(i))
sys.stdout.flush()
suite = obj_build.make_ag_test_suite(project)
for j in range(cases_per_suite):
case = obj_build.make_ag_test_case(suite)
for k in range(cmds_per_suite):
cmd = obj_build.make_full_ag_test_command(case)
cmds.append(cmd)
group_and_submission_data = []
for i in range(num_groups):
sys.stdout.write('\rBuilding group {}'.format(i))
sys.stdout.flush()
group = obj_build.make_group(project=project)
best_sub = obj_build.make_finished_submission(group=group)
for cmd in cmds:
obj_build.make_correct_ag_test_command_result(
cmd, submission=best_sub)
best_sub = update_denormalized_ag_test_results(best_sub.pk)
for j in range(num_other_submissions):
sub = obj_build.make_finished_submission(group=group)
for cmd in cmds:
obj_build.make_incorrect_ag_test_command_result(
cmd, submission=sub)
sub = update_denormalized_ag_test_results(sub.pk)
most_recent = obj_build.make_finished_submission(group=group)
for cmd in cmds:
obj_build.make_incorrect_ag_test_command_result(
cmd, submission=most_recent)
most_recent = update_denormalized_ag_test_results(most_recent.pk)
group_and_submission_data.append(
self.GroupAndSubmissionData(group, best_sub, most_recent))
return group_and_submission_data
def test_max_fdbk_some_incorrect(self):
# Make something incorrect, re-check total points and total points
# possible.
self.ag_cmd_result1.return_code_correct = False
self.ag_cmd_result1.stdout_correct = False
self.ag_cmd_result1.stderr_correct = False
self.ag_cmd_result1.save()
self.submission = update_denormalized_ag_test_results(
self.submission.pk)
self.student_suite_result1.bugs_exposed = []
self.student_suite_result1.save()
fdbk = get_submission_fdbk(self.submission,
ag_models.FeedbackCategory.max)
self.assertEqual(
self.total_points_per_ag_suite +
self.total_points_per_student_suite, fdbk.total_points)
self.assertEqual(self.total_points_possible,
fdbk.total_points_possible)
# Make sure that adjusting max_points for a student test suite propagates
max_points = self.points_per_bug_exposed.to_integral_value()
self.student_suite2.validate_and_update(max_points=max_points)
fdbk = get_submission_fdbk(self.submission,
ag_models.FeedbackCategory.max)
self.assertEqual(self.total_points_per_ag_suite + max_points,
fdbk.total_points)
self.assertEqual(
self.total_points_per_ag_suite * 2 +
self.total_points_per_student_suite + max_points,
fdbk.total_points_possible)
def test_delete_suite_result_denormed_results_updated(self):
suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_suite, submission=self.submission)
self.assertNotEqual(suite_result.pk, self.ag_suite.pk)
dont_delete_suite = obj_build.make_ag_test_suite(self.project)
dont_delete_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=dont_delete_suite, submission=self.submission)
self.assertNotEqual(dont_delete_suite.pk, dont_delete_suite_result.pk)
submission2 = obj_build.make_submission(group=obj_build.make_group(
project=self.project))
suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_suite, submission=submission2)
self.assertNotEqual(suite_result2.pk, self.ag_suite.pk)
self.submission = update_denormalized_ag_test_results(
self.submission.pk)
self.assertIn(str(dont_delete_suite.pk),
self.submission.denormalized_ag_test_results)
submission2 = update_denormalized_ag_test_results(submission2.pk)
for submission, suite_res in ((self.submission, suite_result),
(submission2, suite_result2)):
submission.refresh_from_db()
self.assertIn(str(suite_res.ag_test_suite_id),
submission.denormalized_ag_test_results)
self.ag_suite.delete()
for submission, suite_res in ((self.submission, suite_result),
(submission2, suite_result2)):
submission.refresh_from_db()
self.assertNotIn(str(suite_res.ag_test_suite_id),
submission.denormalized_ag_test_results)
self.submission.refresh_from_db()
self.assertIn(str(dont_delete_suite.pk),
self.submission.denormalized_ag_test_results)
def _add_results_to_submission(
self, submission: ag_models.Submission, *,
results_correct: bool) -> ag_models.Submission:
if results_correct:
obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd, submission=submission)
else:
obj_build.make_incorrect_ag_test_command_result(
self.ag_test_cmd, submission=submission)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
submission=submission, student_test_suite=self.student_test_suite)
return update_denormalized_ag_test_results(submission.pk)
def set_denormalized_results(apps, schema_editor):
Submission = apps.get_model('core', 'Submission')
for submission in Submission.objects.all():
update_denormalized_ag_test_results(submission.pk)
def setUp(self):
super().setUp()
self.client = APIClient()
max_group_size = 3
self.project = obj_build.make_project(
visible_to_students=True,
ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.best,
max_group_size=max_group_size)
self.suite1 = obj_build.make_ag_test_suite(self.project)
self.suite1_case = obj_build.make_ag_test_case(self.suite1)
self.suite1_cmd = obj_build.make_full_ag_test_command(self.suite1_case)
self.suite2 = obj_build.make_ag_test_suite(self.project)
self.suite2_case = obj_build.make_ag_test_case(self.suite2)
self.suite2_cmd = obj_build.make_full_ag_test_command(self.suite2_case)
self.student_suite1_bugs = ['suite1_bug{}'.format(i) for i in range(3)]
self.student_suite1 = obj_build.make_student_test_suite(
project=self.project, buggy_impl_names=self.student_suite1_bugs)
self.student_suite2_bugs = ['suite2_bug{}'.format(i) for i in range(5)]
self.student_suite2 = obj_build.make_student_test_suite(
project=self.project, buggy_impl_names=self.student_suite2_bugs)
self.student_group1 = obj_build.make_group(project=self.project)
self.group1_submission1_best = obj_build.make_finished_submission(
group=self.student_group1)
obj_build.make_correct_ag_test_command_result(
self.suite1_cmd, submission=self.group1_submission1_best)
obj_build.make_correct_ag_test_command_result(
self.suite2_cmd, submission=self.group1_submission1_best)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.group1_submission1_best,
bugs_exposed=self.student_suite1_bugs)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.group1_submission1_best,
bugs_exposed=self.student_suite2_bugs)
self.group1_submission2 = obj_build.make_finished_submission(
group=self.student_group1)
obj_build.make_correct_ag_test_command_result(
self.suite1_cmd, submission=self.group1_submission2)
obj_build.make_incorrect_ag_test_command_result(
self.suite2_cmd, submission=self.group1_submission2)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.group1_submission2,
bugs_exposed=[])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.group1_submission2,
bugs_exposed=[])
self.group1_submission1_best = update_denormalized_ag_test_results(
self.group1_submission1_best.pk)
self.group1_submission2 = update_denormalized_ag_test_results(
self.group1_submission2.pk)
self.student_group2 = obj_build.make_group(num_members=max_group_size,
project=self.project)
self.group2_only_submission = obj_build.make_finished_submission(
group=self.student_group2)
obj_build.make_incorrect_ag_test_command_result(
self.suite1_cmd, submission=self.group2_only_submission)
obj_build.make_incorrect_ag_test_command_result(
self.suite2_cmd, submission=self.group2_only_submission)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.group2_only_submission,
bugs_exposed=self.student_suite1_bugs[:-1])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.group2_only_submission,
bugs_exposed=self.student_suite2_bugs[:-1])
self.group2_only_submission = update_denormalized_ag_test_results(
self.group2_only_submission.pk)
self.staff_group = obj_build.make_group(
project=self.project, members_role=obj_build.UserRole.staff)
self.staff_submission1 = obj_build.make_finished_submission(
group=self.staff_group)
obj_build.make_correct_ag_test_command_result(
self.suite1_cmd, submission=self.staff_submission1)
obj_build.make_correct_ag_test_command_result(
self.suite2_cmd, submission=self.staff_submission1)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.staff_submission1,
bugs_exposed=self.student_suite1_bugs[:-1])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.staff_submission1,
bugs_exposed=self.student_suite2_bugs)
self.staff_submission1 = update_denormalized_ag_test_results(
self.staff_submission1.pk)
self.no_submissions_group = obj_build.make_group(project=self.project)
[self.admin] = obj_build.make_admin_users(self.project.course, 1)
self.maxDiff = None
def setUp(self):
super().setUp()
self.maxDiff = None
self.project = obj_build.make_project()
self.course = self.project.course
self.ag_test_suite1 = obj_build.make_ag_test_suite(self.project)
self.ag_test_case1 = obj_build.make_ag_test_case(self.ag_test_suite1)
self.ag_test_cmd1 = obj_build.make_full_ag_test_command(
self.ag_test_case1, set_arbitrary_points=True)
self.ag_test_suite2 = obj_build.make_ag_test_suite(self.project)
self.ag_test_case2 = obj_build.make_ag_test_case(self.ag_test_suite2)
self.ag_test_cmd2 = obj_build.make_full_ag_test_command(
self.ag_test_case2, set_arbitrary_points=True)
self.points_per_bug_exposed = Decimal('3.5')
self.num_buggy_impls = 4
self.student_suite1 = ag_models.StudentTestSuite.objects.validate_and_create(
name='suite1',
project=self.project,
buggy_impl_names=[
'bug{}'.format(i) for i in range(self.num_buggy_impls)
],
points_per_exposed_bug=self.points_per_bug_exposed
) # type: ag_models.StudentTestSuite
self.student_suite2 = ag_models.StudentTestSuite.objects.validate_and_create(
name='suite2',
project=self.project,
buggy_impl_names=[
'bug{}'.format(i) for i in range(self.num_buggy_impls)
],
points_per_exposed_bug=self.points_per_bug_exposed
) # type: ag_models.StudentTestSuite
self.group = obj_build.make_group(num_members=1, project=self.project)
self.submission = obj_build.make_submission(group=self.group)
self.ag_suite_result1 = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_test_suite1,
submission=self.submission) # type: ag_models.AGTestSuiteResult
self.ag_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_suite_result=self.ag_suite_result1,
ag_test_case=self.ag_test_case1
) # type: ag_models.AGTestCaseResult
self.ag_cmd_result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_case_result1)
self.ag_suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_test_suite2,
submission=self.submission) # type: ag_models.AGTestSuiteResult
self.ag_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_suite_result=self.ag_suite_result2,
ag_test_case=self.ag_test_case2
) # type: ag_models.AGTestCaseResult
self.ag_cmd_result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_case_result2)
self.num_student_tests = 6
self.student_tests = [
'test{}'.format(i) for i in range(self.num_student_tests)
]
self.student_suite_result1 = ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.submission,
student_tests=self.student_tests,
bugs_exposed=self.student_suite1.buggy_impl_names
) # type: ag_models.StudentTestSuiteResult
self.student_suite_result2 = ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.submission,
student_tests=self.student_tests,
bugs_exposed=self.student_suite2.buggy_impl_names
) # type: ag_models.StudentTestSuiteResult
self.total_points_per_ag_suite = get_suite_fdbk(
self.ag_suite_result1, ag_models.FeedbackCategory.max).total_points
self.total_points_per_student_suite = self.num_buggy_impls * self.points_per_bug_exposed
self.total_points = (self.total_points_per_ag_suite * 2 +
self.total_points_per_student_suite * 2)
self.total_points_possible = self.total_points
self.assertEqual(
self.total_points_per_ag_suite,
get_suite_fdbk(self.ag_suite_result2,
ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
self.total_points_per_student_suite,
self.student_suite_result1.get_fdbk(
ag_models.FeedbackCategory.max).total_points)
print(self.total_points)
self.assertNotEqual(0, self.total_points_per_ag_suite)
self.assertNotEqual(0, self.total_points_per_student_suite)
self.assertNotEqual(0, self.total_points)
self.submission = update_denormalized_ag_test_results(
self.submission.pk)