def test_no_failed_tests_all_get_normal_fdbk(self):
self.case2_cmd_res.delete()
self.case3_cmd_res.delete()
ag_test_case2_correct_result = self.ag_test_case2_incorrect_result
ag_test_case3_correct_result = self.ag_test_case3_incorrect_result
self.case2_cmd_res = obj_build.make_correct_ag_test_command_result(
self.ag_test_case2_cmd,
ag_test_case_result=ag_test_case2_correct_result)
self.case3_cmd_res = obj_build.make_correct_ag_test_command_result(
self.ag_test_case3_cmd,
ag_test_case_result=ag_test_case3_correct_result)
fdbk = get_suite_fdbk(self.ag_suite1_result,
ag_models.FeedbackCategory.normal)
expected_case_fdbks = [
get_case_fdbk(self.ag_test_case1_correct_result,
ag_models.FeedbackCategory.normal).to_dict(),
get_case_fdbk(ag_test_case2_correct_result,
ag_models.FeedbackCategory.normal).to_dict(),
get_case_fdbk(ag_test_case3_correct_result,
ag_models.FeedbackCategory.normal).to_dict()
]
self.assertEqual(
expected_case_fdbks,
[case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_some_commands_not_visible(self):
self.ag_test_cmd1.validate_and_update(ultimate_submission_fdbk_config={'visible': False})
cmd1_pts = 5
cmd2_pts = 3
self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=cmd1_pts)
self.ag_test_cmd2.validate_and_update(points_for_correct_return_code=cmd2_pts)
cmd_result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_test_case_result)
cmd_result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result)
self.assertEqual(
cmd1_pts + cmd2_pts,
get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
cmd1_pts + cmd2_pts,
get_case_fdbk(
self.ag_test_case_result,
ag_models.FeedbackCategory.max).total_points_possible)
fdbk = get_case_fdbk(
self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission)
self.assertSequenceEqual([cmd_result2.pk],
[res.pk for res in fdbk.ag_test_command_results])
self.assertEqual(
cmd2_pts,
get_case_fdbk(
self.ag_test_case_result,
ag_models.FeedbackCategory.ultimate_submission).total_points)
self.assertEqual(
cmd2_pts,
get_case_fdbk(
self.ag_test_case_result,
ag_models.FeedbackCategory.ultimate_submission).total_points_possible)
def test_show_individual_commands(self):
total_cmd_points = 6
self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=total_cmd_points)
self.ag_test_cmd1.validate_and_update(
normal_fdbk_config={
'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect,
'show_points': True
}
)
self.assertTrue(self.ag_test_case.normal_fdbk_config.show_individual_commands)
result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_test_case_result)
result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result)
fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max)
self.assertEqual([result1.pk, result2.pk],
[res.pk for res in fdbk.ag_test_command_results])
self.assertEqual(total_cmd_points, fdbk.total_points)
self.assertEqual(total_cmd_points, fdbk.total_points_possible)
self.ag_test_case.validate_and_update(
normal_fdbk_config={'show_individual_commands': False})
fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal)
self.assertEqual([], fdbk.ag_test_command_results)
self.assertEqual(total_cmd_points, fdbk.total_points)
self.assertEqual(total_cmd_points, fdbk.total_points_possible)
def test_cmd_result_query_order(self):
result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_test_case_result)
result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result)
self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd2.pk, self.ag_test_cmd1.pk])
self.ag_test_case_result = ag_models.AGTestCaseResult.objects.get(
pk=self.ag_test_case_result.pk)
fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max)
self.assertEqual([result2.pk, result1.pk],
[res.pk for res in fdbk.ag_test_command_results])
def _add_results_to_submission(
self, submission: ag_models.Submission, *,
results_correct: bool) -> ag_models.Submission:
if results_correct:
obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd, submission=submission)
else:
obj_build.make_incorrect_ag_test_command_result(
self.ag_test_cmd, submission=submission)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
submission=submission, student_test_suite=self.student_test_suite)
return update_denormalized_ag_test_results(submission.pk)
def test_total_points_some_positive_some_negative_positive_total(self):
self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=5)
self.ag_test_cmd2.validate_and_update(deduction_for_wrong_stdout=-2)
obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result)
obj_build.make_incorrect_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result)
self.assertEqual(
3,
get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
5,
get_case_fdbk(self.ag_test_case_result,
ag_models.FeedbackCategory.max).total_points_possible)
def test_total_points_all_positive(self):
# make sure sum is correct and takes into account deductions
# make sure sum doesn't go below zero
self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=5)
self.ag_test_cmd2.validate_and_update(points_for_correct_stdout=3)
obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result)
obj_build.make_correct_ag_test_command_result(self.ag_test_cmd2, self.ag_test_case_result)
self.assertEqual(
8,
get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
8,
get_case_fdbk(self.ag_test_case_result,
ag_models.FeedbackCategory.max).total_points_possible)
def test_ag_test_cmd_result_ordering(self):
cmd_result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, ag_test_case_result=self.ag_test_case_result)
cmd_result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, ag_test_case_result=self.ag_test_case_result)
for i in range(2):
self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd2.pk, self.ag_test_cmd1.pk])
fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max)
self.assertSequenceEqual([cmd_result2.pk, cmd_result1.pk],
[res.pk for res in fdbk.ag_test_command_results])
self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd1.pk, self.ag_test_cmd2.pk])
fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max)
self.assertSequenceEqual([cmd_result1.pk, cmd_result2.pk],
[res.pk for res in fdbk.ag_test_command_results])
def test_fdbk_to_dict(self):
result = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result)
expected_keys = [
'pk',
'ag_test_command_name',
'ag_test_command_pk',
'fdbk_settings',
'timed_out',
'return_code_correct',
'expected_return_code',
'actual_return_code',
'return_code_points',
'return_code_points_possible',
'stdout_correct',
'stdout_points',
'stdout_points_possible',
'stderr_correct',
'stderr_points',
'stderr_points_possible',
'total_points',
'total_points_possible',
]
self.assertCountEqual(
expected_keys,
get_cmd_fdbk(result,
ag_models.FeedbackCategory.max).to_dict().keys())
def test_cmd_in_first_failed_test_override_normal_feedback(self):
cmd_res = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result)
fdbk = AGTestCommandResultFeedback(cmd_res,
ag_models.FeedbackCategory.normal,
self.ag_test_loader,
is_in_first_failed_test=True)
self.assertNotEqual(
self.ag_test_command.normal_fdbk_config.to_dict(),
self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict(
))
self.assertEqual(fdbk.fdbk_conf.return_code_fdbk_level,
ag_models.ValueFeedbackLevel.correct_or_incorrect)
self.assertEqual(fdbk.fdbk_conf.stdout_fdbk_level,
ag_models.ValueFeedbackLevel.expected_and_actual)
self.assertEqual(
fdbk.fdbk_settings,
self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict(
))
self.assertTrue(fdbk.return_code_correct)
self.assertTrue(fdbk.stdout_correct)
def setUp(self):
super().setUp()
self.maxDiff = None
submission = obj_build.make_submission()
project = submission.group.project
self.ag_test_suite = ag_models.AGTestSuite.objects.validate_and_create(
name='kajsdhf',
project=project,
setup_suite_cmd_name='asdlkjfa;skldjf;aksdf'
) # type: ag_models.AGTestSuite
self.ag_test_case1 = ag_models.AGTestCase.objects.validate_and_create(
name='aksdbva',
ag_test_suite=self.ag_test_suite) # type: ag_models.AGTestCase
self.ag_test_case2 = ag_models.AGTestCase.objects.validate_and_create(
name='ndkaadjhfa',
ag_test_suite=self.ag_test_suite,
) # type: ag_models.AGTestCase
self.ag_test_cmd1 = obj_build.make_full_ag_test_command(
self.ag_test_case1,
set_arbitrary_points=False,
points_for_correct_return_code=4)
self.ag_test_cmd2 = obj_build.make_full_ag_test_command(
self.ag_test_case2,
set_arbitrary_points=False,
points_for_correct_return_code=5)
self.total_points = 9
self.ag_test_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
submission=submission, ag_test_suite=self.ag_test_suite
) # type: ag_models.AGTestSuiteResult
self.ag_test_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_case=self.ag_test_case1,
ag_test_suite_result=self.ag_test_suite_result
) # type: ag_models.AGTestCaseResult
self.ag_test_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_case=self.ag_test_case2,
ag_test_suite_result=self.ag_test_suite_result
) # type: ag_models.AGTestCaseResult
self.cmd_result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_test_case_result1)
self.cmd_result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result2)
def setUp(self):
super().setUp()
self.maxDiff = None
self.client = APIClient()
self.project = obj_build.make_project(
ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.
most_recent, )
self.url = reverse('project-ultimate-submission-scores',
kwargs={'pk': self.project.pk})
self.admin = obj_build.make_admin_user(self.project.course)
self.ag_test_suite = obj_build.make_ag_test_suite(project=self.project)
self.ag_test_case = obj_build.make_ag_test_case(
ag_test_suite=self.ag_test_suite)
self.ag_test_cmd = obj_build.make_full_ag_test_command(
ag_test_case=self.ag_test_case)
self.student_group = obj_build.make_group(project=self.project,
num_members=2)
self.student_submission = obj_build.make_finished_submission(
self.student_group)
self.student_result = obj_build.make_incorrect_ag_test_command_result(
ag_test_command=self.ag_test_cmd,
submission=self.student_submission)
self.student_submission = update_denormalized_ag_test_results(
self.student_submission.pk)
self.student_result_fdbk = SubmissionResultFeedback(
self.student_submission, ag_models.FeedbackCategory.max,
AGTestPreLoader(self.project))
self.assertEqual(0, self.student_result_fdbk.total_points)
self.assertNotEqual(0, self.student_result_fdbk.total_points_possible)
self.staff_group = obj_build.make_group(
project=self.project, members_role=obj_build.UserRole.admin)
self.staff_submission = obj_build.make_finished_submission(
self.staff_group)
self.staff_result = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_cmd, submission=self.staff_submission)
self.staff_submission = update_denormalized_ag_test_results(
self.staff_submission.pk)
self.staff_result_fdbk = SubmissionResultFeedback(
self.staff_submission, ag_models.FeedbackCategory.max,
AGTestPreLoader(self.project))
self.assertNotEqual(0, self.staff_result_fdbk.total_points)
self.assertNotEqual(0, self.staff_result_fdbk.total_points_possible)
# Make sure we use the right queryset
other_project = obj_build.make_project(course=self.project.course)
other_group = obj_build.make_group(project=other_project)
other_submission = obj_build.make_finished_submission(other_group)
class _TestCase(UnitTestBase):
class GroupAndSubmissionData:
def __init__(self, group: ag_models.Group,
best_submission: ag_models.Submission,
most_recent_submission: ag_models.Submission):
self.group = group
self.best_submission = best_submission
self.most_recent_submission = most_recent_submission
def prepare_data(self,
project: ag_models.Project,
num_suites=1,
cases_per_suite=1,
cmds_per_suite=1,
num_groups=1,
num_other_submissions=0) -> List[GroupAndSubmissionData]:
cmds = []
for i in range(num_suites):
sys.stdout.write('\rBuilding suite {}'.format(i))
sys.stdout.flush()
suite = obj_build.make_ag_test_suite(project)
for j in range(cases_per_suite):
case = obj_build.make_ag_test_case(suite)
for k in range(cmds_per_suite):
cmd = obj_build.make_full_ag_test_command(case)
cmds.append(cmd)
group_and_submission_data = []
for i in range(num_groups):
sys.stdout.write('\rBuilding group {}'.format(i))
sys.stdout.flush()
group = obj_build.make_group(project=project)
best_sub = obj_build.make_finished_submission(group=group)
for cmd in cmds:
obj_build.make_correct_ag_test_command_result(
cmd, submission=best_sub)
best_sub = update_denormalized_ag_test_results(best_sub.pk)
for j in range(num_other_submissions):
sub = obj_build.make_finished_submission(group=group)
for cmd in cmds:
obj_build.make_incorrect_ag_test_command_result(
cmd, submission=sub)
sub = update_denormalized_ag_test_results(sub.pk)
most_recent = obj_build.make_finished_submission(group=group)
for cmd in cmds:
obj_build.make_incorrect_ag_test_command_result(
cmd, submission=most_recent)
most_recent = update_denormalized_ag_test_results(most_recent.pk)
group_and_submission_data.append(
self.GroupAndSubmissionData(group, best_sub, most_recent))
return group_and_submission_data
def do_get_ultimate_submission_test(self,
projects,
group_funcs,
users,
closing_time,
extension=None,
hide_ultimates=False):
for project in projects:
project.validate_and_update(
closing_time=closing_time,
hide_ultimate_submission_fdbk=hide_ultimates)
for group_func in group_funcs:
group = group_func(project)
group.validate_and_update(extended_due_date=extension)
suite = obj_build.make_ag_test_suite(project)
case = obj_build.make_ag_test_case(suite)
cmd = obj_build.make_full_ag_test_command(case)
best_submission = obj_build.make_finished_submission(
group=group)
most_recent_submission = obj_build.make_finished_submission(
group=group)
obj_build.make_correct_ag_test_command_result(
cmd, submission=best_submission)
obj_build.make_incorrect_ag_test_command_result(
cmd, submission=most_recent_submission)
for user in users:
url = self.ultimate_submission_url(group)
project.validate_and_update(
ultimate_submission_policy=ag_models.
UltimateSubmissionPolicy.most_recent)
self.do_get_object_test(self.client, user, url,
most_recent_submission.to_dict())
project.validate_and_update(
ultimate_submission_policy=ag_models.
UltimateSubmissionPolicy.best)
self.do_get_object_test(self.client, user, url,
best_submission.to_dict())
def test_total_points_minimum_ag_test_command_fdbk(self):
self.ag_test_cmd1.validate_and_update(
points_for_correct_return_code=1,
points_for_correct_stdout=2,
points_for_correct_stderr=3)
self.ag_test_cmd2.validate_and_update(
deduction_for_wrong_return_code=-4,
deduction_for_wrong_stdout=-2,
deduction_for_wrong_stderr=-1)
obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result)
obj_build.make_correct_ag_test_command_result(self.ag_test_cmd2, self.ag_test_case_result)
self.assertEqual(
0,
get_case_fdbk(self.ag_test_case_result,
ag_models.FeedbackCategory.normal).total_points)
self.assertEqual(
0,
get_case_fdbk(self.ag_test_case_result,
ag_models.FeedbackCategory.normal).total_points_possible)
def setUp(self):
super().setUp()
submission = obj_build.make_submission()
self.project = submission.group.project
suite1 = ag_models.AGTestSuite.objects.validate_and_create(
name='kajsdhf', project=self.project)
self.ag_test_case1 = ag_models.AGTestCase.objects.validate_and_create(
name='aksdbva', ag_test_suite=suite1)
self.ag_test_case2 = ag_models.AGTestCase.objects.validate_and_create(
name='noniresta', ag_test_suite=suite1)
suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
submission=submission, ag_test_suite=suite1)
self.ag_test_case1_correct_result = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_case=self.ag_test_case1, ag_test_suite_result=suite_result)
self.ag_test_case2_incorrect_result = (
ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_case=self.ag_test_case2, ag_test_suite_result=suite_result)
)
self.ag_test_case1_cmd = obj_build.make_full_ag_test_command(
self.ag_test_case1,
first_failed_test_normal_fdbk_config=(
ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config())
)
self.total_points_possible = (self.ag_test_case1_cmd.points_for_correct_return_code
+ self.ag_test_case1_cmd.points_for_correct_stdout
+ self.ag_test_case1_cmd.points_for_correct_stderr)
self.ag_test_case2_cmd = obj_build.make_full_ag_test_command(
self.ag_test_case2,
first_failed_test_normal_fdbk_config=(
ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config())
)
self.case1_cmd_res = obj_build.make_correct_ag_test_command_result(
self.ag_test_case1_cmd, ag_test_case_result=self.ag_test_case1_correct_result)
self.case2_cmd_res = obj_build.make_incorrect_ag_test_command_result(
self.ag_test_case2_cmd, ag_test_case_result=self.ag_test_case2_incorrect_result)
case1_max_fdbk = get_case_fdbk(
self.ag_test_case1_correct_result, ag_models.FeedbackCategory.max)
self.assertEqual(self.total_points_possible, case1_max_fdbk.total_points)
self.assertEqual(self.total_points_possible, case1_max_fdbk.total_points_possible)
case2_max_fdbk = get_case_fdbk(
self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.max)
self.assertEqual(0, case2_max_fdbk.total_points)
self.assertEqual(self.total_points_possible, case2_max_fdbk.total_points_possible)
def test_fdbk_to_dict(self):
self.maxDiff = None
self.ag_test_cmd1.validate_and_update(
points_for_correct_return_code=2,
points_for_correct_stdout=3,
points_for_correct_stderr=8
)
self.ag_test_cmd2.validate_and_update(
points_for_correct_return_code=4,
points_for_correct_stdout=5,
points_for_correct_stderr=7
)
result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_test_case_result)
result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_test_case_result)
expected_keys = [
'pk',
'ag_test_case_name',
'ag_test_case_pk',
'fdbk_settings',
'total_points',
'total_points_possible',
'ag_test_command_results',
]
for fdbk_category in ag_models.FeedbackCategory:
result_dict = get_case_fdbk(self.ag_test_case_result, fdbk_category).to_dict()
self.assertCountEqual(expected_keys, result_dict.keys())
self.assertCountEqual(
[get_cmd_fdbk(result1, fdbk_category).to_dict(),
get_cmd_fdbk(result2, fdbk_category).to_dict()],
result_dict['ag_test_command_results'])
def test_cmd_not_in_first_failed_test_gets_normal_feedback(self):
cmd_res = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result)
fdbk = AGTestCommandResultFeedback(cmd_res,
ag_models.FeedbackCategory.normal,
self.ag_test_loader,
is_in_first_failed_test=False)
self.assertNotEqual(
self.ag_test_command.normal_fdbk_config.to_dict(),
self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict(
))
self.assertEqual(fdbk.fdbk_settings,
self.ag_test_command.normal_fdbk_config.to_dict())
def test_non_normal_fdbk_no_override(self):
cmd_res = obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result)
fdbk = AGTestCommandResultFeedback(
cmd_res,
ag_models.FeedbackCategory.ultimate_submission,
self.ag_test_loader,
is_in_first_failed_test=True)
self.assertNotEqual(
self.ag_test_command.normal_fdbk_config.to_dict(),
self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict(
))
self.assertEqual(
fdbk.fdbk_settings,
self.ag_test_command.ultimate_submission_fdbk_config.to_dict())
def setUp(self):
super().setUp()
self.client = APIClient()
max_group_size = 3
self.project = obj_build.make_project(
visible_to_students=True,
ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.best,
max_group_size=max_group_size)
self.suite1 = obj_build.make_ag_test_suite(self.project)
self.suite1_case = obj_build.make_ag_test_case(self.suite1)
self.suite1_cmd = obj_build.make_full_ag_test_command(self.suite1_case)
self.suite2 = obj_build.make_ag_test_suite(self.project)
self.suite2_case = obj_build.make_ag_test_case(self.suite2)
self.suite2_cmd = obj_build.make_full_ag_test_command(self.suite2_case)
self.student_suite1_bugs = ['suite1_bug{}'.format(i) for i in range(3)]
self.student_suite1 = obj_build.make_student_test_suite(
project=self.project, buggy_impl_names=self.student_suite1_bugs)
self.student_suite2_bugs = ['suite2_bug{}'.format(i) for i in range(5)]
self.student_suite2 = obj_build.make_student_test_suite(
project=self.project, buggy_impl_names=self.student_suite2_bugs)
self.student_group1 = obj_build.make_group(project=self.project)
self.group1_submission1_best = obj_build.make_finished_submission(
group=self.student_group1)
obj_build.make_correct_ag_test_command_result(
self.suite1_cmd, submission=self.group1_submission1_best)
obj_build.make_correct_ag_test_command_result(
self.suite2_cmd, submission=self.group1_submission1_best)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.group1_submission1_best,
bugs_exposed=self.student_suite1_bugs)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.group1_submission1_best,
bugs_exposed=self.student_suite2_bugs)
self.group1_submission2 = obj_build.make_finished_submission(
group=self.student_group1)
obj_build.make_correct_ag_test_command_result(
self.suite1_cmd, submission=self.group1_submission2)
obj_build.make_incorrect_ag_test_command_result(
self.suite2_cmd, submission=self.group1_submission2)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.group1_submission2,
bugs_exposed=[])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.group1_submission2,
bugs_exposed=[])
self.group1_submission1_best = update_denormalized_ag_test_results(
self.group1_submission1_best.pk)
self.group1_submission2 = update_denormalized_ag_test_results(
self.group1_submission2.pk)
self.student_group2 = obj_build.make_group(num_members=max_group_size,
project=self.project)
self.group2_only_submission = obj_build.make_finished_submission(
group=self.student_group2)
obj_build.make_incorrect_ag_test_command_result(
self.suite1_cmd, submission=self.group2_only_submission)
obj_build.make_incorrect_ag_test_command_result(
self.suite2_cmd, submission=self.group2_only_submission)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.group2_only_submission,
bugs_exposed=self.student_suite1_bugs[:-1])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.group2_only_submission,
bugs_exposed=self.student_suite2_bugs[:-1])
self.group2_only_submission = update_denormalized_ag_test_results(
self.group2_only_submission.pk)
self.staff_group = obj_build.make_group(
project=self.project, members_role=obj_build.UserRole.staff)
self.staff_submission1 = obj_build.make_finished_submission(
group=self.staff_group)
obj_build.make_correct_ag_test_command_result(
self.suite1_cmd, submission=self.staff_submission1)
obj_build.make_correct_ag_test_command_result(
self.suite2_cmd, submission=self.staff_submission1)
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.staff_submission1,
bugs_exposed=self.student_suite1_bugs[:-1])
ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.staff_submission1,
bugs_exposed=self.student_suite2_bugs)
self.staff_submission1 = update_denormalized_ag_test_results(
self.staff_submission1.pk)
self.no_submissions_group = obj_build.make_group(project=self.project)
[self.admin] = obj_build.make_admin_users(self.project.course, 1)
self.maxDiff = None
def test_move_ag_test_case_to_different_suite_in_same_project(self):
other_suite = ag_models.AGTestSuite.objects.validate_and_create(
name='fa;weifjawef',
project=self.project) # type: ag_models.AGTestSuite
ag_test = ag_models.AGTestCase.objects.validate_and_create(
name='asdlkfjaewi;ojf',
ag_test_suite=self.ag_suite) # type: ag_models.AGTestCase
cmd = ag_models.AGTestCommand.objects.validate_and_create(
name='asdfknja;wej',
ag_test_case=ag_test,
cmd='asdklfja;sdjkfaldsf') # type: ag_models.AGTestCommand
# This group has results for ONLY self.ag_suite
submission = obj_build.make_submission(group=obj_build.build_group(
group_kwargs={'project': self.project}))
suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
submission=submission,
ag_test_suite=self.ag_suite) # type: ag_models.AGTestSuiteResult
ag_test_result = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_case=ag_test, ag_test_suite_result=suite_result
) # type: ag_models.AGTestCaseResult
cmd_result = obj_build.make_correct_ag_test_command_result(
cmd, ag_test_result)
# This group has results for self.ag_suite and other_suite
submission2 = obj_build.make_submission(group=obj_build.build_group(
group_kwargs={'project': self.project}))
suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create(
submission=submission2,
ag_test_suite=self.ag_suite) # type: ag_models.AGTestSuiteResult
ag_test_result2 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_case=ag_test, ag_test_suite_result=suite_result2
) # type: ag_models.AGTestCaseResult
cmd_result2 = obj_build.make_correct_ag_test_command_result(
cmd, ag_test_result2)
other_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create(
submission=submission2,
ag_test_suite=other_suite) # type: ag_models.AGTestSuiteResult
self.assertEqual(self.ag_suite, ag_test.ag_test_suite)
# This that should be re-wired after this line:
# - ag_test should belong to other_suite
# - AGTestCaseResults should point to suite results in their
# own submissions that belong to other_suite (creating them
# if necessary).
ag_test.validate_and_update(ag_test_suite=other_suite)
other_suite = ag_models.AGTestSuite.objects.get(pk=other_suite.pk)
self.assertSequenceEqual([ag_test], other_suite.ag_test_cases.all())
self.ag_suite.refresh_from_db()
self.assertSequenceEqual([], self.ag_suite.ag_test_cases.all())
# An AGTestSuiteResult for other_suite should have been created for submission.
self.assertEqual(2, submission.ag_test_suite_results.count())
self.assertEqual(suite_result,
submission.ag_test_suite_results.first())
new_other_suite_result = submission.ag_test_suite_results.last()
self.assertEqual(other_suite, new_other_suite_result.ag_test_suite)
# ag_test should belong to other_suite
self.assertEqual(other_suite, ag_test.ag_test_suite)
# ag_test_result should belong to new_other_suite_result
ag_test_result.refresh_from_db()
self.assertEqual(new_other_suite_result,
ag_test_result.ag_test_suite_result)
# ag_test_result2 should belong to other_suite_result
ag_test_result2.refresh_from_db()
self.assertEqual(other_suite_result,
ag_test_result2.ag_test_suite_result)
# Make sure no errors are thrown
get_suite_fdbk(suite_result,
ag_models.FeedbackCategory.max).ag_test_case_results
get_suite_fdbk(suite_result2,
ag_models.FeedbackCategory.max).ag_test_case_results
get_suite_fdbk(new_other_suite_result,
ag_models.FeedbackCategory.max).ag_test_case_results
def make_correct_result(self) -> ag_models.AGTestCommandResult:
return obj_build.make_correct_ag_test_command_result(
ag_test_command=self.ag_test_command,
ag_test_case_result=self.ag_test_case_result)
def setUp(self):
super().setUp()
self.maxDiff = None
self.project = obj_build.make_project()
self.course = self.project.course
self.ag_test_suite1 = obj_build.make_ag_test_suite(self.project)
self.ag_test_case1 = obj_build.make_ag_test_case(self.ag_test_suite1)
self.ag_test_cmd1 = obj_build.make_full_ag_test_command(
self.ag_test_case1, set_arbitrary_points=True)
self.ag_test_suite2 = obj_build.make_ag_test_suite(self.project)
self.ag_test_case2 = obj_build.make_ag_test_case(self.ag_test_suite2)
self.ag_test_cmd2 = obj_build.make_full_ag_test_command(
self.ag_test_case2, set_arbitrary_points=True)
self.points_per_bug_exposed = Decimal('3.5')
self.num_buggy_impls = 4
self.student_suite1 = ag_models.StudentTestSuite.objects.validate_and_create(
name='suite1',
project=self.project,
buggy_impl_names=[
'bug{}'.format(i) for i in range(self.num_buggy_impls)
],
points_per_exposed_bug=self.points_per_bug_exposed
) # type: ag_models.StudentTestSuite
self.student_suite2 = ag_models.StudentTestSuite.objects.validate_and_create(
name='suite2',
project=self.project,
buggy_impl_names=[
'bug{}'.format(i) for i in range(self.num_buggy_impls)
],
points_per_exposed_bug=self.points_per_bug_exposed
) # type: ag_models.StudentTestSuite
self.group = obj_build.make_group(num_members=1, project=self.project)
self.submission = obj_build.make_submission(group=self.group)
self.ag_suite_result1 = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_test_suite1,
submission=self.submission) # type: ag_models.AGTestSuiteResult
self.ag_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_suite_result=self.ag_suite_result1,
ag_test_case=self.ag_test_case1
) # type: ag_models.AGTestCaseResult
self.ag_cmd_result1 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd1, self.ag_case_result1)
self.ag_suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create(
ag_test_suite=self.ag_test_suite2,
submission=self.submission) # type: ag_models.AGTestSuiteResult
self.ag_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create(
ag_test_suite_result=self.ag_suite_result2,
ag_test_case=self.ag_test_case2
) # type: ag_models.AGTestCaseResult
self.ag_cmd_result2 = obj_build.make_correct_ag_test_command_result(
self.ag_test_cmd2, self.ag_case_result2)
self.num_student_tests = 6
self.student_tests = [
'test{}'.format(i) for i in range(self.num_student_tests)
]
self.student_suite_result1 = ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite1,
submission=self.submission,
student_tests=self.student_tests,
bugs_exposed=self.student_suite1.buggy_impl_names
) # type: ag_models.StudentTestSuiteResult
self.student_suite_result2 = ag_models.StudentTestSuiteResult.objects.validate_and_create(
student_test_suite=self.student_suite2,
submission=self.submission,
student_tests=self.student_tests,
bugs_exposed=self.student_suite2.buggy_impl_names
) # type: ag_models.StudentTestSuiteResult
self.total_points_per_ag_suite = get_suite_fdbk(
self.ag_suite_result1, ag_models.FeedbackCategory.max).total_points
self.total_points_per_student_suite = self.num_buggy_impls * self.points_per_bug_exposed
self.total_points = (self.total_points_per_ag_suite * 2 +
self.total_points_per_student_suite * 2)
self.total_points_possible = self.total_points
self.assertEqual(
self.total_points_per_ag_suite,
get_suite_fdbk(self.ag_suite_result2,
ag_models.FeedbackCategory.max).total_points)
self.assertEqual(
self.total_points_per_student_suite,
self.student_suite_result1.get_fdbk(
ag_models.FeedbackCategory.max).total_points)
print(self.total_points)
self.assertNotEqual(0, self.total_points_per_ag_suite)
self.assertNotEqual(0, self.total_points_per_student_suite)
self.assertNotEqual(0, self.total_points)
self.submission = update_denormalized_ag_test_results(
self.submission.pk)
