def test_no_failed_tests_all_get_normal_fdbk(self): self.case2_cmd_res.delete() self.case3_cmd_res.delete() ag_test_case2_correct_result = self.ag_test_case2_incorrect_result ag_test_case3_correct_result = self.ag_test_case3_incorrect_result self.case2_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case2_cmd, ag_test_case_result=ag_test_case2_correct_result) self.case3_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case3_cmd, ag_test_case_result=ag_test_case3_correct_result) fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.normal) expected_case_fdbks = [ get_case_fdbk(self.ag_test_case1_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(ag_test_case2_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(ag_test_case3_correct_result, ag_models.FeedbackCategory.normal).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_some_commands_not_visible(self): self.ag_test_cmd1.validate_and_update(ultimate_submission_fdbk_config={'visible': False}) cmd1_pts = 5 cmd2_pts = 3 self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=cmd1_pts) self.ag_test_cmd2.validate_and_update(points_for_correct_return_code=cmd2_pts) cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( cmd1_pts + cmd2_pts, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points) self.assertEqual( cmd1_pts + cmd2_pts, get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points_possible) fdbk = get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission) self.assertSequenceEqual([cmd_result2.pk], [res.pk for res in fdbk.ag_test_command_results]) self.assertEqual( cmd2_pts, get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission).total_points) self.assertEqual( cmd2_pts, get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission).total_points_possible)
def test_show_individual_commands(self): total_cmd_points = 6 self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=total_cmd_points) self.ag_test_cmd1.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True } ) self.assertTrue(self.ag_test_case.normal_fdbk_config.show_individual_commands) result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertEqual([result1.pk, result2.pk], [res.pk for res in fdbk.ag_test_command_results]) self.assertEqual(total_cmd_points, fdbk.total_points) self.assertEqual(total_cmd_points, fdbk.total_points_possible) self.ag_test_case.validate_and_update( normal_fdbk_config={'show_individual_commands': False}) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal) self.assertEqual([], fdbk.ag_test_command_results) self.assertEqual(total_cmd_points, fdbk.total_points) self.assertEqual(total_cmd_points, fdbk.total_points_possible)
def test_cmd_result_query_order(self): result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd2.pk, self.ag_test_cmd1.pk]) self.ag_test_case_result = ag_models.AGTestCaseResult.objects.get( pk=self.ag_test_case_result.pk) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertEqual([result2.pk, result1.pk], [res.pk for res in fdbk.ag_test_command_results])
def _add_results_to_submission( self, submission: ag_models.Submission, *, results_correct: bool) -> ag_models.Submission: if results_correct: obj_build.make_correct_ag_test_command_result( self.ag_test_cmd, submission=submission) else: obj_build.make_incorrect_ag_test_command_result( self.ag_test_cmd, submission=submission) ag_models.StudentTestSuiteResult.objects.validate_and_create( submission=submission, student_test_suite=self.student_test_suite) return update_denormalized_ag_test_results(submission.pk)
def test_total_points_some_positive_some_negative_positive_total(self): self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=5) self.ag_test_cmd2.validate_and_update(deduction_for_wrong_stdout=-2) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result) obj_build.make_incorrect_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( 3, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points) self.assertEqual( 5, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points_possible)
def test_total_points_all_positive(self): # make sure sum is correct and takes into account deductions # make sure sum doesn't go below zero self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=5) self.ag_test_cmd2.validate_and_update(points_for_correct_stdout=3) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( 8, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points) self.assertEqual( 8, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points_possible)
def test_ag_test_cmd_result_ordering(self): cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, ag_test_case_result=self.ag_test_case_result) cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, ag_test_case_result=self.ag_test_case_result) for i in range(2): self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd2.pk, self.ag_test_cmd1.pk]) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertSequenceEqual([cmd_result2.pk, cmd_result1.pk], [res.pk for res in fdbk.ag_test_command_results]) self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd1.pk, self.ag_test_cmd2.pk]) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertSequenceEqual([cmd_result1.pk, cmd_result2.pk], [res.pk for res in fdbk.ag_test_command_results])
def test_fdbk_to_dict(self): result = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result) expected_keys = [ 'pk', 'ag_test_command_name', 'ag_test_command_pk', 'fdbk_settings', 'timed_out', 'return_code_correct', 'expected_return_code', 'actual_return_code', 'return_code_points', 'return_code_points_possible', 'stdout_correct', 'stdout_points', 'stdout_points_possible', 'stderr_correct', 'stderr_points', 'stderr_points_possible', 'total_points', 'total_points_possible', ] self.assertCountEqual( expected_keys, get_cmd_fdbk(result, ag_models.FeedbackCategory.max).to_dict().keys())
def test_cmd_in_first_failed_test_override_normal_feedback(self): cmd_res = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result) fdbk = AGTestCommandResultFeedback(cmd_res, ag_models.FeedbackCategory.normal, self.ag_test_loader, is_in_first_failed_test=True) self.assertNotEqual( self.ag_test_command.normal_fdbk_config.to_dict(), self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict( )) self.assertEqual(fdbk.fdbk_conf.return_code_fdbk_level, ag_models.ValueFeedbackLevel.correct_or_incorrect) self.assertEqual(fdbk.fdbk_conf.stdout_fdbk_level, ag_models.ValueFeedbackLevel.expected_and_actual) self.assertEqual( fdbk.fdbk_settings, self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict( )) self.assertTrue(fdbk.return_code_correct) self.assertTrue(fdbk.stdout_correct)
def setUp(self): super().setUp() self.maxDiff = None submission = obj_build.make_submission() project = submission.group.project self.ag_test_suite = ag_models.AGTestSuite.objects.validate_and_create( name='kajsdhf', project=project, setup_suite_cmd_name='asdlkjfa;skldjf;aksdf' ) # type: ag_models.AGTestSuite self.ag_test_case1 = ag_models.AGTestCase.objects.validate_and_create( name='aksdbva', ag_test_suite=self.ag_test_suite) # type: ag_models.AGTestCase self.ag_test_case2 = ag_models.AGTestCase.objects.validate_and_create( name='ndkaadjhfa', ag_test_suite=self.ag_test_suite, ) # type: ag_models.AGTestCase self.ag_test_cmd1 = obj_build.make_full_ag_test_command( self.ag_test_case1, set_arbitrary_points=False, points_for_correct_return_code=4) self.ag_test_cmd2 = obj_build.make_full_ag_test_command( self.ag_test_case2, set_arbitrary_points=False, points_for_correct_return_code=5) self.total_points = 9 self.ag_test_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission, ag_test_suite=self.ag_test_suite ) # type: ag_models.AGTestSuiteResult self.ag_test_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case1, ag_test_suite_result=self.ag_test_suite_result ) # type: ag_models.AGTestCaseResult self.ag_test_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case2, ag_test_suite_result=self.ag_test_suite_result ) # type: ag_models.AGTestCaseResult self.cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result1) self.cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result2)
def setUp(self): super().setUp() self.maxDiff = None self.client = APIClient() self.project = obj_build.make_project( ultimate_submission_policy=ag_models.UltimateSubmissionPolicy. most_recent, ) self.url = reverse('project-ultimate-submission-scores', kwargs={'pk': self.project.pk}) self.admin = obj_build.make_admin_user(self.project.course) self.ag_test_suite = obj_build.make_ag_test_suite(project=self.project) self.ag_test_case = obj_build.make_ag_test_case( ag_test_suite=self.ag_test_suite) self.ag_test_cmd = obj_build.make_full_ag_test_command( ag_test_case=self.ag_test_case) self.student_group = obj_build.make_group(project=self.project, num_members=2) self.student_submission = obj_build.make_finished_submission( self.student_group) self.student_result = obj_build.make_incorrect_ag_test_command_result( ag_test_command=self.ag_test_cmd, submission=self.student_submission) self.student_submission = update_denormalized_ag_test_results( self.student_submission.pk) self.student_result_fdbk = SubmissionResultFeedback( self.student_submission, ag_models.FeedbackCategory.max, AGTestPreLoader(self.project)) self.assertEqual(0, self.student_result_fdbk.total_points) self.assertNotEqual(0, self.student_result_fdbk.total_points_possible) self.staff_group = obj_build.make_group( project=self.project, members_role=obj_build.UserRole.admin) self.staff_submission = obj_build.make_finished_submission( self.staff_group) self.staff_result = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_cmd, submission=self.staff_submission) self.staff_submission = update_denormalized_ag_test_results( self.staff_submission.pk) self.staff_result_fdbk = SubmissionResultFeedback( self.staff_submission, ag_models.FeedbackCategory.max, AGTestPreLoader(self.project)) self.assertNotEqual(0, self.staff_result_fdbk.total_points) self.assertNotEqual(0, self.staff_result_fdbk.total_points_possible) # Make sure we use the right queryset other_project = obj_build.make_project(course=self.project.course) other_group = obj_build.make_group(project=other_project) other_submission = obj_build.make_finished_submission(other_group)
class _TestCase(UnitTestBase): class GroupAndSubmissionData: def __init__(self, group: ag_models.Group, best_submission: ag_models.Submission, most_recent_submission: ag_models.Submission): self.group = group self.best_submission = best_submission self.most_recent_submission = most_recent_submission def prepare_data(self, project: ag_models.Project, num_suites=1, cases_per_suite=1, cmds_per_suite=1, num_groups=1, num_other_submissions=0) -> List[GroupAndSubmissionData]: cmds = [] for i in range(num_suites): sys.stdout.write('\rBuilding suite {}'.format(i)) sys.stdout.flush() suite = obj_build.make_ag_test_suite(project) for j in range(cases_per_suite): case = obj_build.make_ag_test_case(suite) for k in range(cmds_per_suite): cmd = obj_build.make_full_ag_test_command(case) cmds.append(cmd) group_and_submission_data = [] for i in range(num_groups): sys.stdout.write('\rBuilding group {}'.format(i)) sys.stdout.flush() group = obj_build.make_group(project=project) best_sub = obj_build.make_finished_submission(group=group) for cmd in cmds: obj_build.make_correct_ag_test_command_result( cmd, submission=best_sub) best_sub = update_denormalized_ag_test_results(best_sub.pk) for j in range(num_other_submissions): sub = obj_build.make_finished_submission(group=group) for cmd in cmds: obj_build.make_incorrect_ag_test_command_result( cmd, submission=sub) sub = update_denormalized_ag_test_results(sub.pk) most_recent = obj_build.make_finished_submission(group=group) for cmd in cmds: obj_build.make_incorrect_ag_test_command_result( cmd, submission=most_recent) most_recent = update_denormalized_ag_test_results(most_recent.pk) group_and_submission_data.append( self.GroupAndSubmissionData(group, best_sub, most_recent)) return group_and_submission_data
def do_get_ultimate_submission_test(self, projects, group_funcs, users, closing_time, extension=None, hide_ultimates=False): for project in projects: project.validate_and_update( closing_time=closing_time, hide_ultimate_submission_fdbk=hide_ultimates) for group_func in group_funcs: group = group_func(project) group.validate_and_update(extended_due_date=extension) suite = obj_build.make_ag_test_suite(project) case = obj_build.make_ag_test_case(suite) cmd = obj_build.make_full_ag_test_command(case) best_submission = obj_build.make_finished_submission( group=group) most_recent_submission = obj_build.make_finished_submission( group=group) obj_build.make_correct_ag_test_command_result( cmd, submission=best_submission) obj_build.make_incorrect_ag_test_command_result( cmd, submission=most_recent_submission) for user in users: url = self.ultimate_submission_url(group) project.validate_and_update( ultimate_submission_policy=ag_models. UltimateSubmissionPolicy.most_recent) self.do_get_object_test(self.client, user, url, most_recent_submission.to_dict()) project.validate_and_update( ultimate_submission_policy=ag_models. UltimateSubmissionPolicy.best) self.do_get_object_test(self.client, user, url, best_submission.to_dict())
def test_total_points_minimum_ag_test_command_fdbk(self): self.ag_test_cmd1.validate_and_update( points_for_correct_return_code=1, points_for_correct_stdout=2, points_for_correct_stderr=3) self.ag_test_cmd2.validate_and_update( deduction_for_wrong_return_code=-4, deduction_for_wrong_stdout=-2, deduction_for_wrong_stderr=-1) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( 0, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal).total_points) self.assertEqual( 0, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal).total_points_possible)
def setUp(self): super().setUp() submission = obj_build.make_submission() self.project = submission.group.project suite1 = ag_models.AGTestSuite.objects.validate_and_create( name='kajsdhf', project=self.project) self.ag_test_case1 = ag_models.AGTestCase.objects.validate_and_create( name='aksdbva', ag_test_suite=suite1) self.ag_test_case2 = ag_models.AGTestCase.objects.validate_and_create( name='noniresta', ag_test_suite=suite1) suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission, ag_test_suite=suite1) self.ag_test_case1_correct_result = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case1, ag_test_suite_result=suite_result) self.ag_test_case2_incorrect_result = ( ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case2, ag_test_suite_result=suite_result) ) self.ag_test_case1_cmd = obj_build.make_full_ag_test_command( self.ag_test_case1, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config()) ) self.total_points_possible = (self.ag_test_case1_cmd.points_for_correct_return_code + self.ag_test_case1_cmd.points_for_correct_stdout + self.ag_test_case1_cmd.points_for_correct_stderr) self.ag_test_case2_cmd = obj_build.make_full_ag_test_command( self.ag_test_case2, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config()) ) self.case1_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case1_cmd, ag_test_case_result=self.ag_test_case1_correct_result) self.case2_cmd_res = obj_build.make_incorrect_ag_test_command_result( self.ag_test_case2_cmd, ag_test_case_result=self.ag_test_case2_incorrect_result) case1_max_fdbk = get_case_fdbk( self.ag_test_case1_correct_result, ag_models.FeedbackCategory.max) self.assertEqual(self.total_points_possible, case1_max_fdbk.total_points) self.assertEqual(self.total_points_possible, case1_max_fdbk.total_points_possible) case2_max_fdbk = get_case_fdbk( self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.max) self.assertEqual(0, case2_max_fdbk.total_points) self.assertEqual(self.total_points_possible, case2_max_fdbk.total_points_possible)
def test_fdbk_to_dict(self): self.maxDiff = None self.ag_test_cmd1.validate_and_update( points_for_correct_return_code=2, points_for_correct_stdout=3, points_for_correct_stderr=8 ) self.ag_test_cmd2.validate_and_update( points_for_correct_return_code=4, points_for_correct_stdout=5, points_for_correct_stderr=7 ) result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) expected_keys = [ 'pk', 'ag_test_case_name', 'ag_test_case_pk', 'fdbk_settings', 'total_points', 'total_points_possible', 'ag_test_command_results', ] for fdbk_category in ag_models.FeedbackCategory: result_dict = get_case_fdbk(self.ag_test_case_result, fdbk_category).to_dict() self.assertCountEqual(expected_keys, result_dict.keys()) self.assertCountEqual( [get_cmd_fdbk(result1, fdbk_category).to_dict(), get_cmd_fdbk(result2, fdbk_category).to_dict()], result_dict['ag_test_command_results'])
def test_cmd_not_in_first_failed_test_gets_normal_feedback(self): cmd_res = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result) fdbk = AGTestCommandResultFeedback(cmd_res, ag_models.FeedbackCategory.normal, self.ag_test_loader, is_in_first_failed_test=False) self.assertNotEqual( self.ag_test_command.normal_fdbk_config.to_dict(), self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict( )) self.assertEqual(fdbk.fdbk_settings, self.ag_test_command.normal_fdbk_config.to_dict())
def test_non_normal_fdbk_no_override(self): cmd_res = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result) fdbk = AGTestCommandResultFeedback( cmd_res, ag_models.FeedbackCategory.ultimate_submission, self.ag_test_loader, is_in_first_failed_test=True) self.assertNotEqual( self.ag_test_command.normal_fdbk_config.to_dict(), self.ag_test_command.first_failed_test_normal_fdbk_config.to_dict( )) self.assertEqual( fdbk.fdbk_settings, self.ag_test_command.ultimate_submission_fdbk_config.to_dict())
def setUp(self): super().setUp() self.client = APIClient() max_group_size = 3 self.project = obj_build.make_project( visible_to_students=True, ultimate_submission_policy=ag_models.UltimateSubmissionPolicy.best, max_group_size=max_group_size) self.suite1 = obj_build.make_ag_test_suite(self.project) self.suite1_case = obj_build.make_ag_test_case(self.suite1) self.suite1_cmd = obj_build.make_full_ag_test_command(self.suite1_case) self.suite2 = obj_build.make_ag_test_suite(self.project) self.suite2_case = obj_build.make_ag_test_case(self.suite2) self.suite2_cmd = obj_build.make_full_ag_test_command(self.suite2_case) self.student_suite1_bugs = ['suite1_bug{}'.format(i) for i in range(3)] self.student_suite1 = obj_build.make_student_test_suite( project=self.project, buggy_impl_names=self.student_suite1_bugs) self.student_suite2_bugs = ['suite2_bug{}'.format(i) for i in range(5)] self.student_suite2 = obj_build.make_student_test_suite( project=self.project, buggy_impl_names=self.student_suite2_bugs) self.student_group1 = obj_build.make_group(project=self.project) self.group1_submission1_best = obj_build.make_finished_submission( group=self.student_group1) obj_build.make_correct_ag_test_command_result( self.suite1_cmd, submission=self.group1_submission1_best) obj_build.make_correct_ag_test_command_result( self.suite2_cmd, submission=self.group1_submission1_best) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.group1_submission1_best, bugs_exposed=self.student_suite1_bugs) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.group1_submission1_best, bugs_exposed=self.student_suite2_bugs) self.group1_submission2 = obj_build.make_finished_submission( group=self.student_group1) obj_build.make_correct_ag_test_command_result( self.suite1_cmd, submission=self.group1_submission2) obj_build.make_incorrect_ag_test_command_result( self.suite2_cmd, submission=self.group1_submission2) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.group1_submission2, bugs_exposed=[]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.group1_submission2, bugs_exposed=[]) self.group1_submission1_best = update_denormalized_ag_test_results( self.group1_submission1_best.pk) self.group1_submission2 = update_denormalized_ag_test_results( self.group1_submission2.pk) self.student_group2 = obj_build.make_group(num_members=max_group_size, project=self.project) self.group2_only_submission = obj_build.make_finished_submission( group=self.student_group2) obj_build.make_incorrect_ag_test_command_result( self.suite1_cmd, submission=self.group2_only_submission) obj_build.make_incorrect_ag_test_command_result( self.suite2_cmd, submission=self.group2_only_submission) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.group2_only_submission, bugs_exposed=self.student_suite1_bugs[:-1]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.group2_only_submission, bugs_exposed=self.student_suite2_bugs[:-1]) self.group2_only_submission = update_denormalized_ag_test_results( self.group2_only_submission.pk) self.staff_group = obj_build.make_group( project=self.project, members_role=obj_build.UserRole.staff) self.staff_submission1 = obj_build.make_finished_submission( group=self.staff_group) obj_build.make_correct_ag_test_command_result( self.suite1_cmd, submission=self.staff_submission1) obj_build.make_correct_ag_test_command_result( self.suite2_cmd, submission=self.staff_submission1) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.staff_submission1, bugs_exposed=self.student_suite1_bugs[:-1]) ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.staff_submission1, bugs_exposed=self.student_suite2_bugs) self.staff_submission1 = update_denormalized_ag_test_results( self.staff_submission1.pk) self.no_submissions_group = obj_build.make_group(project=self.project) [self.admin] = obj_build.make_admin_users(self.project.course, 1) self.maxDiff = None
def test_move_ag_test_case_to_different_suite_in_same_project(self): other_suite = ag_models.AGTestSuite.objects.validate_and_create( name='fa;weifjawef', project=self.project) # type: ag_models.AGTestSuite ag_test = ag_models.AGTestCase.objects.validate_and_create( name='asdlkfjaewi;ojf', ag_test_suite=self.ag_suite) # type: ag_models.AGTestCase cmd = ag_models.AGTestCommand.objects.validate_and_create( name='asdfknja;wej', ag_test_case=ag_test, cmd='asdklfja;sdjkfaldsf') # type: ag_models.AGTestCommand # This group has results for ONLY self.ag_suite submission = obj_build.make_submission(group=obj_build.build_group( group_kwargs={'project': self.project})) suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission, ag_test_suite=self.ag_suite) # type: ag_models.AGTestSuiteResult ag_test_result = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=ag_test, ag_test_suite_result=suite_result ) # type: ag_models.AGTestCaseResult cmd_result = obj_build.make_correct_ag_test_command_result( cmd, ag_test_result) # This group has results for self.ag_suite and other_suite submission2 = obj_build.make_submission(group=obj_build.build_group( group_kwargs={'project': self.project})) suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission2, ag_test_suite=self.ag_suite) # type: ag_models.AGTestSuiteResult ag_test_result2 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=ag_test, ag_test_suite_result=suite_result2 ) # type: ag_models.AGTestCaseResult cmd_result2 = obj_build.make_correct_ag_test_command_result( cmd, ag_test_result2) other_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission2, ag_test_suite=other_suite) # type: ag_models.AGTestSuiteResult self.assertEqual(self.ag_suite, ag_test.ag_test_suite) # This that should be re-wired after this line: # - ag_test should belong to other_suite # - AGTestCaseResults should point to suite results in their # own submissions that belong to other_suite (creating them # if necessary). ag_test.validate_and_update(ag_test_suite=other_suite) other_suite = ag_models.AGTestSuite.objects.get(pk=other_suite.pk) self.assertSequenceEqual([ag_test], other_suite.ag_test_cases.all()) self.ag_suite.refresh_from_db() self.assertSequenceEqual([], self.ag_suite.ag_test_cases.all()) # An AGTestSuiteResult for other_suite should have been created for submission. self.assertEqual(2, submission.ag_test_suite_results.count()) self.assertEqual(suite_result, submission.ag_test_suite_results.first()) new_other_suite_result = submission.ag_test_suite_results.last() self.assertEqual(other_suite, new_other_suite_result.ag_test_suite) # ag_test should belong to other_suite self.assertEqual(other_suite, ag_test.ag_test_suite) # ag_test_result should belong to new_other_suite_result ag_test_result.refresh_from_db() self.assertEqual(new_other_suite_result, ag_test_result.ag_test_suite_result) # ag_test_result2 should belong to other_suite_result ag_test_result2.refresh_from_db() self.assertEqual(other_suite_result, ag_test_result2.ag_test_suite_result) # Make sure no errors are thrown get_suite_fdbk(suite_result, ag_models.FeedbackCategory.max).ag_test_case_results get_suite_fdbk(suite_result2, ag_models.FeedbackCategory.max).ag_test_case_results get_suite_fdbk(new_other_suite_result, ag_models.FeedbackCategory.max).ag_test_case_results
def make_correct_result(self) -> ag_models.AGTestCommandResult: return obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result)
def setUp(self): super().setUp() self.maxDiff = None self.project = obj_build.make_project() self.course = self.project.course self.ag_test_suite1 = obj_build.make_ag_test_suite(self.project) self.ag_test_case1 = obj_build.make_ag_test_case(self.ag_test_suite1) self.ag_test_cmd1 = obj_build.make_full_ag_test_command( self.ag_test_case1, set_arbitrary_points=True) self.ag_test_suite2 = obj_build.make_ag_test_suite(self.project) self.ag_test_case2 = obj_build.make_ag_test_case(self.ag_test_suite2) self.ag_test_cmd2 = obj_build.make_full_ag_test_command( self.ag_test_case2, set_arbitrary_points=True) self.points_per_bug_exposed = Decimal('3.5') self.num_buggy_impls = 4 self.student_suite1 = ag_models.StudentTestSuite.objects.validate_and_create( name='suite1', project=self.project, buggy_impl_names=[ 'bug{}'.format(i) for i in range(self.num_buggy_impls) ], points_per_exposed_bug=self.points_per_bug_exposed ) # type: ag_models.StudentTestSuite self.student_suite2 = ag_models.StudentTestSuite.objects.validate_and_create( name='suite2', project=self.project, buggy_impl_names=[ 'bug{}'.format(i) for i in range(self.num_buggy_impls) ], points_per_exposed_bug=self.points_per_bug_exposed ) # type: ag_models.StudentTestSuite self.group = obj_build.make_group(num_members=1, project=self.project) self.submission = obj_build.make_submission(group=self.group) self.ag_suite_result1 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_test_suite1, submission=self.submission) # type: ag_models.AGTestSuiteResult self.ag_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_suite_result=self.ag_suite_result1, ag_test_case=self.ag_test_case1 ) # type: ag_models.AGTestCaseResult self.ag_cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_case_result1) self.ag_suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_test_suite2, submission=self.submission) # type: ag_models.AGTestSuiteResult self.ag_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_suite_result=self.ag_suite_result2, ag_test_case=self.ag_test_case2 ) # type: ag_models.AGTestCaseResult self.ag_cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_case_result2) self.num_student_tests = 6 self.student_tests = [ 'test{}'.format(i) for i in range(self.num_student_tests) ] self.student_suite_result1 = ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.submission, student_tests=self.student_tests, bugs_exposed=self.student_suite1.buggy_impl_names ) # type: ag_models.StudentTestSuiteResult self.student_suite_result2 = ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.submission, student_tests=self.student_tests, bugs_exposed=self.student_suite2.buggy_impl_names ) # type: ag_models.StudentTestSuiteResult self.total_points_per_ag_suite = get_suite_fdbk( self.ag_suite_result1, ag_models.FeedbackCategory.max).total_points self.total_points_per_student_suite = self.num_buggy_impls * self.points_per_bug_exposed self.total_points = (self.total_points_per_ag_suite * 2 + self.total_points_per_student_suite * 2) self.total_points_possible = self.total_points self.assertEqual( self.total_points_per_ag_suite, get_suite_fdbk(self.ag_suite_result2, ag_models.FeedbackCategory.max).total_points) self.assertEqual( self.total_points_per_student_suite, self.student_suite_result1.get_fdbk( ag_models.FeedbackCategory.max).total_points) print(self.total_points) self.assertNotEqual(0, self.total_points_per_ag_suite) self.assertNotEqual(0, self.total_points_per_student_suite) self.assertNotEqual(0, self.total_points) self.submission = update_denormalized_ag_test_results( self.submission.pk)