def test_individual_suite_result_order(self): self.project.set_agtestsuite_order( [self.ag_test_suite2.pk, self.ag_test_suite1.pk]) self.project.set_studenttestsuite_order( [self.student_suite2.pk, self.student_suite1.pk]) self.submission = get_submissions_with_results_queryset().get( pk=self.submission.pk) fdbk = get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max) self.assertSequenceEqual( [self.ag_suite_result2.pk, self.ag_suite_result1.pk], [res.pk for res in fdbk.ag_test_suite_results]) self.assertSequenceEqual([ get_suite_fdbk(self.ag_suite_result2, ag_models.FeedbackCategory.max).to_dict(), get_suite_fdbk(self.ag_suite_result1, ag_models.FeedbackCategory.max).to_dict() ], fdbk.to_dict()['ag_test_suite_results']) self.assertSequenceEqual( [self.student_suite_result2, self.student_suite_result1], fdbk.student_test_suite_results) self.assertSequenceEqual([ self.student_suite_result2.get_fdbk( ag_models.FeedbackCategory.max).to_dict(), self.student_suite_result1.get_fdbk( ag_models.FeedbackCategory.max).to_dict() ], fdbk.to_dict()['student_test_suite_results'])
def test_fdbk_to_dict(self): expected = { 'pk': self.submission.pk, 'total_points': str(self.total_points.quantize(Decimal('.01'))), 'total_points_possible': str(self.total_points.quantize(Decimal('.01'))), 'ag_test_suite_results': [ get_suite_fdbk(self.ag_suite_result1, ag_models.FeedbackCategory.max).to_dict(), get_suite_fdbk(self.ag_suite_result2, ag_models.FeedbackCategory.max).to_dict() ], 'student_test_suite_results': [ self.student_suite_result1.get_fdbk( ag_models.FeedbackCategory.max).to_dict(), self.student_suite_result2.get_fdbk( ag_models.FeedbackCategory.max).to_dict(), ] } actual = get_submission_fdbk(self.submission, ag_models.FeedbackCategory.max).to_dict() print(json.dumps(actual, indent=4, sort_keys=True)) self.assertEqual(expected, actual)
def test_show_individual_tests(self): self.ag_test_cmd1.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True }) self.ag_test_cmd2.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True }) self.assertTrue( self.ag_test_suite.normal_fdbk_config.show_individual_tests) fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.normal) self.assertEqual( [self.ag_test_case_result1.pk, self.ag_test_case_result2.pk], [res.pk for res in fdbk.ag_test_case_results]) self.assertEqual(self.total_points, fdbk.total_points) self.assertEqual(self.total_points, fdbk.total_points_possible) self.ag_test_suite.validate_and_update( normal_fdbk_config={'show_individual_tests': False}) fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.normal) self.assertEqual([], fdbk.ag_test_case_results) self.assertEqual(self.total_points, fdbk.total_points) self.assertEqual(self.total_points, fdbk.total_points_possible)
def test_feedback_calculator_ctor(self): self.assertEqual( self.ag_test_suite.normal_fdbk_config.to_dict(), get_suite_fdbk( self.ag_test_suite_result, ag_models.FeedbackCategory.normal).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_suite.ultimate_submission_fdbk_config.to_dict(), get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.ultimate_submission). fdbk_conf.to_dict()) self.assertEqual( self.ag_test_suite.past_limit_submission_fdbk_config.to_dict(), get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.past_limit_submission). fdbk_conf.to_dict()) self.assertEqual( self.ag_test_suite.staff_viewer_fdbk_config.to_dict(), get_suite_fdbk( self.ag_test_suite_result, ag_models.FeedbackCategory.staff_viewer).fdbk_conf.to_dict()) max_config = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.max).fdbk_conf self.assertTrue(max_config.show_individual_tests) self.assertTrue(max_config.show_setup_stdout) self.assertTrue(max_config.show_setup_stderr)
def test_no_failed_tests_all_get_normal_fdbk(self): self.case2_cmd_res.delete() self.case3_cmd_res.delete() ag_test_case2_correct_result = self.ag_test_case2_incorrect_result ag_test_case3_correct_result = self.ag_test_case3_incorrect_result self.case2_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case2_cmd, ag_test_case_result=ag_test_case2_correct_result) self.case3_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case3_cmd, ag_test_case_result=ag_test_case3_correct_result) fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.normal) expected_case_fdbks = [ get_case_fdbk(self.ag_test_case1_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(ag_test_case2_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(ag_test_case3_correct_result, ag_models.FeedbackCategory.normal).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_fdbk_to_dict(self): self.ag_test_case1.validate_and_update( normal_fdbk_config={'show_individual_commands': False}) expected_keys = [ 'pk', 'ag_test_suite_name', 'ag_test_suite_pk', 'fdbk_settings', 'setup_name', 'setup_return_code', 'setup_timed_out', 'total_points', 'total_points_possible', 'ag_test_case_results', ] for fdbk_category in ag_models.FeedbackCategory: result_dict = get_suite_fdbk(self.ag_test_suite_result, fdbk_category).to_dict() self.assertCountEqual(expected_keys, result_dict.keys()) self.assertCountEqual([ get_case_fdbk(self.ag_test_case_result1, fdbk_category).to_dict(), get_case_fdbk(self.ag_test_case_result2, fdbk_category).to_dict() ], result_dict['ag_test_case_results'])
def test_some_ag_and_student_test_suites_not_visible(self): self.ag_test_suite2.validate_and_update( ultimate_submission_fdbk_config={'visible': False}) self.student_suite2.validate_and_update( ultimate_submission_fdbk_config={'visible': False}) fdbk = get_submission_fdbk( self.submission, ag_models.FeedbackCategory.ultimate_submission) self.assertEqual( self.total_points_per_ag_suite + self.total_points_per_student_suite, fdbk.total_points) self.assertEqual( self.total_points_per_ag_suite + self.total_points_per_student_suite, fdbk.total_points_possible) self.assertSequenceEqual( [self.ag_suite_result1.pk], [res.pk for res in fdbk.ag_test_suite_results]) self.assertSequenceEqual([ get_suite_fdbk( self.ag_suite_result1, ag_models.FeedbackCategory.ultimate_submission).to_dict() ], fdbk.to_dict()['ag_test_suite_results']) self.assertSequenceEqual([self.student_suite_result1], fdbk.student_test_suite_results) self.assertSequenceEqual([ self.student_suite_result1.get_fdbk( ag_models.FeedbackCategory.ultimate_submission).to_dict() ], fdbk.to_dict()['student_test_suite_results'])
def test_individual_test_result_order(self): self.ag_test_suite.set_agtestcase_order( [self.ag_test_case2.pk, self.ag_test_case1.pk]) fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.max) self.assertEqual( [self.ag_test_case_result2.pk, self.ag_test_case_result1.pk], [res.pk for res in fdbk.ag_test_case_results])
def test_ag_test_case_result_ordering(self): for i in range(2): self.ag_test_suite.set_agtestcase_order( [self.ag_test_case2.pk, self.ag_test_case1.pk]) fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.max) print(fdbk) self.assertSequenceEqual( [self.ag_test_case_result2.pk, self.ag_test_case_result1.pk], [res.pk for res in fdbk.ag_test_case_results]) self.ag_test_suite.set_agtestcase_order( [self.ag_test_case1.pk, self.ag_test_case2.pk]) fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.max) self.assertSequenceEqual( [self.ag_test_case_result1.pk, self.ag_test_case_result2.pk], [res.pk for res in fdbk.ag_test_case_results])
def test_some_ag_test_cases_not_visible(self): self.ag_test_case2.validate_and_update( ultimate_submission_fdbk_config={'visible': False}) expected_points = get_cmd_fdbk( self.cmd_result1, ag_models.FeedbackCategory.max).total_points_possible fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.ultimate_submission) self.assertSequenceEqual([self.ag_test_case_result1.pk], [res.pk for res in fdbk.ag_test_case_results]) self.assertEqual(expected_points, fdbk.total_points) self.assertEqual(expected_points, fdbk.total_points_possible)
def test_show_setup_output_return_code_and_timed_out(self): setup_return_code = 3 setup_timed_out = True setup_stdout = 'adfjka;dskjf' setup_stderr = 'a,xcmvnaieo;sdf' self.ag_test_suite_result.setup_return_code = setup_return_code self.ag_test_suite_result.setup_timed_out = setup_timed_out with self.ag_test_suite_result.open_setup_stdout('w') as f: f.write(setup_stdout) with self.ag_test_suite_result.open_setup_stderr('w') as f: f.write(setup_stderr) self.ag_test_suite_result.save() fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.max) self.assertEqual(self.ag_test_suite.setup_suite_cmd_name, fdbk.setup_name) self.assertEqual(setup_return_code, fdbk.setup_return_code) self.assertEqual(setup_timed_out, fdbk.setup_timed_out) self.assertEqual(setup_stdout, fdbk.setup_stdout.read().decode()) self.assertEqual(setup_stderr, fdbk.setup_stderr.read().decode()) self.ag_test_suite.validate_and_update( normal_fdbk_config={ 'show_setup_return_code': False, 'show_setup_timed_out': False, 'show_setup_stdout': False, 'show_setup_stderr': False }) fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.normal) self.assertIsNone(fdbk.setup_name) self.assertIsNone(fdbk.setup_return_code) self.assertIsNone(fdbk.setup_timed_out) self.assertIsNone(fdbk.setup_stdout) self.assertIsNone(fdbk.setup_stderr)
def test_first_failed_test_gets_overriden_normal_feedback(self): fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.normal) expected_case_fdbks = [ get_case_fdbk(self.ag_test_case1_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.normal, is_first_failure=True).to_dict(), get_case_fdbk(self.ag_test_case3_incorrect_result, ag_models.FeedbackCategory.normal).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_non_normal_fdbk_no_override(self): fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.past_limit_submission) self.assertEqual(0, fdbk.total_points) self.assertEqual(0, fdbk.total_points_possible) expected_case_fdbks = [ get_case_fdbk( self.ag_test_case1_correct_result, ag_models.FeedbackCategory.past_limit_submission).to_dict(), get_case_fdbk( self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.past_limit_submission).to_dict(), get_case_fdbk( self.ag_test_case3_incorrect_result, ag_models.FeedbackCategory.past_limit_submission).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def setUp(self): super().setUp() submission = obj_build.make_submission() self.project = submission.group.project self.ag_suite1 = ag_models.AGTestSuite.objects.validate_and_create( name='kajsdhf', project=self.project) self.ag_suite1_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission, ag_test_suite=self.ag_suite1) self.ag_test_case1 = ag_models.AGTestCase.objects.validate_and_create( name='aksdbva', ag_test_suite=self.ag_suite1) self.ag_test_case2 = ag_models.AGTestCase.objects.validate_and_create( name='noniresta', ag_test_suite=self.ag_suite1) self.ag_test_case3 = ag_models.AGTestCase.objects.validate_and_create( name='eaoneastno', ag_test_suite=self.ag_suite1) self.ag_test_case1_correct_result = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case1, ag_test_suite_result=self.ag_suite1_result) self.ag_test_case2_incorrect_result = ( ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case2, ag_test_suite_result=self.ag_suite1_result)) self.ag_test_case3_incorrect_result = ( ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case3, ag_test_suite_result=self.ag_suite1_result)) self.ag_test_case1_cmd = obj_build.make_full_ag_test_command( self.ag_test_case1, normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True }, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config())) self.total_points_possible = ( self.ag_test_case1_cmd.points_for_correct_return_code + self.ag_test_case1_cmd.points_for_correct_stdout + self.ag_test_case1_cmd.points_for_correct_stderr) self.ag_test_case2_cmd = obj_build.make_full_ag_test_command( self.ag_test_case2, normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True }, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config())) self.ag_test_case3_cmd = obj_build.make_full_ag_test_command( self.ag_test_case3, normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True }, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config())) self.case1_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case1_cmd, ag_test_case_result=self.ag_test_case1_correct_result) self.case2_cmd_res = obj_build.make_incorrect_ag_test_command_result( self.ag_test_case2_cmd, ag_test_case_result=self.ag_test_case2_incorrect_result) self.case3_cmd_res = obj_build.make_incorrect_ag_test_command_result( self.ag_test_case3_cmd, ag_test_case_result=self.ag_test_case3_incorrect_result) max_fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.max) self.assertEqual(self.total_points_possible, max_fdbk.total_points) self.assertEqual(self.total_points_possible * 3, max_fdbk.total_points_possible)
def test_points_minimum_ag_test_command_fdbk(self): fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.normal) self.assertEqual(0, fdbk.total_points) self.assertEqual(0, fdbk.total_points_possible)
def test_points_max_fdbk(self): fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.max) self.assertEqual(self.total_points, fdbk.total_points) self.assertEqual(self.total_points, fdbk.total_points_possible)
def test_move_ag_test_case_to_different_suite_in_same_project(self): other_suite = ag_models.AGTestSuite.objects.validate_and_create( name='fa;weifjawef', project=self.project) # type: ag_models.AGTestSuite ag_test = ag_models.AGTestCase.objects.validate_and_create( name='asdlkfjaewi;ojf', ag_test_suite=self.ag_suite) # type: ag_models.AGTestCase cmd = ag_models.AGTestCommand.objects.validate_and_create( name='asdfknja;wej', ag_test_case=ag_test, cmd='asdklfja;sdjkfaldsf') # type: ag_models.AGTestCommand # This group has results for ONLY self.ag_suite submission = obj_build.make_submission(group=obj_build.build_group( group_kwargs={'project': self.project})) suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission, ag_test_suite=self.ag_suite) # type: ag_models.AGTestSuiteResult ag_test_result = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=ag_test, ag_test_suite_result=suite_result ) # type: ag_models.AGTestCaseResult cmd_result = obj_build.make_correct_ag_test_command_result( cmd, ag_test_result) # This group has results for self.ag_suite and other_suite submission2 = obj_build.make_submission(group=obj_build.build_group( group_kwargs={'project': self.project})) suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission2, ag_test_suite=self.ag_suite) # type: ag_models.AGTestSuiteResult ag_test_result2 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=ag_test, ag_test_suite_result=suite_result2 ) # type: ag_models.AGTestCaseResult cmd_result2 = obj_build.make_correct_ag_test_command_result( cmd, ag_test_result2) other_suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission2, ag_test_suite=other_suite) # type: ag_models.AGTestSuiteResult self.assertEqual(self.ag_suite, ag_test.ag_test_suite) # This that should be re-wired after this line: # - ag_test should belong to other_suite # - AGTestCaseResults should point to suite results in their # own submissions that belong to other_suite (creating them # if necessary). ag_test.validate_and_update(ag_test_suite=other_suite) other_suite = ag_models.AGTestSuite.objects.get(pk=other_suite.pk) self.assertSequenceEqual([ag_test], other_suite.ag_test_cases.all()) self.ag_suite.refresh_from_db() self.assertSequenceEqual([], self.ag_suite.ag_test_cases.all()) # An AGTestSuiteResult for other_suite should have been created for submission. self.assertEqual(2, submission.ag_test_suite_results.count()) self.assertEqual(suite_result, submission.ag_test_suite_results.first()) new_other_suite_result = submission.ag_test_suite_results.last() self.assertEqual(other_suite, new_other_suite_result.ag_test_suite) # ag_test should belong to other_suite self.assertEqual(other_suite, ag_test.ag_test_suite) # ag_test_result should belong to new_other_suite_result ag_test_result.refresh_from_db() self.assertEqual(new_other_suite_result, ag_test_result.ag_test_suite_result) # ag_test_result2 should belong to other_suite_result ag_test_result2.refresh_from_db() self.assertEqual(other_suite_result, ag_test_result2.ag_test_suite_result) # Make sure no errors are thrown get_suite_fdbk(suite_result, ag_models.FeedbackCategory.max).ag_test_case_results get_suite_fdbk(suite_result2, ag_models.FeedbackCategory.max).ag_test_case_results get_suite_fdbk(new_other_suite_result, ag_models.FeedbackCategory.max).ag_test_case_results
def setUp(self): super().setUp() self.maxDiff = None self.project = obj_build.make_project() self.course = self.project.course self.ag_test_suite1 = obj_build.make_ag_test_suite(self.project) self.ag_test_case1 = obj_build.make_ag_test_case(self.ag_test_suite1) self.ag_test_cmd1 = obj_build.make_full_ag_test_command( self.ag_test_case1, set_arbitrary_points=True) self.ag_test_suite2 = obj_build.make_ag_test_suite(self.project) self.ag_test_case2 = obj_build.make_ag_test_case(self.ag_test_suite2) self.ag_test_cmd2 = obj_build.make_full_ag_test_command( self.ag_test_case2, set_arbitrary_points=True) self.points_per_bug_exposed = Decimal('3.5') self.num_buggy_impls = 4 self.student_suite1 = ag_models.StudentTestSuite.objects.validate_and_create( name='suite1', project=self.project, buggy_impl_names=[ 'bug{}'.format(i) for i in range(self.num_buggy_impls) ], points_per_exposed_bug=self.points_per_bug_exposed ) # type: ag_models.StudentTestSuite self.student_suite2 = ag_models.StudentTestSuite.objects.validate_and_create( name='suite2', project=self.project, buggy_impl_names=[ 'bug{}'.format(i) for i in range(self.num_buggy_impls) ], points_per_exposed_bug=self.points_per_bug_exposed ) # type: ag_models.StudentTestSuite self.group = obj_build.make_group(num_members=1, project=self.project) self.submission = obj_build.make_submission(group=self.group) self.ag_suite_result1 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_test_suite1, submission=self.submission) # type: ag_models.AGTestSuiteResult self.ag_case_result1 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_suite_result=self.ag_suite_result1, ag_test_case=self.ag_test_case1 ) # type: ag_models.AGTestCaseResult self.ag_cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_case_result1) self.ag_suite_result2 = ag_models.AGTestSuiteResult.objects.validate_and_create( ag_test_suite=self.ag_test_suite2, submission=self.submission) # type: ag_models.AGTestSuiteResult self.ag_case_result2 = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_suite_result=self.ag_suite_result2, ag_test_case=self.ag_test_case2 ) # type: ag_models.AGTestCaseResult self.ag_cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_case_result2) self.num_student_tests = 6 self.student_tests = [ 'test{}'.format(i) for i in range(self.num_student_tests) ] self.student_suite_result1 = ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite1, submission=self.submission, student_tests=self.student_tests, bugs_exposed=self.student_suite1.buggy_impl_names ) # type: ag_models.StudentTestSuiteResult self.student_suite_result2 = ag_models.StudentTestSuiteResult.objects.validate_and_create( student_test_suite=self.student_suite2, submission=self.submission, student_tests=self.student_tests, bugs_exposed=self.student_suite2.buggy_impl_names ) # type: ag_models.StudentTestSuiteResult self.total_points_per_ag_suite = get_suite_fdbk( self.ag_suite_result1, ag_models.FeedbackCategory.max).total_points self.total_points_per_student_suite = self.num_buggy_impls * self.points_per_bug_exposed self.total_points = (self.total_points_per_ag_suite * 2 + self.total_points_per_student_suite * 2) self.total_points_possible = self.total_points self.assertEqual( self.total_points_per_ag_suite, get_suite_fdbk(self.ag_suite_result2, ag_models.FeedbackCategory.max).total_points) self.assertEqual( self.total_points_per_student_suite, self.student_suite_result1.get_fdbk( ag_models.FeedbackCategory.max).total_points) print(self.total_points) self.assertNotEqual(0, self.total_points_per_ag_suite) self.assertNotEqual(0, self.total_points_per_student_suite) self.assertNotEqual(0, self.total_points) self.submission = update_denormalized_ag_test_results( self.submission.pk)