def test_some_commands_not_visible(self): self.ag_test_cmd1.validate_and_update(ultimate_submission_fdbk_config={'visible': False}) cmd1_pts = 5 cmd2_pts = 3 self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=cmd1_pts) self.ag_test_cmd2.validate_and_update(points_for_correct_return_code=cmd2_pts) cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( cmd1_pts + cmd2_pts, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points) self.assertEqual( cmd1_pts + cmd2_pts, get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points_possible) fdbk = get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission) self.assertSequenceEqual([cmd_result2.pk], [res.pk for res in fdbk.ag_test_command_results]) self.assertEqual( cmd2_pts, get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission).total_points) self.assertEqual( cmd2_pts, get_case_fdbk( self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission).total_points_possible)
def test_show_individual_commands(self): total_cmd_points = 6 self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=total_cmd_points) self.ag_test_cmd1.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_points': True } ) self.assertTrue(self.ag_test_case.normal_fdbk_config.show_individual_commands) result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertEqual([result1.pk, result2.pk], [res.pk for res in fdbk.ag_test_command_results]) self.assertEqual(total_cmd_points, fdbk.total_points) self.assertEqual(total_cmd_points, fdbk.total_points_possible) self.ag_test_case.validate_and_update( normal_fdbk_config={'show_individual_commands': False}) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal) self.assertEqual([], fdbk.ag_test_command_results) self.assertEqual(total_cmd_points, fdbk.total_points) self.assertEqual(total_cmd_points, fdbk.total_points_possible)
def test_no_failed_tests_all_get_normal_fdbk(self): self.case2_cmd_res.delete() self.case3_cmd_res.delete() ag_test_case2_correct_result = self.ag_test_case2_incorrect_result ag_test_case3_correct_result = self.ag_test_case3_incorrect_result self.case2_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case2_cmd, ag_test_case_result=ag_test_case2_correct_result) self.case3_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case3_cmd, ag_test_case_result=ag_test_case3_correct_result) fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.normal) expected_case_fdbks = [ get_case_fdbk(self.ag_test_case1_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(ag_test_case2_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(ag_test_case3_correct_result, ag_models.FeedbackCategory.normal).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_fdbk_to_dict(self): self.ag_test_case1.validate_and_update( normal_fdbk_config={'show_individual_commands': False}) expected_keys = [ 'pk', 'ag_test_suite_name', 'ag_test_suite_pk', 'fdbk_settings', 'setup_name', 'setup_return_code', 'setup_timed_out', 'total_points', 'total_points_possible', 'ag_test_case_results', ] for fdbk_category in ag_models.FeedbackCategory: result_dict = get_suite_fdbk(self.ag_test_suite_result, fdbk_category).to_dict() self.assertCountEqual(expected_keys, result_dict.keys()) self.assertCountEqual([ get_case_fdbk(self.ag_test_case_result1, fdbk_category).to_dict(), get_case_fdbk(self.ag_test_case_result2, fdbk_category).to_dict() ], result_dict['ag_test_case_results'])
def setUp(self): super().setUp() submission = obj_build.make_submission() self.project = submission.group.project suite1 = ag_models.AGTestSuite.objects.validate_and_create( name='kajsdhf', project=self.project) self.ag_test_case1 = ag_models.AGTestCase.objects.validate_and_create( name='aksdbva', ag_test_suite=suite1) self.ag_test_case2 = ag_models.AGTestCase.objects.validate_and_create( name='noniresta', ag_test_suite=suite1) suite_result = ag_models.AGTestSuiteResult.objects.validate_and_create( submission=submission, ag_test_suite=suite1) self.ag_test_case1_correct_result = ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case1, ag_test_suite_result=suite_result) self.ag_test_case2_incorrect_result = ( ag_models.AGTestCaseResult.objects.validate_and_create( ag_test_case=self.ag_test_case2, ag_test_suite_result=suite_result) ) self.ag_test_case1_cmd = obj_build.make_full_ag_test_command( self.ag_test_case1, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config()) ) self.total_points_possible = (self.ag_test_case1_cmd.points_for_correct_return_code + self.ag_test_case1_cmd.points_for_correct_stdout + self.ag_test_case1_cmd.points_for_correct_stderr) self.ag_test_case2_cmd = obj_build.make_full_ag_test_command( self.ag_test_case2, first_failed_test_normal_fdbk_config=( ag_models.NewAGTestCommandFeedbackConfig.max_fdbk_config()) ) self.case1_cmd_res = obj_build.make_correct_ag_test_command_result( self.ag_test_case1_cmd, ag_test_case_result=self.ag_test_case1_correct_result) self.case2_cmd_res = obj_build.make_incorrect_ag_test_command_result( self.ag_test_case2_cmd, ag_test_case_result=self.ag_test_case2_incorrect_result) case1_max_fdbk = get_case_fdbk( self.ag_test_case1_correct_result, ag_models.FeedbackCategory.max) self.assertEqual(self.total_points_possible, case1_max_fdbk.total_points) self.assertEqual(self.total_points_possible, case1_max_fdbk.total_points_possible) case2_max_fdbk = get_case_fdbk( self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.max) self.assertEqual(0, case2_max_fdbk.total_points) self.assertEqual(self.total_points_possible, case2_max_fdbk.total_points_possible)
def test_total_points_some_positive_some_negative_positive_total(self): self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=5) self.ag_test_cmd2.validate_and_update(deduction_for_wrong_stdout=-2) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result) obj_build.make_incorrect_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( 3, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points) self.assertEqual( 5, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points_possible)
def test_total_points_all_positive(self): # make sure sum is correct and takes into account deductions # make sure sum doesn't go below zero self.ag_test_cmd1.validate_and_update(points_for_correct_return_code=5) self.ag_test_cmd2.validate_and_update(points_for_correct_stdout=3) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( 8, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points) self.assertEqual( 8, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).total_points_possible)
def test_ag_test_cmd_result_ordering(self): cmd_result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, ag_test_case_result=self.ag_test_case_result) cmd_result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, ag_test_case_result=self.ag_test_case_result) for i in range(2): self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd2.pk, self.ag_test_cmd1.pk]) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertSequenceEqual([cmd_result2.pk, cmd_result1.pk], [res.pk for res in fdbk.ag_test_command_results]) self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd1.pk, self.ag_test_cmd2.pk]) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertSequenceEqual([cmd_result1.pk, cmd_result2.pk], [res.pk for res in fdbk.ag_test_command_results])
def test_ag_case_is_not_first_failure_gets_normal_feedback(self): correct_res1_fdbk = get_case_fdbk( self.ag_test_case1_correct_result, ag_models.FeedbackCategory.normal, is_first_failure=False ) self.assertEqual(0, correct_res1_fdbk.total_points) self.assertEqual(0, correct_res1_fdbk.total_points_possible)
def test_ag_case_is_first_failure_override_normal_feedback(self): incorrect_res2_fdbk = get_case_fdbk( self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.normal, is_first_failure=True ) self.assertEqual(0, incorrect_res2_fdbk.total_points) self.assertEqual(self.total_points_possible, incorrect_res2_fdbk.total_points_possible)
def test_first_failed_test_gets_overriden_normal_feedback(self): fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.normal) expected_case_fdbks = [ get_case_fdbk(self.ag_test_case1_correct_result, ag_models.FeedbackCategory.normal).to_dict(), get_case_fdbk(self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.normal, is_first_failure=True).to_dict(), get_case_fdbk(self.ag_test_case3_incorrect_result, ag_models.FeedbackCategory.normal).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_non_normal_fdbk_no_override(self): correct_res1_fdbk = get_case_fdbk( self.ag_test_case1_correct_result, ag_models.FeedbackCategory.past_limit_submission, is_first_failure=False ) self.assertEqual(0, correct_res1_fdbk.total_points) self.assertEqual(0, correct_res1_fdbk.total_points_possible) incorrect_res2_fdbk = get_case_fdbk( self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.past_limit_submission, is_first_failure=True ) self.assertEqual(0, incorrect_res2_fdbk.total_points) self.assertEqual(0, incorrect_res2_fdbk.total_points_possible)
def test_feedback_calculator_ctor(self): self.assertEqual( self.ag_test_case.normal_fdbk_config.to_dict(), get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_case.ultimate_submission_fdbk_config.to_dict(), get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.ultimate_submission).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_case.past_limit_submission_fdbk_config.to_dict(), get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.past_limit_submission).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_case.staff_viewer_fdbk_config.to_dict(), get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.staff_viewer).fdbk_conf.to_dict()) max_config = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max).fdbk_conf self.assertTrue(max_config.show_individual_commands)
def test_total_points_minimum_ag_test_command_fdbk(self): self.ag_test_cmd1.validate_and_update( points_for_correct_return_code=1, points_for_correct_stdout=2, points_for_correct_stderr=3) self.ag_test_cmd2.validate_and_update( deduction_for_wrong_return_code=-4, deduction_for_wrong_stdout=-2, deduction_for_wrong_stderr=-1) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd1, self.ag_test_case_result) obj_build.make_correct_ag_test_command_result(self.ag_test_cmd2, self.ag_test_case_result) self.assertEqual( 0, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal).total_points) self.assertEqual( 0, get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.normal).total_points_possible)
def test_cmd_result_query_order(self): result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) self.ag_test_case.set_agtestcommand_order([self.ag_test_cmd2.pk, self.ag_test_cmd1.pk]) self.ag_test_case_result = ag_models.AGTestCaseResult.objects.get( pk=self.ag_test_case_result.pk) fdbk = get_case_fdbk(self.ag_test_case_result, ag_models.FeedbackCategory.max) self.assertEqual([result2.pk, result1.pk], [res.pk for res in fdbk.ag_test_command_results])
def test_non_normal_fdbk_no_override(self): fdbk = get_suite_fdbk(self.ag_suite1_result, ag_models.FeedbackCategory.past_limit_submission) self.assertEqual(0, fdbk.total_points) self.assertEqual(0, fdbk.total_points_possible) expected_case_fdbks = [ get_case_fdbk( self.ag_test_case1_correct_result, ag_models.FeedbackCategory.past_limit_submission).to_dict(), get_case_fdbk( self.ag_test_case2_incorrect_result, ag_models.FeedbackCategory.past_limit_submission).to_dict(), get_case_fdbk( self.ag_test_case3_incorrect_result, ag_models.FeedbackCategory.past_limit_submission).to_dict() ] self.assertEqual( expected_case_fdbks, [case_fdbk.to_dict() for case_fdbk in fdbk.ag_test_case_results])
def test_fdbk_to_dict(self): self.maxDiff = None self.ag_test_cmd1.validate_and_update( points_for_correct_return_code=2, points_for_correct_stdout=3, points_for_correct_stderr=8 ) self.ag_test_cmd2.validate_and_update( points_for_correct_return_code=4, points_for_correct_stdout=5, points_for_correct_stderr=7 ) result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) expected_keys = [ 'pk', 'ag_test_case_name', 'ag_test_case_pk', 'fdbk_settings', 'total_points', 'total_points_possible', 'ag_test_command_results', ] for fdbk_category in ag_models.FeedbackCategory: result_dict = get_case_fdbk(self.ag_test_case_result, fdbk_category).to_dict() self.assertCountEqual(expected_keys, result_dict.keys()) self.assertCountEqual( [get_cmd_fdbk(result1, fdbk_category).to_dict(), get_cmd_fdbk(result2, fdbk_category).to_dict()], result_dict['ag_test_command_results'])