def test_stderr_correctness_show_expected_and_actual(self): self.ag_test_command.validate_and_update(normal_fdbk_config={ 'stderr_fdbk_level': ag_models.ValueFeedbackLevel.expected_and_actual }) correct_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal) self.assertTrue(fdbk.stderr_correct) diff = _get_expected_diff(self.ag_test_command.expected_stderr_text, correct_result.stderr_filename) self.assertEqual(diff.diff_content, fdbk.stderr_diff.diff_content) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points_possible) correct_result.delete() incorrect_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_result, ag_models.FeedbackCategory.normal) self.assertFalse(fdbk.stderr_correct) diff = _get_expected_diff(self.ag_test_command.expected_stderr_text, incorrect_result.stderr_filename) self.assertEqual(diff.diff_content, fdbk.stderr_diff.diff_content) self.assertEqual(self.ag_test_command.deduction_for_wrong_stderr, fdbk.stderr_points) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points_possible)
def test_stderr_correctness_show_correct_or_incorrect(self): self.ag_test_command.validate_and_update(normal_fdbk_config={ 'stderr_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect }) correct_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal) self.assertTrue(fdbk.stderr_correct) self.assertIsNone(fdbk.stderr_diff) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points_possible) correct_result.delete() incorrect_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_result, ag_models.FeedbackCategory.normal) self.assertFalse(fdbk.stderr_correct) self.assertEqual(self.ag_test_command.deduction_for_wrong_stderr, fdbk.stderr_points) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points_possible)
def test_stderr_correctness_show_diff_from_file(self): instructor_file = obj_build.make_instructor_file(self.project) self.ag_test_command.expected_stderr_source = ( ag_models.ExpectedOutputSource.instructor_file) self.ag_test_command.expected_stderr_instructor_file = instructor_file self.ag_test_command.save() result = self.make_correct_result() with instructor_file.open() as f: expected_stderr = f.read() result.stderr = expected_stderr result.save() diff = _get_expected_diff(expected_stderr, result.stderr_filename) self.assertEqual( diff.diff_content, get_cmd_fdbk( result, ag_models.FeedbackCategory.normal).stderr_diff.diff_content) result.stderr = 'the wrong stderr' result.save() diff = _get_expected_diff(expected_stderr, result.stderr_filename) self.assertEqual( diff.diff_content, get_cmd_fdbk( result, ag_models.FeedbackCategory.normal).stderr_diff.diff_content)
def test_return_code_correctness_show_expected_and_actual(self): self.ag_test_command.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.expected_and_actual }) correct_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal) self.assertTrue(fdbk.return_code_correct) self.assertEqual(self.ag_test_command.expected_return_code, fdbk.expected_return_code) self.assertEqual(correct_result.return_code, fdbk.actual_return_code) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points_possible) correct_result.delete() incorrect_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_result, ag_models.FeedbackCategory.normal) self.assertFalse(fdbk.return_code_correct) self.assertEqual(self.ag_test_command.expected_return_code, fdbk.expected_return_code) self.assertEqual(incorrect_result.return_code, fdbk.actual_return_code) self.assertEqual(self.ag_test_command.deduction_for_wrong_return_code, fdbk.return_code_points) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points_possible)
def test_stderr_not_checked(self): self.ag_test_command.validate_and_update( expected_stderr_source=ag_models.ExpectedOutputSource.none) correct_cmd_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_cmd_result, ag_models.FeedbackCategory.max) self.assertIsNone(fdbk.stderr_correct) self.assertIsNone(fdbk.stderr_diff) self.assertEqual(_stderr_text(correct_cmd_result), _stderr_text(fdbk)) self.assertEqual(0, fdbk.stderr_points) self.assertEqual(0, fdbk.stderr_points_possible) expected_total_pts = ( self.ag_test_command.points_for_correct_return_code + self.ag_test_command.points_for_correct_stdout) expected_total_pts_possible = expected_total_pts self.assertEqual(expected_total_pts, fdbk.total_points) self.assertEqual(expected_total_pts_possible, fdbk.total_points_possible) correct_cmd_result.delete() incorrect_cmd_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_cmd_result, ag_models.FeedbackCategory.max) self.assertIsNone(fdbk.stderr_correct) self.assertEqual(0, fdbk.stderr_points) self.assertEqual(0, fdbk.stderr_points_possible) expected_total_pts = ( self.ag_test_command.deduction_for_wrong_return_code + self.ag_test_command.deduction_for_wrong_stdout) self.assertEqual(expected_total_pts, fdbk.total_points) self.assertEqual(expected_total_pts_possible, fdbk.total_points_possible)
def test_return_code_not_checked(self): self.ag_test_command.validate_and_update( expected_return_code=ag_models.ExpectedReturnCode.none) correct_cmd_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_cmd_result, ag_models.FeedbackCategory.max) self.assertEqual(ag_models.ExpectedReturnCode.none, fdbk.expected_return_code) self.assertIsNone(fdbk.return_code_correct) self.assertEqual(0, fdbk.return_code_points) self.assertEqual(0, fdbk.return_code_points_possible) expected_total_pts = (self.ag_test_command.points_for_correct_stdout + self.ag_test_command.points_for_correct_stderr) self.assertEqual(expected_total_pts, fdbk.total_points) expected_total_pts_possible = expected_total_pts self.assertEqual(expected_total_pts_possible, fdbk.total_points_possible) correct_cmd_result.delete() incorrect_cmd_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_cmd_result, ag_models.FeedbackCategory.max) self.assertIsNone(fdbk.return_code_correct) self.assertEqual(0, fdbk.return_code_points) self.assertEqual(0, fdbk.return_code_points_possible) expected_total_pts = (self.ag_test_command.deduction_for_wrong_stdout + self.ag_test_command.deduction_for_wrong_stderr) self.assertEqual(expected_total_pts, fdbk.total_points) self.assertEqual(expected_total_pts_possible, fdbk.total_points_possible)
def test_feedback_calculator_factory_method(self): # check against the actual objects (their pks) result = self.make_correct_result() self.assertEqual( self.ag_test_command.normal_fdbk_config.to_dict(), get_cmd_fdbk( result, ag_models.FeedbackCategory.normal).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_command.ultimate_submission_fdbk_config.to_dict(), get_cmd_fdbk(result, ag_models.FeedbackCategory.ultimate_submission ).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_command.past_limit_submission_fdbk_config.to_dict(), get_cmd_fdbk(result, ag_models.FeedbackCategory. past_limit_submission).fdbk_conf.to_dict()) self.assertEqual( self.ag_test_command.staff_viewer_fdbk_config.to_dict(), get_cmd_fdbk( result, ag_models.FeedbackCategory.staff_viewer).fdbk_conf.to_dict()) max_fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.max).fdbk_conf self.assertEqual(ag_models.ValueFeedbackLevel.get_max(), max_fdbk.return_code_fdbk_level) self.assertEqual(ag_models.ValueFeedbackLevel.get_max(), max_fdbk.stdout_fdbk_level) self.assertEqual(ag_models.ValueFeedbackLevel.get_max(), max_fdbk.stderr_fdbk_level) self.assertTrue(max_fdbk.show_points) self.assertTrue(max_fdbk.show_actual_return_code) self.assertTrue(max_fdbk.show_actual_stdout) self.assertTrue(max_fdbk.show_actual_stderr) self.assertTrue(max_fdbk.show_whether_timed_out)
def test_show_timed_out(self): result = self.make_correct_result() self.assertFalse( get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out) result.delete() result = self.make_incorrect_result() self.assertTrue( get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out)
def test_hide_timed_out(self): self.ag_test_command.validate_and_update( normal_fdbk_config={'show_whether_timed_out': False}) result = self.make_correct_result() self.assertIsNone( get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out) result.delete() result = self.make_incorrect_result() self.assertIsNone( get_cmd_fdbk(result, ag_models.FeedbackCategory.normal).timed_out)
def test_fdbk_to_dict(self): result = obj_build.make_correct_ag_test_command_result( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result) expected_keys = [ 'pk', 'ag_test_command_name', 'ag_test_command_pk', 'fdbk_settings', 'timed_out', 'return_code_correct', 'expected_return_code', 'actual_return_code', 'return_code_points', 'return_code_points_possible', 'stdout_correct', 'stdout_points', 'stdout_points_possible', 'stderr_correct', 'stderr_points', 'stderr_points_possible', 'total_points', 'total_points_possible', ] self.assertCountEqual( expected_keys, get_cmd_fdbk(result, ag_models.FeedbackCategory.max).to_dict().keys())
def test_points_visibility(self): self.ag_test_command.validate_and_update( normal_fdbk_config={'show_points': False}) correct_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal) self.assertEqual(0, fdbk.total_points) self.assertEqual(0, fdbk.total_points_possible) correct_result.delete() incorrect_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_result, ag_models.FeedbackCategory.normal) self.assertEqual(0, fdbk.total_points) self.assertEqual(0, fdbk.total_points_possible)
def test_timed_out_with_return_code_stdout_and_stderr_None_count_as_wrong( self): result = ag_models.AGTestCommandResult.objects.validate_and_create( ag_test_command=self.ag_test_command, ag_test_case_result=self.ag_test_case_result, timed_out=True) fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.max) self.assertFalse(fdbk.return_code_correct) self.assertFalse(fdbk.stdout_correct) self.assertFalse(fdbk.stderr_correct)
def test_return_code_correctness_hidden(self): self.ag_test_command.validate_and_update(normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.no_feedback }) correct_result = self.make_correct_result() fdbk = get_cmd_fdbk(correct_result, ag_models.FeedbackCategory.normal) self.assertIsNone(fdbk.return_code_correct) self.assertEqual(0, fdbk.return_code_points) self.assertEqual(0, fdbk.return_code_points_possible) correct_result.delete() incorrect_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(incorrect_result, ag_models.FeedbackCategory.normal) self.assertIsNone(fdbk.return_code_correct) self.assertEqual(0, fdbk.return_code_points) self.assertEqual(0, fdbk.return_code_points_possible)
def test_some_ag_test_cases_not_visible(self): self.ag_test_case2.validate_and_update( ultimate_submission_fdbk_config={'visible': False}) expected_points = get_cmd_fdbk( self.cmd_result1, ag_models.FeedbackCategory.max).total_points_possible fdbk = get_suite_fdbk(self.ag_test_suite_result, ag_models.FeedbackCategory.ultimate_submission) self.assertSequenceEqual([self.ag_test_case_result1.pk], [res.pk for res in fdbk.ag_test_case_results]) self.assertEqual(expected_points, fdbk.total_points) self.assertEqual(expected_points, fdbk.total_points_possible)
def test_stderr_show_actual(self): self.ag_test_command.validate_and_update( normal_fdbk_config={ 'stderr_fdbk_level': ag_models.ValueFeedbackLevel.no_feedback, 'show_actual_stderr': True }) result = self.make_correct_result() fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal) self.assertEqual(_stderr_text(result), _stderr_text(fdbk)) self.assertIsNone(fdbk.stderr_correct) self.assertEqual(0, fdbk.stderr_points) self.assertEqual(0, fdbk.stderr_points_possible)
def test_fdbk_to_dict(self): self.maxDiff = None self.ag_test_cmd1.validate_and_update( points_for_correct_return_code=2, points_for_correct_stdout=3, points_for_correct_stderr=8 ) self.ag_test_cmd2.validate_and_update( points_for_correct_return_code=4, points_for_correct_stdout=5, points_for_correct_stderr=7 ) result1 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd1, self.ag_test_case_result) result2 = obj_build.make_correct_ag_test_command_result( self.ag_test_cmd2, self.ag_test_case_result) expected_keys = [ 'pk', 'ag_test_case_name', 'ag_test_case_pk', 'fdbk_settings', 'total_points', 'total_points_possible', 'ag_test_command_results', ] for fdbk_category in ag_models.FeedbackCategory: result_dict = get_case_fdbk(self.ag_test_case_result, fdbk_category).to_dict() self.assertCountEqual(expected_keys, result_dict.keys()) self.assertCountEqual( [get_cmd_fdbk(result1, fdbk_category).to_dict(), get_cmd_fdbk(result2, fdbk_category).to_dict()], result_dict['ag_test_command_results'])
def test_stderr_show_expected_and_actual_overrides_show_actual(self): self.ag_test_command.validate_and_update( normal_fdbk_config={ 'stderr_fdbk_level': ag_models.ValueFeedbackLevel.expected_and_actual, 'show_actual_stderr': False }) result = self.make_correct_result() fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal) self.assertIsNotNone(fdbk.stderr_diff) self.assertIsNotNone(fdbk.stderr) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points)
def do_diff_options_test(self, expected_stdout='', actual_stdout='', expected_stderr='', actual_stderr='', expect_stdout_correct=True, expect_stderr_correct=True, **diff_options): self.ag_test_command.validate_and_update( expected_stdout_text=expected_stdout, expected_stderr_text=expected_stderr, **diff_options) result = self.make_correct_result() with open(result.stdout_filename, 'w') as f: f.write(actual_stdout) actual_stdout_filename = f.name with open(result.stderr_filename, 'w') as f: f.write(actual_stderr) actual_stderr_filename = f.name mock_path = 'autograder.core.utils.get_diff' with mock.patch(mock_path) as mock_differ_cls: diff = get_cmd_fdbk(result, ag_models.FeedbackCategory.max).stdout_diff mock_differ_cls.assert_called_with(mock.ANY, actual_stdout_filename, **diff_options) with mock.patch(mock_path) as mock_differ_cls: diff = get_cmd_fdbk(result, ag_models.FeedbackCategory.max).stderr_diff mock_differ_cls.assert_called_with(mock.ANY, actual_stderr_filename, **diff_options) if expect_stdout_correct: diff = get_cmd_fdbk(result, ag_models.FeedbackCategory.max).stdout_diff self.assertTrue(diff.diff_pass, msg=diff.diff_content) else: self.assertFalse( get_cmd_fdbk( result, ag_models.FeedbackCategory.max).stdout_diff.diff_pass) if expect_stderr_correct: self.assertTrue( get_cmd_fdbk( result, ag_models.FeedbackCategory.max).stderr_diff.diff_pass) else: self.assertFalse( get_cmd_fdbk( result, ag_models.FeedbackCategory.max).stderr_diff.diff_pass)
def test_return_code_show_actual(self): # Show actual return code, but not expected return code or points. self.ag_test_command.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.no_feedback, 'show_actual_return_code': True }) result = self.make_correct_result() fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal) self.assertEqual(result.return_code, fdbk.actual_return_code) self.assertIsNone(fdbk.return_code_correct) self.assertEqual(0, fdbk.return_code_points) self.assertEqual(0, fdbk.return_code_points_possible)
def test_return_code_show_expected_and_actual_overrides_show_actual(self): # Show expected and actual return code (overrides show_actual_return_code setting). self.ag_test_command.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.expected_and_actual, 'show_actual_return_code': False }) result = self.make_correct_result() fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal) self.assertIsNotNone(fdbk.actual_return_code) self.assertIsNotNone(fdbk.expected_return_code) self.assertTrue(fdbk.return_code_correct) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points)
def test_stderr_show_correctness_but_not_actual(self): self.ag_test_command.validate_and_update( normal_fdbk_config={ 'stderr_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_actual_stderr': False }) result = self.make_correct_result() fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal) self.assertIsNone(fdbk.stderr) self.assertTrue(fdbk.stderr_correct) self.assertIsNone(fdbk.stderr_diff) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points_possible)
def test_return_code_show_correctness_but_not_actual(self): # Show return code correctness and points, but not actual return code. self.ag_test_command.validate_and_update( normal_fdbk_config={ 'return_code_fdbk_level': ag_models.ValueFeedbackLevel.correct_or_incorrect, 'show_actual_return_code': False }) result = self.make_correct_result() fdbk = get_cmd_fdbk(result, ag_models.FeedbackCategory.normal) self.assertIsNone(fdbk.actual_return_code) self.assertIsNone(fdbk.expected_return_code) self.assertTrue(fdbk.return_code_correct) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points_possible)
def test_points_everything_incorrect_max_fdbk(self): cmd_result = self.make_incorrect_result() fdbk = get_cmd_fdbk(cmd_result, ag_models.FeedbackCategory.max) self.assertEqual(self.ag_test_command.deduction_for_wrong_return_code, fdbk.return_code_points) self.assertEqual(self.ag_test_command.points_for_correct_return_code, fdbk.return_code_points_possible) self.assertEqual(self.ag_test_command.deduction_for_wrong_stdout, fdbk.stdout_points) self.assertEqual(self.ag_test_command.points_for_correct_stdout, fdbk.stdout_points_possible) self.assertEqual(self.ag_test_command.deduction_for_wrong_stderr, fdbk.stderr_points) self.assertEqual(self.ag_test_command.points_for_correct_stderr, fdbk.stderr_points_possible) self.assertEqual(self.min_points_possible, fdbk.total_points) self.assertEqual(self.max_points_possible, fdbk.total_points_possible)