def test_database_filter_error_handling(self, mock_filter): # Create a submission mock_filter.return_value = Assessment.objects.none() tim_sub, _ = self._create_student_and_submission("Tim", "Tim's answer") # Note that we have to define this side effect *after* creating the submission mock_filter.side_effect = DatabaseError("KABOOM!") # Try to get the latest staff assessment, handle database errors with self.assertRaises(StaffAssessmentInternalError) as context_manager: staff_api.get_latest_staff_assessment(tim_sub["uuid"]) self.assertEqual( str(context_manager.exception), ( "An error occurred while retrieving staff assessments for the submission with UUID {uuid}: {ex}" ).format(uuid=tim_sub["uuid"], ex="KABOOM!") ) # Try to get staff assessment scores by criteria, handle database errors with self.assertRaises(StaffAssessmentInternalError) as context_manager: staff_api.get_assessment_scores_by_criteria(tim_sub["uuid"]) self.assertEqual( str(context_manager.exception), "Error getting staff assessment scores for {}".format(tim_sub["uuid"]) )
def test_database_filter_error_handling(self, mock_filter): # Create a submission tim_sub, _ = self._create_student_and_submission("Tim", "Tim's answer") # Note that we have to define this side effect *after* creating the submission mock_filter.side_effect = DatabaseError("KABOOM!") # Try to get the latest staff assessment, handle database errors with self.assertRaises(StaffAssessmentInternalError) as context_manager: staff_api.get_latest_staff_assessment(tim_sub["uuid"]) self.assertEqual( str(context_manager.exception), ( u"An error occurred while retrieving staff assessments for the submission with UUID {uuid}: {ex}" ).format(uuid=tim_sub["uuid"], ex="KABOOM!") ) # Try to get staff assessment scores by criteria, handle database errors with self.assertRaises(StaffAssessmentInternalError) as context_manager: staff_api.get_assessment_scores_by_criteria(tim_sub["uuid"]) self.assertEqual( str(context_manager.exception), u"Error getting staff assessment scores for {}".format(tim_sub["uuid"]) )
def test_staff_assess_handler(self, xblock): student_item = xblock.get_student_item_dict() # Create a submission for the student submission = xblock.create_submission(student_item, self.SUBMISSION) # Submit a staff-assessment self.submit_staff_assessment(xblock, submission, assessment=STAFF_GOOD_ASSESSMENT) # Expect that a staff-assessment was created assessment = staff_api.get_latest_staff_assessment(submission['uuid']) self.assertEqual(assessment['submission_uuid'], submission['uuid']) self.assertEqual(assessment['points_earned'], 5) self.assertEqual(assessment['points_possible'], 6) self.assertEqual(assessment['scorer_id'], 'Bob') self.assertEqual(assessment['score_type'], 'ST') self.assertEqual(assessment['feedback'], 'Staff: good job!') self.assert_assessment_event_published( xblock, 'openassessmentblock.staff_assess', assessment, type='full-grade') parts = assessment['parts'] parts.sort(key=lambda x: x['option']['name']) self.assertEqual(len(parts), 2) self.assertEqual(parts[0]['option']['criterion']['name'], 'Form') self.assertEqual(parts[0]['option']['name'], 'Fair') self.assertEqual(parts[1]['option']['criterion']['name'], '𝓒𝓸𝓷𝓬𝓲𝓼𝓮') self.assertEqual(parts[1]['option']['name'], 'ﻉซƈﻉɭɭﻉกՇ') # get the assessment scores by criteria assessment_by_crit = staff_api.get_assessment_scores_by_criteria( submission["uuid"]) self.assertEqual(assessment_by_crit['𝓒𝓸𝓷𝓬𝓲𝓼𝓮'], 3) self.assertEqual(assessment_by_crit['Form'], 2) score = staff_api.get_score(submission["uuid"], None) self.assertEqual(assessment['points_earned'], score['points_earned']) self.assertEqual(assessment['points_possible'], score['points_possible'])
def test_staff_assess_handler(self, xblock): student_item = xblock.get_student_item_dict() # Create a submission for the student submission = xblock.create_submission(student_item, self.SUBMISSION) # Submit a staff-assessment self.submit_staff_assessment(xblock, submission, assessment=STAFF_GOOD_ASSESSMENT) # Expect that a staff-assessment was created assessment = staff_api.get_latest_staff_assessment(submission['uuid']) self.assertEqual(assessment['submission_uuid'], submission['uuid']) self.assertEqual(assessment['points_earned'], 5) self.assertEqual(assessment['points_possible'], 6) self.assertEqual(assessment['scorer_id'], 'Bob') self.assertEqual(assessment['score_type'], 'ST') self.assertEqual(assessment['feedback'], u'Staff: good job!') parts = sorted(assessment['parts']) self.assertEqual(len(parts), 2) self.assertEqual(parts[0]['option']['criterion']['name'], u'Form') self.assertEqual(parts[0]['option']['name'], 'Fair') self.assertEqual(parts[1]['option']['criterion']['name'], u'𝓒𝓸𝓷𝓬𝓲𝓼𝓮') self.assertEqual(parts[1]['option']['name'], u'ﻉซƈﻉɭɭﻉกՇ') # get the assessment scores by criteria assessment_by_crit = staff_api.get_assessment_scores_by_criteria(submission["uuid"]) self.assertEqual(assessment_by_crit[u'𝓒𝓸𝓷𝓬𝓲𝓼𝓮'], 3) self.assertEqual(assessment_by_crit[u'Form'], 2) score = staff_api.get_score(submission["uuid"], None) self.assertEqual(assessment['points_earned'], score['points_earned']) self.assertEqual(assessment['points_possible'], score['points_possible']) self.assert_assessment_event_published( xblock, 'openassessmentblock.staff_assess', assessment, type='full-grade' )
def _assert_assessment_data_values(self, xblock, submission, student_id, assessment): """ Helper to assert that the assessment data was saved correctly """ self.assertEqual(assessment['submission_uuid'], submission['uuid']) self.assertEqual(assessment['points_earned'], 5) self.assertEqual(assessment['points_possible'], 6) self.assertEqual(assessment['scorer_id'], 'Bob') self.assertEqual(assessment['score_type'], 'ST') self.assertEqual(assessment['feedback'], f"overall feedback for {student_id}") self.assert_assessment_event_published( xblock, 'openassessmentblock.staff_assess', assessment, type='full-grade') parts = assessment['parts'] parts.sort(key=lambda x: x['option']['name']) self.assertEqual(len(parts), 2) self.assertEqual(parts[0]['option']['criterion']['name'], 'Form') self.assertEqual(parts[0]['option']['name'], 'Fair') self.assertEqual(parts[1]['option']['criterion']['name'], '𝓒𝓸𝓷𝓬𝓲𝓼𝓮') self.assertEqual(parts[1]['option']['name'], 'ﻉซƈﻉɭɭﻉกՇ') # get the assessment scores by criteria assessment_by_crit = staff_api.get_assessment_scores_by_criteria( submission["uuid"]) self.assertEqual(assessment_by_crit['𝓒𝓸𝓷𝓬𝓲𝓼𝓮'], 3) self.assertEqual(assessment_by_crit['Form'], 2) score = staff_api.get_score(submission["uuid"], None) self.assertEqual(assessment['points_earned'], score['points_earned']) self.assertEqual(assessment['points_possible'], score['points_possible'])
def grade_details( self, submission_uuid, peer_assessments, self_assessment, staff_assessment, is_staff=False ): """ Returns details about the grade assigned to the submission. Args: submission_uuid (str): The id of the submission being graded. peer_assessments (list of dict): Serialized assessment models from the peer API. self_assessment (dict): Serialized assessment model from the self API staff_assessment (dict): Serialized assessment model from the staff API is_staff (bool): True if the grade details are being displayed to staff, else False. Default value is False (meaning grade details are being shown to the learner). Returns: A dictionary with full details about the submission's grade. Example: { criteria: [{ 'label': 'Test name', 'name': 'f78ac7d4ca1e4134b0ba4b40ca212e72', 'prompt': 'Test prompt', 'order_num': 2, 'options': [...] 'feedback': [ 'Good job!', 'Excellent work!', ] }], additional_feedback: [{ }] ... } """ # Import is placed here to avoid model import at project startup. from openassessment.assessment.api import peer as peer_api from openassessment.assessment.api import self as self_api from openassessment.assessment.api import staff as staff_api criteria = copy.deepcopy(self.rubric_criteria_with_labels) def has_feedback(assessments): """ Returns True if at least one assessment has feedback. Args: assessments: A list of assessments Returns: Returns True if at least one assessment has feedback. """ return any( ( assessment and (assessment.get('feedback', None) or has_feedback(assessment.get('individual_assessments', []))) ) for assessment in assessments ) max_scores = peer_api.get_rubric_max_scores(submission_uuid) median_scores = None assessment_steps = self.assessment_steps if staff_assessment: median_scores = staff_api.get_assessment_scores_by_criteria(submission_uuid) elif "peer-assessment" in assessment_steps: median_scores = peer_api.get_assessment_median_scores(submission_uuid) elif "self-assessment" in assessment_steps: median_scores = self_api.get_assessment_scores_by_criteria(submission_uuid) for criterion in criteria: criterion_name = criterion['name'] # Record assessment info for the current criterion criterion['assessments'] = self._graded_assessments( submission_uuid, criterion, assessment_steps, staff_assessment, peer_assessments, self_assessment, is_staff=is_staff, ) # Record whether there is any feedback provided in the assessments criterion['has_feedback'] = has_feedback(criterion['assessments']) # Although we prevent course authors from modifying criteria post-release, # it's still possible for assessments created by course staff to # have criteria that differ from the current problem definition. # It's also possible to circumvent the post-release restriction # if course authors directly import a course into Studio. # If this happens, we simply leave the score blank so that the grade # section can render without error. criterion['median_score'] = median_scores.get(criterion_name, '') criterion['total_value'] = max_scores.get(criterion_name, '') return { 'criteria': criteria, 'additional_feedback': self._additional_feedback( staff_assessment=staff_assessment, peer_assessments=peer_assessments, self_assessment=self_assessment, ), }
def grade_details(self, submission_uuid, peer_assessments, self_assessment, staff_assessment, is_staff=False): """ Returns details about the grade assigned to the submission. Args: submission_uuid (str): The id of the submission being graded. peer_assessments (list of dict): Serialized assessment models from the peer API. self_assessment (dict): Serialized assessment model from the self API staff_assessment (dict): Serialized assessment model from the staff API is_staff (bool): True if the grade details are being displayed to staff, else False. Default value is False (meaning grade details are being shown to the learner). Returns: A dictionary with full details about the submission's grade. Example: { criteria: [{ 'label': 'Test name', 'name': 'f78ac7d4ca1e4134b0ba4b40ca212e72', 'prompt': 'Test prompt', 'order_num': 2, 'options': [...] 'feedback': [ 'Good job!', 'Excellent work!', ] }], additional_feedback: [{ }] ... } """ # Import is placed here to avoid model import at project startup. from openassessment.assessment.api import peer as peer_api from openassessment.assessment.api import self as self_api from openassessment.assessment.api import staff as staff_api criteria = copy.deepcopy(self.rubric_criteria_with_labels) def has_feedback(assessments): """ Returns True if at least one assessment has feedback. Args: assessments: A list of assessments Returns: Returns True if at least one assessment has feedback. """ return any((assessment and ( assessment.get('feedback', None) or has_feedback(assessment.get('individual_assessments', [])))) for assessment in assessments) max_scores = peer_api.get_rubric_max_scores(submission_uuid) median_scores = None assessment_steps = self.assessment_steps if staff_assessment: median_scores = staff_api.get_assessment_scores_by_criteria( submission_uuid) elif "peer-assessment" in assessment_steps: median_scores = peer_api.get_assessment_median_scores( submission_uuid) elif "self-assessment" in assessment_steps: median_scores = self_api.get_assessment_scores_by_criteria( submission_uuid) for criterion in criteria: criterion_name = criterion['name'] # Record assessment info for the current criterion criterion['assessments'] = self._graded_assessments( submission_uuid, criterion, assessment_steps, staff_assessment, peer_assessments, self_assessment, is_staff=is_staff, ) # Record whether there is any feedback provided in the assessments criterion['has_feedback'] = has_feedback(criterion['assessments']) # Although we prevent course authors from modifying criteria post-release, # it's still possible for assessments created by course staff to # have criteria that differ from the current problem definition. # It's also possible to circumvent the post-release restriction # if course authors directly import a course into Studio. # If this happens, we simply leave the score blank so that the grade # section can render without error. criterion['median_score'] = median_scores.get(criterion_name, '') criterion['total_value'] = max_scores.get(criterion_name, '') return { 'criteria': criteria, 'additional_feedback': self._additional_feedback( staff_assessment=staff_assessment, peer_assessments=peer_assessments, self_assessment=self_assessment, ), }