def test_database_filter_error_handling(self, mock_filter):
# Create a submission
mock_filter.return_value = Assessment.objects.none()
tim_sub, _ = self._create_student_and_submission("Tim", "Tim's answer")
# Note that we have to define this side effect *after* creating the submission
mock_filter.side_effect = DatabaseError("KABOOM!")
# Try to get the latest staff assessment, handle database errors
with self.assertRaises(StaffAssessmentInternalError) as context_manager:
staff_api.get_latest_staff_assessment(tim_sub["uuid"])
self.assertEqual(
str(context_manager.exception),
(
"An error occurred while retrieving staff assessments for the submission with UUID {uuid}: {ex}"
).format(uuid=tim_sub["uuid"], ex="KABOOM!")
)
# Try to get staff assessment scores by criteria, handle database errors
with self.assertRaises(StaffAssessmentInternalError) as context_manager:
staff_api.get_assessment_scores_by_criteria(tim_sub["uuid"])
self.assertEqual(
str(context_manager.exception),
"Error getting staff assessment scores for {}".format(tim_sub["uuid"])
)
def test_database_filter_error_handling(self, mock_filter):
# Create a submission
tim_sub, _ = self._create_student_and_submission("Tim", "Tim's answer")
# Note that we have to define this side effect *after* creating the submission
mock_filter.side_effect = DatabaseError("KABOOM!")
# Try to get the latest staff assessment, handle database errors
with self.assertRaises(StaffAssessmentInternalError) as context_manager:
staff_api.get_latest_staff_assessment(tim_sub["uuid"])
self.assertEqual(
str(context_manager.exception),
(
u"An error occurred while retrieving staff assessments for the submission with UUID {uuid}: {ex}"
).format(uuid=tim_sub["uuid"], ex="KABOOM!")
)
# Try to get staff assessment scores by criteria, handle database errors
with self.assertRaises(StaffAssessmentInternalError) as context_manager:
staff_api.get_assessment_scores_by_criteria(tim_sub["uuid"])
self.assertEqual(
str(context_manager.exception),
u"Error getting staff assessment scores for {}".format(tim_sub["uuid"])
)
def test_staff_assess_handler(self, xblock):
student_item = xblock.get_student_item_dict()
# Create a submission for the student
submission = xblock.create_submission(student_item, self.SUBMISSION)
# Submit a staff-assessment
self.submit_staff_assessment(xblock,
submission,
assessment=STAFF_GOOD_ASSESSMENT)
# Expect that a staff-assessment was created
assessment = staff_api.get_latest_staff_assessment(submission['uuid'])
self.assertEqual(assessment['submission_uuid'], submission['uuid'])
self.assertEqual(assessment['points_earned'], 5)
self.assertEqual(assessment['points_possible'], 6)
self.assertEqual(assessment['scorer_id'], 'Bob')
self.assertEqual(assessment['score_type'], 'ST')
self.assertEqual(assessment['feedback'], 'Staff: good job!')
self.assert_assessment_event_published(
xblock,
'openassessmentblock.staff_assess',
assessment,
type='full-grade')
parts = assessment['parts']
parts.sort(key=lambda x: x['option']['name'])
self.assertEqual(len(parts), 2)
self.assertEqual(parts[0]['option']['criterion']['name'], 'Form')
self.assertEqual(parts[0]['option']['name'], 'Fair')
self.assertEqual(parts[1]['option']['criterion']['name'], '𝓒𝓸𝓷𝓬𝓲𝓼𝓮')
self.assertEqual(parts[1]['option']['name'], 'ﻉซƈﻉɭɭﻉกՇ')
# get the assessment scores by criteria
assessment_by_crit = staff_api.get_assessment_scores_by_criteria(
submission["uuid"])
self.assertEqual(assessment_by_crit['𝓒𝓸𝓷𝓬𝓲𝓼𝓮'], 3)
self.assertEqual(assessment_by_crit['Form'], 2)
score = staff_api.get_score(submission["uuid"], None)
self.assertEqual(assessment['points_earned'], score['points_earned'])
self.assertEqual(assessment['points_possible'],
score['points_possible'])
def test_staff_assess_handler(self, xblock):
student_item = xblock.get_student_item_dict()
# Create a submission for the student
submission = xblock.create_submission(student_item, self.SUBMISSION)
# Submit a staff-assessment
self.submit_staff_assessment(xblock, submission, assessment=STAFF_GOOD_ASSESSMENT)
# Expect that a staff-assessment was created
assessment = staff_api.get_latest_staff_assessment(submission['uuid'])
self.assertEqual(assessment['submission_uuid'], submission['uuid'])
self.assertEqual(assessment['points_earned'], 5)
self.assertEqual(assessment['points_possible'], 6)
self.assertEqual(assessment['scorer_id'], 'Bob')
self.assertEqual(assessment['score_type'], 'ST')
self.assertEqual(assessment['feedback'], u'Staff: good job!')
parts = sorted(assessment['parts'])
self.assertEqual(len(parts), 2)
self.assertEqual(parts[0]['option']['criterion']['name'], u'Form')
self.assertEqual(parts[0]['option']['name'], 'Fair')
self.assertEqual(parts[1]['option']['criterion']['name'], u'𝓒𝓸𝓷𝓬𝓲𝓼𝓮')
self.assertEqual(parts[1]['option']['name'], u'ﻉซƈﻉɭɭﻉกՇ')
# get the assessment scores by criteria
assessment_by_crit = staff_api.get_assessment_scores_by_criteria(submission["uuid"])
self.assertEqual(assessment_by_crit[u'𝓒𝓸𝓷𝓬𝓲𝓼𝓮'], 3)
self.assertEqual(assessment_by_crit[u'Form'], 2)
score = staff_api.get_score(submission["uuid"], None)
self.assertEqual(assessment['points_earned'], score['points_earned'])
self.assertEqual(assessment['points_possible'], score['points_possible'])
self.assert_assessment_event_published(
xblock, 'openassessmentblock.staff_assess', assessment, type='full-grade'
)
def _assert_assessment_data_values(self, xblock, submission, student_id,
assessment):
""" Helper to assert that the assessment data was saved correctly """
self.assertEqual(assessment['submission_uuid'], submission['uuid'])
self.assertEqual(assessment['points_earned'], 5)
self.assertEqual(assessment['points_possible'], 6)
self.assertEqual(assessment['scorer_id'], 'Bob')
self.assertEqual(assessment['score_type'], 'ST')
self.assertEqual(assessment['feedback'],
f"overall feedback for {student_id}")
self.assert_assessment_event_published(
xblock,
'openassessmentblock.staff_assess',
assessment,
type='full-grade')
parts = assessment['parts']
parts.sort(key=lambda x: x['option']['name'])
self.assertEqual(len(parts), 2)
self.assertEqual(parts[0]['option']['criterion']['name'], 'Form')
self.assertEqual(parts[0]['option']['name'], 'Fair')
self.assertEqual(parts[1]['option']['criterion']['name'], '𝓒𝓸𝓷𝓬𝓲𝓼𝓮')
self.assertEqual(parts[1]['option']['name'], 'ﻉซƈﻉɭɭﻉกՇ')
# get the assessment scores by criteria
assessment_by_crit = staff_api.get_assessment_scores_by_criteria(
submission["uuid"])
self.assertEqual(assessment_by_crit['𝓒𝓸𝓷𝓬𝓲𝓼𝓮'], 3)
self.assertEqual(assessment_by_crit['Form'], 2)
score = staff_api.get_score(submission["uuid"], None)
self.assertEqual(assessment['points_earned'], score['points_earned'])
self.assertEqual(assessment['points_possible'],
score['points_possible'])
def grade_details(
self, submission_uuid, peer_assessments, self_assessment, staff_assessment,
is_staff=False
):
"""
Returns details about the grade assigned to the submission.
Args:
submission_uuid (str): The id of the submission being graded.
peer_assessments (list of dict): Serialized assessment models from the peer API.
self_assessment (dict): Serialized assessment model from the self API
staff_assessment (dict): Serialized assessment model from the staff API
is_staff (bool): True if the grade details are being displayed to staff, else False.
Default value is False (meaning grade details are being shown to the learner).
Returns:
A dictionary with full details about the submission's grade.
Example:
{
criteria: [{
'label': 'Test name',
'name': 'f78ac7d4ca1e4134b0ba4b40ca212e72',
'prompt': 'Test prompt',
'order_num': 2,
'options': [...]
'feedback': [
'Good job!',
'Excellent work!',
]
}],
additional_feedback: [{
}]
...
}
"""
# Import is placed here to avoid model import at project startup.
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
from openassessment.assessment.api import staff as staff_api
criteria = copy.deepcopy(self.rubric_criteria_with_labels)
def has_feedback(assessments):
"""
Returns True if at least one assessment has feedback.
Args:
assessments: A list of assessments
Returns:
Returns True if at least one assessment has feedback.
"""
return any(
(
assessment and
(assessment.get('feedback', None) or has_feedback(assessment.get('individual_assessments', [])))
)
for assessment in assessments
)
max_scores = peer_api.get_rubric_max_scores(submission_uuid)
median_scores = None
assessment_steps = self.assessment_steps
if staff_assessment:
median_scores = staff_api.get_assessment_scores_by_criteria(submission_uuid)
elif "peer-assessment" in assessment_steps:
median_scores = peer_api.get_assessment_median_scores(submission_uuid)
elif "self-assessment" in assessment_steps:
median_scores = self_api.get_assessment_scores_by_criteria(submission_uuid)
for criterion in criteria:
criterion_name = criterion['name']
# Record assessment info for the current criterion
criterion['assessments'] = self._graded_assessments(
submission_uuid, criterion,
assessment_steps,
staff_assessment,
peer_assessments,
self_assessment,
is_staff=is_staff,
)
# Record whether there is any feedback provided in the assessments
criterion['has_feedback'] = has_feedback(criterion['assessments'])
# Although we prevent course authors from modifying criteria post-release,
# it's still possible for assessments created by course staff to
# have criteria that differ from the current problem definition.
# It's also possible to circumvent the post-release restriction
# if course authors directly import a course into Studio.
# If this happens, we simply leave the score blank so that the grade
# section can render without error.
criterion['median_score'] = median_scores.get(criterion_name, '')
criterion['total_value'] = max_scores.get(criterion_name, '')
return {
'criteria': criteria,
'additional_feedback': self._additional_feedback(
staff_assessment=staff_assessment,
peer_assessments=peer_assessments,
self_assessment=self_assessment,
),
}
def grade_details(self,
submission_uuid,
peer_assessments,
self_assessment,
staff_assessment,
is_staff=False):
"""
Returns details about the grade assigned to the submission.
Args:
submission_uuid (str): The id of the submission being graded.
peer_assessments (list of dict): Serialized assessment models from the peer API.
self_assessment (dict): Serialized assessment model from the self API
staff_assessment (dict): Serialized assessment model from the staff API
is_staff (bool): True if the grade details are being displayed to staff, else False.
Default value is False (meaning grade details are being shown to the learner).
Returns:
A dictionary with full details about the submission's grade.
Example:
{
criteria: [{
'label': 'Test name',
'name': 'f78ac7d4ca1e4134b0ba4b40ca212e72',
'prompt': 'Test prompt',
'order_num': 2,
'options': [...]
'feedback': [
'Good job!',
'Excellent work!',
]
}],
additional_feedback: [{
}]
...
}
"""
# Import is placed here to avoid model import at project startup.
from openassessment.assessment.api import peer as peer_api
from openassessment.assessment.api import self as self_api
from openassessment.assessment.api import staff as staff_api
criteria = copy.deepcopy(self.rubric_criteria_with_labels)
def has_feedback(assessments):
"""
Returns True if at least one assessment has feedback.
Args:
assessments: A list of assessments
Returns:
Returns True if at least one assessment has feedback.
"""
return any((assessment and (
assessment.get('feedback', None)
or has_feedback(assessment.get('individual_assessments', []))))
for assessment in assessments)
max_scores = peer_api.get_rubric_max_scores(submission_uuid)
median_scores = None
assessment_steps = self.assessment_steps
if staff_assessment:
median_scores = staff_api.get_assessment_scores_by_criteria(
submission_uuid)
elif "peer-assessment" in assessment_steps:
median_scores = peer_api.get_assessment_median_scores(
submission_uuid)
elif "self-assessment" in assessment_steps:
median_scores = self_api.get_assessment_scores_by_criteria(
submission_uuid)
for criterion in criteria:
criterion_name = criterion['name']
# Record assessment info for the current criterion
criterion['assessments'] = self._graded_assessments(
submission_uuid,
criterion,
assessment_steps,
staff_assessment,
peer_assessments,
self_assessment,
is_staff=is_staff,
)
# Record whether there is any feedback provided in the assessments
criterion['has_feedback'] = has_feedback(criterion['assessments'])
# Although we prevent course authors from modifying criteria post-release,
# it's still possible for assessments created by course staff to
# have criteria that differ from the current problem definition.
# It's also possible to circumvent the post-release restriction
# if course authors directly import a course into Studio.
# If this happens, we simply leave the score blank so that the grade
# section can render without error.
criterion['median_score'] = median_scores.get(criterion_name, '')
criterion['total_value'] = max_scores.get(criterion_name, '')
return {
'criteria':
criteria,
'additional_feedback':
self._additional_feedback(
staff_assessment=staff_assessment,
peer_assessments=peer_assessments,
self_assessment=self_assessment,
),
}
