def on_init(team_submission_uuid):
"""
Create a new team staff workflow for a student item and submission.
Creates a unique team staff workflow for a student item, associated with a
team submission.
Note that the staff workflow begins things in on_init() instead of
on_start(), because staff shoud be able to access the submission
regardless of which state the workflow is currently in.
Args:
team_submission_uuid (str): The team submission associated with this workflow.
Returns:
None
Raises:
StaffAssessmentInternalError: Raised when there is an internal error
creating the Workflow.
"""
try:
team_submission = team_submissions_api.get_team_submission(
team_submission_uuid)
TeamStaffWorkflow.objects.get_or_create(
course_id=team_submission['course_id'],
item_id=team_submission['item_id'],
team_submission_uuid=team_submission_uuid)
except DatabaseError:
error_message = (
"An internal error occurred while creating a new team staff workflow for team submission {}"
).format(team_submission_uuid)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
def on_cancel(team_submission_uuid):
"""
Cancel the team staff workflow for submission.
Sets the cancelled_at field in team staff workflow.
Args:
team_submission_uuid (str): The team_submission UUID associated with this workflow.
Returns:
None
"""
try:
workflow = TeamStaffWorkflow.objects.get(
team_submission_uuid=team_submission_uuid)
workflow.cancelled_at = now()
workflow.save(update_fields=['cancelled_at'])
except TeamStaffWorkflow.DoesNotExist:
# If we can't find a workflow, then we don't have to do anything to
# cancel it.
pass
except DatabaseError:
error_message = (
"An internal error occurred while cancelling the team staff workflow for submission {}"
).format(team_submission_uuid)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
def get_assessment_scores_by_criteria(team_submission_uuid):
"""Get the staff score for each rubric criterion
Args:
team_submission_uuid (str): The team submission uuid is used to get the
assessment used to score this submission.
Returns:
(dict): A dictionary of rubric criterion names, with a score of
the staff assessments.
Raises:
StaffAssessmentInternalError: If any error occurs while retrieving
information from the scores, an error is raised.
"""
try:
# Get most recently graded assessment for a team submission
team_submission = team_submissions_api.get_team_submission(
team_submission_uuid)
assessments = list(
Assessment.objects.filter(
submission_uuid__in=team_submission['submission_uuids'],
score_type=STAFF_TYPE,
)[:1])
scores = Assessment.scores_by_criterion(assessments)
# Since this is only being sent one score, the median score will be the
# same as the only score.
return Assessment.get_median_score_dict(scores)
except DatabaseError:
error_message = "Error getting staff assessment scores for {}".format(
team_submission_uuid)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
def get_assessment_scores_by_criteria(submission_uuid):
"""Get the staff score for each rubric criterion
Args:
submission_uuid (str): The submission uuid is used to get the
assessment used to score this submission.
Returns:
(dict): A dictionary of rubric criterion names, with a score of
the staff assessments.
Raises:
StaffAssessmentInternalError: If any error occurs while retrieving
information from the scores, an error is raised.
"""
try:
# This will always create a list of length 1
assessments = list(
Assessment.objects.filter(
score_type=STAFF_TYPE, submission_uuid=submission_uuid
)[:1]
)
scores = Assessment.scores_by_criterion(assessments)
# Since this is only being sent one score, the median score will be the
# same as the only score.
return Assessment.get_median_score_dict(scores)
except DatabaseError:
error_message = u"Error getting staff assessment scores for {}".format(submission_uuid)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
def create_assessment(
team_submission_uuid,
scorer_id,
options_selected,
criterion_feedback,
overall_feedback,
rubric_dict,
scored_at=None
):
"""
Creates an assessment for each member of the submitting team.
Closely mirrors openassessment.assessment.api.staff.py::create_assessment
Can use _complete_assessment from Staff API as is, but has the side-effect
of only associating the last graded assessment with the workflow
Returns:
dict: the Assessment model, serialized as a dict.
"""
try:
try:
scorer_workflow = TeamStaffWorkflow.objects.get(team_submission_uuid=team_submission_uuid)
except TeamStaffWorkflow.DoesNotExist:
scorer_workflow = None
# Get the submissions for a team
team_submission = team_submissions_api.get_team_submission(team_submission_uuid)
assessment_dicts = []
for submission_uuid in team_submission['submission_uuids']:
assessment = _complete_assessment(
submission_uuid,
scorer_id,
options_selected,
criterion_feedback,
overall_feedback,
rubric_dict,
scored_at,
scorer_workflow
)
assessment_dicts.append(full_assessment_dict(assessment))
return assessment_dicts
except InvalidRubric as ex:
error_message = "The rubric definition is not valid."
logger.exception(error_message)
raise StaffAssessmentRequestError(error_message) from ex
except InvalidRubricSelection as ex:
error_message = "Invalid options were selected in the rubric."
logger.warning(error_message, exc_info=True)
raise StaffAssessmentRequestError(error_message) from ex
except DatabaseError as ex:
error_message = (
"An error occurred while creating an assessment by the scorer with this ID: {}"
).format(scorer_id)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message) from ex
def get_submission_for_review(cls, course_id, item_id, scorer_id):
"""
Find a submission for staff assessment. This function will find the next
submission that requires assessment, excluding any submission that has been
completely graded, or is actively being reviewed by other staff members.
Args:
submission_uuid (str): The submission UUID from the student
requesting a submission for assessment. This is used to explicitly
avoid giving the student their own submission, and determines the
associated Peer Workflow.
item_id (str): The student_item that we would like to retrieve submissions for.
scorer_id (str): The user id of the staff member scoring this submission
Returns:
submission_uuid (str): The submission_uuid for the submission to review.
Raises:
StaffAssessmentInternalError: Raised when there is an error retrieving
the workflows for this request.
"""
# pylint: disable=unicode-format-string
timeout = (now() - cls.TIME_LIMIT).strftime("%Y-%m-%d %H:%M:%S")
try:
# Search for existing submissions that the scorer has worked on.
staff_workflows = StaffWorkflow.objects.filter(
course_id=course_id,
item_id=item_id,
scorer_id=scorer_id,
grading_completed_at=None,
cancelled_at=None,
)
# If no existing submissions exist, then get any other
# available workflows.
if not staff_workflows:
staff_workflows = StaffWorkflow.objects.filter(
models.Q(scorer_id='')
| models.Q(grading_started_at__lte=timeout),
course_id=course_id,
item_id=item_id,
grading_completed_at=None,
cancelled_at=None,
)
if not staff_workflows:
return None
workflow = staff_workflows[0]
workflow.scorer_id = scorer_id
workflow.grading_started_at = now()
workflow.save()
return workflow.submission_uuid
except DatabaseError:
error_message = (
u"An internal error occurred while retrieving a submission for staff grading"
)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
def get_submission_to_assess(course_id, item_id, scorer_id):
"""
Get a team submission for staff evaluation.
Retrieves a team submission for assessment for the given staff member.
Args:
course_id (str): The course that we would like to fetch submissions from.
item_id (str): The student_item (problem) that we would like to retrieve submissions for.
scorer_id (str): The user id of the staff member scoring this submission
Returns:
dict: A student submission for assessment. This contains a 'student_item',
'attempt_number', 'submitted_at', 'created_at', and 'answer' field to be
used for assessment.
Raises:
StaffAssessmentInternalError: Raised when there is an internal error
retrieving staff workflow information.
Examples:
>>> get_submission_to_assess("a_course_id", "an_item_id", "a_scorer_id")
{
'student_item': 2,
'attempt_number': 1,
'submitted_at': datetime.datetime(2014, 1, 29, 23, 14, 52, 649284, tzinfo=<UTC>),
'created_at': datetime.datetime(2014, 1, 29, 17, 14, 52, 668850, tzinfo=<UTC>),
'answer': { ... }
}
"""
team_submission_uuid = TeamStaffWorkflow.get_submission_for_review(
course_id, item_id, scorer_id)
if team_submission_uuid:
try:
submission_data = team_submissions_api.get_team_submission(
team_submission_uuid)
return submission_data
except DatabaseError:
error_message = (
"Could not find a team submission with the uuid {}"
).format(team_submission_uuid)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
else:
logger.info(
"No team submission found for staff to assess ({}, {})".format(
course_id,
item_id,
))
return None
def get_latest_staff_assessment(team_submission_uuid):
"""
Retrieve the latest staff assessment for a team submission.
Args:
team_submission_uuid (str): The UUID of the team submission being assessed.
Returns:
dict: The serialized assessment model
or None if no assessments are available
Raises:
StaffAssessmentInternalError if there are problems connecting to the database.
Example usage:
>>> get_latest_staff_assessment('10df7db776686822e501b05f452dc1e4b9141fe5')
{
'points_earned': 6,
'points_possible': 12,
'scored_at': datetime.datetime(2014, 1, 29, 17, 14, 52, 649284 tzinfo=<UTC>),
'scorer': "staff",
'feedback': ''
}
"""
try:
# Get the reference submissions
team_submission = team_submissions_api.get_team_submission(
team_submission_uuid)
# Return the most-recently graded assessment for any team member's submisison
assessment = Assessment.objects.filter(
submission_uuid__in=team_submission['submission_uuids'],
score_type=STAFF_TYPE,
).first()
except DatabaseError as ex:
msg = ("An error occurred while retrieving staff assessments "
"for the submission with UUID {uuid}: {ex}").format(
uuid=team_submission_uuid, ex=ex)
logger.exception(msg)
raise StaffAssessmentInternalError(msg)
if assessment:
return full_assessment_dict(assessment)
return None
def get_latest_staff_assessment(submission_uuid):
"""
Retrieve the latest staff assessment for a submission.
Args:
submission_uuid (str): The UUID of the submission being assessed.
Returns:
dict: The serialized assessment model
or None if no assessments are available
Raises:
StaffAssessmentInternalError if there are problems connecting to the database.
Example usage:
>>> get_latest_staff_assessment('10df7db776686822e501b05f452dc1e4b9141fe5')
{
'points_earned': 6,
'points_possible': 12,
'scored_at': datetime.datetime(2014, 1, 29, 17, 14, 52, 649284 tzinfo=<UTC>),
'scorer': u"staff",
'feedback': u''
}
"""
try:
assessments = Assessment.objects.filter(
submission_uuid=submission_uuid,
score_type=STAFF_TYPE,
)[:1]
except DatabaseError as ex:
msg = (
u"An error occurred while retrieving staff assessments "
u"for the submission with UUID {uuid}: {ex}"
).format(uuid=submission_uuid, ex=ex)
logger.exception(msg)
raise StaffAssessmentInternalError(msg)
if len(assessments) > 0:
return full_assessment_dict(assessments[0])
else:
return None
def on_init(team_submission_uuid):
"""
Create a new team staff workflow for a student item and submission.
Creates a unique team staff workflow for a student item, associated with a
team submission.
Note that the staff workflow begins things in on_init() instead of
on_start(), because staff shoud be able to access the submission
regardless of which state the workflow is currently in.
Args:
team_submission_uuid (str): The team submission associated with this workflow.
Returns:
None
Raises:
StaffAssessmentInternalError: Raised when there is an internal error
creating the Workflow.
"""
try:
team_submission = team_submissions_api.get_team_submission(
team_submission_uuid)
TeamStaffWorkflow.objects.get_or_create(
course_id=team_submission['course_id'],
item_id=team_submission['item_id'],
team_submission_uuid=team_submission_uuid,
# submission_uuid is currently not used in any logic in TeamStaffWorkflow, so we don't
# realy care which submission is chosen and it doesn't need to match the TeamAssessment Workflow.
# It must be filled because of the unique constraint on the field (can't have multiple '' values)
submission_uuid=team_submission['submission_uuids'][0],
)
except DatabaseError:
error_message = (
"An internal error occurred while creating a new team staff workflow for team submission {}"
).format(team_submission_uuid)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
def create_assessment(
submission_uuid,
scorer_id,
options_selected,
criterion_feedback,
overall_feedback,
rubric_dict,
scored_at=None
):
"""Creates an assessment on the given submission.
Assessments are created based on feedback associated with a particular
rubric.
Assumes that the user creating the assessment has the permissions to do so.
Args:
submission_uuid (str): The submission uuid for the submission being
assessed.
scorer_id (str): The user ID for the user giving this assessment. This
is required to create an assessment on a submission.
options_selected (dict): Dictionary mapping criterion names to the
option names the user selected for that criterion.
criterion_feedback (dict): Dictionary mapping criterion names to the
free-form text feedback the user gave for the criterion.
Since criterion feedback is optional, some criteria may not appear
in the dictionary.
overall_feedback (unicode): Free-form text feedback on the submission overall.
rubric_dict (dict): The rubric model associated with this assessment
scored_at (datetime): Optional argument to override the time in which
the assessment took place. If not specified, scored_at is set to
now.
Keyword Args:
scored_at (datetime): Optional argument to override the time in which
the assessment took place. If not specified, scored_at is set to
now.
Returns:
dict: the Assessment model, serialized as a dict.
Raises:
StaffAssessmentRequestError: Raised when the submission_id is invalid, or
the assessment_dict does not contain the required values to create
an assessment.
StaffAssessmentInternalError: Raised when there is an internal error
while creating a new assessment.
Examples:
>>> options_selected = {"clarity": "Very clear", "precision": "Somewhat precise"}
>>> criterion_feedback = {"clarity": "I thought this essay was very clear."}
>>> feedback = "Your submission was thrilling."
>>> create_assessment("Tim", options_selected, criterion_feedback, feedback, rubric_dict)
"""
try:
try:
scorer_workflow = StaffWorkflow.objects.get(submission_uuid=submission_uuid)
except StaffWorkflow.DoesNotExist:
scorer_workflow = None
assessment = _complete_assessment(
submission_uuid,
scorer_id,
options_selected,
criterion_feedback,
overall_feedback,
rubric_dict,
scored_at,
scorer_workflow
)
return full_assessment_dict(assessment)
except InvalidRubric:
error_message = u"The rubric definition is not valid."
logger.exception(error_message)
raise StaffAssessmentRequestError(error_message)
except InvalidRubricSelection:
error_message = u"Invalid options were selected in the rubric."
logger.warning(error_message, exc_info=True)
raise StaffAssessmentRequestError(error_message)
except DatabaseError:
error_message = (
u"An error occurred while creating an assessment by the scorer with this ID: {}"
).format(scorer_id)
logger.exception(error_message)
raise StaffAssessmentInternalError(error_message)
