def test_complex_peer_assessment_workflow(self):
"""
Intended to mimic a more complicated scenario where people do not
necessarily always finish their assessments.
1) Angel submits
2) Angel waits for Peer Assessments
3) Bob submits and pulls Angel's submission but never reviews it.
4) Sally submits
5) Sally pulls Angel's Submission but never reviews it.
6) Jim submits
7) Jim also doesn't care about Angel and does not bother to review.
8) Buffy comes along and she submits
9) Buffy cares about Angel, but she won't get Angel's submission;
it's held by Bob, Sally, and Jim.
10) Buffy goes on to review Bob, Sally, and Jim, but needs two more.
11) Xander comes along and submits.
12) Xander means well, so Xander grades Bob, Sally, and Jim, but gets
lazy and doesn't grade Buffy when her submission comes along.
13) Buffy is waiting in the wings. She pulls Xander's submission and
grades it.
14) Spike submits.
15) Spike reviews Bob, Sally, Jim, Buffy, and Xander.
16) Buffy reviews Spike
17) Willow comes along and submits
18) Willow goes to grade, and should get Xander
19) Xander comes back and gets Buffy's submission, and grades it.
20) Buffy should now have a grade.
"""
# Buffy should not have a score, because she has not evaluated enough
# peer submissions.
requirements = {
"peer": {
"must_grade": REQUIRED_GRADED,
"must_be_graded_by": REQUIRED_GRADED_BY,
}
}
# 1) Angel Submits
angel_sub, _ = self._create_student_and_submission("Angel", "Angel's answer")
# 2) Angel waits for peers
sub = peer_api.get_submission_to_assess(angel_sub['uuid'], REQUIRED_GRADED_BY)
self.assertIsNone(sub)
# 3) Bob submits
bob_sub, _ = self._create_student_and_submission("Bob", "Bob's answer")
sub = peer_api.get_submission_to_assess(bob_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(angel_sub["uuid"], sub["uuid"])
# 4) Sally submits
sally_sub, _ = self._create_student_and_submission("Sally", "Sally's answer")
# 5) Sally pulls Angel's Submission but never reviews it.
sub = peer_api.get_submission_to_assess(sally_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(angel_sub["uuid"], sub["uuid"])
# 6) Jim submits
jim_sub, _ = self._create_student_and_submission("Jim", "Jim's answer")
# 7) Jim also doesn't care about Angel and does not bother to review.
sub = peer_api.get_submission_to_assess(jim_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(angel_sub["uuid"], sub["uuid"])
# 8) Buffy comes along and she submits
buffy_sub, buffy = self._create_student_and_submission("Buffy", "Buffy's answer")
# 9) Buffy cares about Angel, but she won't get Angel's submission;
# it's held by Bob, Sally, and Jim.
sub = peer_api.get_submission_to_assess(buffy_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(bob_sub["uuid"], sub["uuid"])
# 10) Buffy goes on to review Bob, Sally, and Jim, but needs two more.
peer_api.create_assessment(
buffy_sub["uuid"], buffy["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(buffy_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(sally_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
buffy_sub["uuid"], buffy["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(buffy_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(jim_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
buffy_sub["uuid"], buffy["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(buffy_sub['uuid'], REQUIRED_GRADED_BY)
self.assertIsNone(sub)
# 11) Xander comes along and submits.
xander_sub, xander = self._create_student_and_submission("Xander", "Xander's answer")
# 12) Xander means well, so Xander grades Bob, Sally, and Jim, but gets
# lazy and doesn't grade Buffy when her submission comes along.
sub = peer_api.get_submission_to_assess(xander_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(bob_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
xander_sub["uuid"], xander["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(xander_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(sally_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
xander_sub["uuid"], xander["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(xander_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(jim_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
xander_sub["uuid"], xander["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 13) Buffy is waiting in the wings. She pulls Xander's submission and
# grades it.
sub = peer_api.get_submission_to_assess(buffy_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(xander_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
buffy_sub["uuid"], buffy["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 14) Spike submits.
spike_sub, spike = self._create_student_and_submission("Spike", "Spike's answer")
# 15) Spike reviews Bob, Sally, Jim, Buffy, and Xander.
sub = peer_api.get_submission_to_assess(spike_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(bob_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
spike_sub["uuid"], spike["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(sally_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
spike_sub["uuid"], spike["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(jim_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
spike_sub["uuid"], spike["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(buffy_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
spike_sub["uuid"], spike["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(xander_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
spike_sub["uuid"], spike["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 16) Buffy reviews Spike
sub = peer_api.get_submission_to_assess(buffy_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(spike_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
buffy_sub["uuid"], buffy["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 17) Willow comes along and submits
willow_sub, willow = self._create_student_and_submission("Willow", "Willow's answer")
# 18) Willow goes to grade, and should get Buffy
sub = peer_api.get_submission_to_assess(willow_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(buffy_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
willow_sub["uuid"], willow["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 19) Xander comes back and gets Buffy's submission, and grades it.
sub = peer_api.get_submission_to_assess(xander_sub['uuid'], REQUIRED_GRADED_BY)
self.assertEquals(buffy_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
xander_sub["uuid"], xander["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 20) Buffy should now have a grade.
requirements = {
'must_grade': REQUIRED_GRADED,
'must_be_graded_by': REQUIRED_GRADED_BY
}
self.assertTrue(peer_api.is_complete(buffy_sub["uuid"], requirements))
def _is_peer_complete(self, assessment_requirements):
from openassessment.assessment import peer_api
peer_requirements = assessment_requirements["peer"]
return peer_api.is_complete(self.submission_uuid, peer_requirements)
def test_peer_assessment_workflow(self):
tim_sub, tim = self._create_student_and_submission("Tim", "Tim's answer")
bob_sub, bob = self._create_student_and_submission("Bob", "Bob's answer")
sally_sub, sally = self._create_student_and_submission("Sally", "Sally's answer")
jim_sub, jim = self._create_student_and_submission("Jim", "Jim's answer")
self._create_student_and_submission("Buffy", "Buffy's answer")
self._create_student_and_submission("Xander", "Xander's answer")
# Tim should not have a score, because he has not evaluated enough
# peer submissions.
requirements = {
"peer": {
"must_grade": REQUIRED_GRADED,
"must_be_graded_by": REQUIRED_GRADED_BY,
}
}
score = workflow_api.get_workflow_for_submission(
tim_sub["uuid"], requirements
)["score"]
self.assertIsNone(score)
for i in range(5):
self.assertEquals((False, i), peer_api.has_finished_required_evaluating(tim_sub['uuid'], REQUIRED_GRADED))
sub = peer_api.get_submission_to_assess(tim_sub['uuid'], REQUIRED_GRADED)
peer_api.create_assessment(
tim_sub["uuid"], tim["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
self.assertEquals((True, 5), peer_api.has_finished_required_evaluating(tim_sub['uuid'], REQUIRED_GRADED))
# Tim should not have a score, because his submission does not have
# enough assessments.
self.assertIsNone(sub_api.get_score(STUDENT_ITEM))
sub = peer_api.get_submission_to_assess(bob_sub['uuid'], REQUIRED_GRADED)
self.assertEqual(sub["uuid"], tim_sub["uuid"])
peer_api.create_assessment(
bob_sub["uuid"], bob["student_id"],
ASSESSMENT_DICT['options_selected'],
ASSESSMENT_DICT['criterion_feedback'],
ASSESSMENT_DICT['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(sally_sub['uuid'], REQUIRED_GRADED)
self.assertEqual(sub["uuid"], tim_sub["uuid"])
peer_api.create_assessment(
sally_sub["uuid"], sally["student_id"],
ASSESSMENT_DICT_FAIL['options_selected'],
ASSESSMENT_DICT_FAIL['criterion_feedback'],
ASSESSMENT_DICT_FAIL['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(jim_sub['uuid'], REQUIRED_GRADED)
self.assertEqual(sub["uuid"], tim_sub["uuid"])
peer_api.create_assessment(
jim_sub["uuid"], jim["student_id"],
ASSESSMENT_DICT_PASS['options_selected'],
ASSESSMENT_DICT_PASS['criterion_feedback'],
ASSESSMENT_DICT_PASS['overall_feedback'],
RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# Tim has met the critera, and should now be complete.
requirements = {
'must_grade': REQUIRED_GRADED,
'must_be_graded_by': REQUIRED_GRADED_BY
}
self.assertTrue(peer_api.is_complete(tim_sub["uuid"], requirements))
def test_complex_peer_assessment_workflow(self):
"""
Intended to mimic a more complicated scenario where people do not
necessarily always finish their assessments.
1) Angel submits
2) Angel waits for Peer Assessments
3) Bob submits and pulls Angel's submission but never reviews it.
4) Sally submits
5) Sally pulls Angel's Submission but never reviews it.
6) Jim submits
7) Jim also doesn't care about Angel and does not bother to review.
8) Buffy comes along and she submits
9) Buffy cares about Angel, but she won't get Angel's submission;
it's held by Bob, Sally, and Jim.
10) Buffy goes on to review Bob, Sally, and Jim, but needs two more.
11) Xander comes along and submits.
12) Xander means well, so Xander grades Bob, Sally, and Jim, but gets
lazy and doesn't grade Buffy when her submission comes along.
13) Buffy is waiting in the wings. She pulls Xander's submission and
grades it.
14) Spike submits.
15) Spike reviews Bob, Sally, Jim, Buffy, and Xander.
16) Buffy reviews Spike
17) Willow comes along and submits
18) Willow goes to grade, and should get Xander
19) Xander comes back and gets Buffy's submission, and grades it.
20) Buffy should now have a grade.
"""
# Buffy should not have a score, because she has not evaluated enough
# peer submissions.
requirements = {
"peer": {
"must_grade": REQUIRED_GRADED,
"must_be_graded_by": REQUIRED_GRADED_BY,
}
}
# 1) Angel Submits
angel_sub, angel = self._create_student_and_submission("Angel", "Angel's answer")
# 2) Angel waits for peers
sub = peer_api.get_submission_to_assess(angel, REQUIRED_GRADED_BY)
self.assertIsNone(sub)
# 3) Bob submits
bob_sub, bob = self._create_student_and_submission("Bob", "Bob's answer")
sub = peer_api.get_submission_to_assess(bob, REQUIRED_GRADED_BY)
self.assertEquals(angel_sub["uuid"], sub["uuid"])
# 4) Sally submits
sally_sub, sally = self._create_student_and_submission("Sally", "Sally's answer")
# 5) Sally pulls Angel's Submission but never reviews it.
sub = peer_api.get_submission_to_assess(sally, REQUIRED_GRADED_BY)
self.assertEquals(angel_sub["uuid"], sub["uuid"])
# 6) Jim submits
jim_sub, jim = self._create_student_and_submission("Jim", "Jim's answer")
# 7) Jim also doesn't care about Angel and does not bother to review.
sub = peer_api.get_submission_to_assess(jim, REQUIRED_GRADED_BY)
self.assertEquals(angel_sub["uuid"], sub["uuid"])
# 8) Buffy comes along and she submits
buffy_sub, buffy = self._create_student_and_submission("Buffy", "Buffy's answer")
# 9) Buffy cares about Angel, but she won't get Angel's submission;
# it's held by Bob, Sally, and Jim.
sub = peer_api.get_submission_to_assess(buffy, REQUIRED_GRADED_BY)
self.assertEquals(bob_sub["uuid"], sub["uuid"])
# 10) Buffy goes on to review Bob, Sally, and Jim, but needs two more.
peer_api.create_assessment(
sub["uuid"], buffy["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(buffy, REQUIRED_GRADED_BY)
self.assertEquals(sally_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], buffy["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(buffy, REQUIRED_GRADED_BY)
self.assertEquals(jim_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], buffy["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(buffy, REQUIRED_GRADED_BY)
self.assertIsNone(sub)
# 11) Xander comes along and submits.
xander_sub, xander = self._create_student_and_submission("Xander", "Xander's answer")
# 12) Xander means well, so Xander grades Bob, Sally, and Jim, but gets
# lazy and doesn't grade Buffy when her submission comes along.
sub = peer_api.get_submission_to_assess(xander, REQUIRED_GRADED_BY)
self.assertEquals(bob_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], xander["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(xander, REQUIRED_GRADED_BY)
self.assertEquals(sally_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], xander["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(xander, REQUIRED_GRADED_BY)
self.assertEquals(jim_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], xander["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# Tim has met the critera, and should now have a score.
# We patch the call to `self_api.is_complete()` simulate having completed a self-assessment.
# TODO: currently, we need to import `self_api` within the `_is_self_complete` method
# to avoid circular imports. This means we can't patch self_api directly.
from openassessment.workflow.models import AssessmentWorkflow
with patch.object(AssessmentWorkflow, '_is_self_complete') as mock_complete:
mock_complete.return_value = True
score = workflow_api.get_workflow_for_submission(sub["uuid"], requirements)["score"]
# 13) Buffy is waiting in the wings. She pulls Xander's submission and
# grades it.
sub = peer_api.get_submission_to_assess(buffy, REQUIRED_GRADED_BY)
self.assertEquals(xander_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], buffy["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 14) Spike submits.
spike_sub, spike = self._create_student_and_submission("Spike", "Spike's answer")
# 15) Spike reviews Bob, Sally, Jim, Buffy, and Xander.
sub = peer_api.get_submission_to_assess(spike, REQUIRED_GRADED_BY)
self.assertEquals(bob_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], spike["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike, REQUIRED_GRADED_BY)
self.assertEquals(sally_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], spike["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike, REQUIRED_GRADED_BY)
self.assertEquals(jim_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], spike["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike, REQUIRED_GRADED_BY)
self.assertEquals(buffy_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], spike["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
sub = peer_api.get_submission_to_assess(spike, REQUIRED_GRADED_BY)
self.assertEquals(xander_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], spike["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 16) Buffy reviews Spike
sub = peer_api.get_submission_to_assess(buffy, REQUIRED_GRADED_BY)
self.assertEquals(spike_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], buffy["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 17) Willow comes along and submits
willow_sub, willow = self._create_student_and_submission("Willow", "Willow's answer")
# 18) Willow goes to grade, and should get Buffy
sub = peer_api.get_submission_to_assess(willow, REQUIRED_GRADED_BY)
self.assertEquals(buffy_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], willow["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 19) Xander comes back and gets Buffy's submission, and grades it.
sub = peer_api.get_submission_to_assess(xander, REQUIRED_GRADED_BY)
self.assertEquals(buffy_sub["uuid"], sub["uuid"])
peer_api.create_assessment(
sub["uuid"], xander["student_id"], ASSESSMENT_DICT, RUBRIC_DICT,
REQUIRED_GRADED_BY,
)
# 20) Buffy should now have a grade.
requirements = {
'must_grade': REQUIRED_GRADED,
'must_be_graded_by': REQUIRED_GRADED_BY
}
self.assertTrue(peer_api.is_complete(buffy_sub["uuid"], requirements))
