def test_update_peer_workflow(self):
submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
workflow = workflow_api.create_workflow(submission["uuid"],
["training", "peer"])
StudentTrainingWorkflow.get_or_create_workflow(
submission_uuid=submission["uuid"])
requirements = {
"training": {
"num_required": 2
},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
}
}
workflow_keys = set(workflow.keys())
self.assertEqual(workflow_keys, {
'submission_uuid', 'uuid', 'status', 'created', 'modified', 'score'
})
self.assertEqual(workflow["submission_uuid"], submission["uuid"])
self.assertEqual(workflow["status"], "training")
peer_workflows = list(
PeerWorkflow.objects.filter(submission_uuid=submission["uuid"]))
self.assertFalse(peer_workflows)
workflow_from_get = workflow_api.get_workflow_for_submission(
submission["uuid"], requirements)
del workflow_from_get['status_details']
self.assertEqual(workflow, workflow_from_get)
requirements["training"]["num_required"] = 0
workflow = workflow_api.update_from_assessments(
submission["uuid"], requirements)
# New step is Peer, and a Workflow has been created.
self.assertEqual(workflow["status"], "peer")
peer_workflow = PeerWorkflow.objects.get(
submission_uuid=submission["uuid"])
self.assertIsNotNone(peer_workflow)
def test_update_peer_workflow(self):
submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"])
StudentTrainingWorkflow.get_or_create_workflow(submission_uuid=submission["uuid"])
requirements = {
"training": {
"num_required": 2
},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
}
}
workflow_keys = set(workflow.keys())
self.assertEqual(
workflow_keys,
{
'submission_uuid', 'uuid', 'status', 'created', 'modified', 'score'
}
)
self.assertEqual(workflow["submission_uuid"], submission["uuid"])
self.assertEqual(workflow["status"], "training")
peer_workflows = list(PeerWorkflow.objects.filter(submission_uuid=submission["uuid"]))
self.assertFalse(peer_workflows)
workflow_from_get = workflow_api.get_workflow_for_submission(
submission["uuid"], requirements
)
del workflow_from_get['status_details']
self.assertEqual(workflow, workflow_from_get)
requirements["training"]["num_required"] = 0
workflow = workflow_api.update_from_assessments(submission["uuid"], requirements)
# New step is Peer, and a Workflow has been created.
self.assertEqual(workflow["status"], "peer")
peer_workflow = PeerWorkflow.objects.get(submission_uuid=submission["uuid"])
self.assertIsNotNone(peer_workflow)
def test_create_workflow_item_integrity_error(self, mock_create, mock_get):
# Create a submission and workflow
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.get_or_create_workflow(submission['uuid'])
# Simulate a race condition in which someone creates a workflow item
# after we check if it exists.
mock_workflow_item = mock.MagicMock(StudentTrainingWorkflowItem)
mock_create.side_effect = IntegrityError
mock_get.return_value = mock_workflow_item
# Expect that we retry and retrieve the workflow item created by someone else
self.assertEqual(workflow.next_training_example(EXAMPLES), mock_workflow_item.training_example)
def test_create_workflow_item_integrity_error(self, mock_create, mock_get):
# Create a submission and workflow
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.get_or_create_workflow(
submission['uuid'])
# Simulate a race condition in which someone creates a workflow item
# after we check if it exists.
mock_workflow_item = mock.MagicMock(StudentTrainingWorkflowItem)
mock_create.side_effect = IntegrityError
mock_get.return_value = mock_workflow_item
# Expect that we retry and retrieve the workflow item created by someone else
self.assertEqual(workflow.next_training_example(EXAMPLES),
mock_workflow_item.training_example)
def test_create_workflow_integrity_error(self, mock_create, mock_get):
# Simulate a race condition in which someone creates a workflow
# after we check if it exists. This will violate the database uniqueness
# constraints, so we need to handle this case gracefully.
mock_create.side_effect = IntegrityError
# The first time we check, we should see that no workflow exists.
# The second time, we should get the workflow created by someone else
mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
mock_get.side_effect = [
StudentTrainingWorkflow.DoesNotExist, mock_workflow
]
# Expect that we retry and retrieve the workflow that someone else created
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.get_or_create_workflow(
submission['uuid'])
self.assertEqual(workflow, mock_workflow)
def test_create_workflow_integrity_error(self, mock_create, mock_get):
# Simulate a race condition in which someone creates a workflow
# after we check if it exists. This will violate the database uniqueness
# constraints, so we need to handle this case gracefully.
mock_create.side_effect = IntegrityError
# The first time we check, we should see that no workflow exists.
# The second time, we should get the workflow created by someone else
mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
mock_get.side_effect = [
StudentTrainingWorkflow.DoesNotExist,
mock_workflow
]
# Expect that we retry and retrieve the workflow that someone else created
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.get_or_create_workflow(submission['uuid'])
self.assertEqual(workflow, mock_workflow)
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
If no training examples are available (the student has already assessed every example,
or no examples are defined), returns None.
Raises:
StudentTrainingInternalError
Example usage:
>>> examples = [
>>> {
>>> 'answer': u'Doler',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': u'Lorem ipsum',
'rubric': {
"prompt": "Write an essay!",
"criteria": [
{
"order_num": 0,
"name": "vocabulary",
"prompt": "How varied is the vocabulary?",
"options": options
},
{
"order_num": 1,
"name": "grammar",
"prompt": "How correct is the grammar?",
"options": options
}
],
},
'options_selected': {
'vocabulary': 'good',
'grammar': 'excellent'
}
}
"""
try:
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if len(errors) > 0:
msg = _(u"Training examples do not match the rubric: {errors}").format(
errors="\n".join(errors)
)
raise StudentTrainingRequestError(msg)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_or_create_workflow(submission_uuid=submission_uuid)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
next_example = workflow.next_training_example(examples)
return None if next_example is None else serialize_training_example(next_example)
except (InvalidRubric, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex)
except sub_api.SubmissionNotFoundError as ex:
msg = _(u"Could not retrieve the submission with UUID {}").format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg)
except DatabaseError:
msg = _(
u"Could not retrieve a training example "
u"for the student with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
If no training examples are available (the student has already assessed every example,
or no examples are defined), returns None.
Raises:
StudentTrainingInternalError
Example usage:
>>> examples = [
>>> {
>>> 'answer': u'Doler',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': u'Lorem ipsum',
'rubric': {
"prompt": "Write an essay!",
"criteria": [
{
"order_num": 0,
"name": "vocabulary",
"prompt": "How varied is the vocabulary?",
"options": options
},
{
"order_num": 1,
"name": "grammar",
"prompt": "How correct is the grammar?",
"options": options
}
],
},
'options_selected': {
'vocabulary': 'good',
'grammar': 'excellent'
}
}
"""
try:
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if len(errors) > 0:
msg = _(u"Training examples do not match the rubric: {errors}"
).format(errors="\n".join(errors))
raise StudentTrainingRequestError(msg)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_or_create_workflow(
submission_uuid=submission_uuid)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
next_example = workflow.next_training_example(examples)
return None if next_example is None else serialize_training_example(
next_example)
except (InvalidRubric, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".
format(submission_uuid))
raise StudentTrainingRequestError(ex)
except sub_api.SubmissionNotFoundError as ex:
msg = _(u"Could not retrieve the submission with UUID {}").format(
submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg)
except DatabaseError:
msg = _(
u"Could not retrieve a training example "
u"for the student with submission UUID {}").format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
