def on_start(submission_uuid):
"""
Creates a new student training workflow.
This function should be called to indicate that a submission has entered the
student training workflow part of the assessment process.
Args:
submission_uuid (str): The submission UUID for the student that is
initiating training.
Returns:
None
Raises:
StudentTrainingInternalError: Raised when an error occurs persisting the
Student Training Workflow
"""
try:
StudentTrainingWorkflow.create_workflow(submission_uuid)
except Exception as ex:
msg = (
"An internal error has occurred while creating the learner "
"training workflow for submission UUID {}".format(submission_uuid)
)
logger.exception(msg)
raise StudentTrainingInternalError(msg) from ex
def on_start(submission_uuid):
"""
Creates a new student training workflow.
This function should be called to indicate that a submission has entered the
student training workflow part of the assessment process.
Args:
submission_uuid (str): The submission UUID for the student that is
initiating training.
Returns:
None
Raises:
StudentTrainingInternalError: Raised when an error occurs persisting the
Student Training Workflow
"""
try:
StudentTrainingWorkflow.create_workflow(submission_uuid)
except Exception:
msg = (
u"An internal error has occurred while creating the student "
u"training workflow for submission UUID {}".format(submission_uuid)
)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
def test_create_workflow_integrity_error(self, mock_create, mock_get):
# Simulate a race condition in which someone creates a workflow
# after we check if it exists. This will violate the database uniqueness
# constraints, so we need to handle this case gracefully.
mock_create.side_effect = IntegrityError
# The first time we check, we should see that no workflow exists.
# The second time, we should get the workflow created by someone else
mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
mock_get.side_effect = [mock_workflow]
# Expect that we retry and retrieve the workflow that someone else created
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
StudentTrainingWorkflow.create_workflow(submission['uuid'])
workflow = StudentTrainingWorkflow.get_workflow(submission['uuid'])
self.assertEqual(workflow, mock_workflow)
def test_create_workflow_integrity_error(self, mock_create, mock_get):
# Simulate a race condition in which someone creates a workflow
# after we check if it exists. This will violate the database uniqueness
# constraints, so we need to handle this case gracefully.
mock_create.side_effect = IntegrityError
# The first time we check, we should see that no workflow exists.
# The second time, we should get the workflow created by someone else
mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
mock_get.side_effect = [
mock_workflow
]
# Expect that we retry and retrieve the workflow that someone else created
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
StudentTrainingWorkflow.create_workflow(submission['uuid'])
workflow = StudentTrainingWorkflow.get_workflow(submission['uuid'])
self.assertEqual(workflow, mock_workflow)
def test_update_peer_workflow(self):
submission = sub_api.create_submission(ITEM_1, ANSWER_1)
workflow = workflow_api.create_workflow(submission["uuid"],
["training", "peer"])
StudentTrainingWorkflow.create_workflow(
submission_uuid=submission["uuid"])
requirements = {
"training": {
"num_required": 2
},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
}
}
workflow_keys = set(workflow.keys())
self.assertEqual(
workflow_keys, {
'submission_uuid', 'status', 'created', 'modified', 'score',
'assessment_score_priority'
})
self.assertEqual(workflow["submission_uuid"], submission["uuid"])
self.assertEqual(workflow["status"], "training")
peer_workflows = list(
PeerWorkflow.objects.filter(submission_uuid=submission["uuid"]))
self.assertFalse(peer_workflows)
workflow_from_get = workflow_api.get_workflow_for_submission(
submission["uuid"], requirements)
del workflow_from_get['status_details']
self.assertEqual(workflow, workflow_from_get)
requirements["training"]["num_required"] = 0
workflow = workflow_api.update_from_assessments(
submission["uuid"], requirements)
# New step is Peer, and a Workflow has been created.
self.assertEqual(workflow["status"], "peer")
peer_workflow = PeerWorkflow.objects.get(
submission_uuid=submission["uuid"])
self.assertIsNotNone(peer_workflow)
def test_update_peer_workflow(self):
submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"], ON_INIT_PARAMS)
StudentTrainingWorkflow.create_workflow(submission_uuid=submission["uuid"])
requirements = {
"training": {
"num_required": 2
},
"peer": {
"must_grade": 5,
"must_be_graded_by": 3
}
}
workflow_keys = set(workflow.keys())
self.assertEqual(
workflow_keys,
{
'override_score',
'submission_uuid', 'uuid', 'status', 'created', 'modified', 'score'
}
)
self.assertEqual(workflow["submission_uuid"], submission["uuid"])
self.assertEqual(workflow["status"], "training")
peer_workflows = list(PeerWorkflow.objects.filter(submission_uuid=submission["uuid"]))
self.assertFalse(peer_workflows)
workflow_from_get = workflow_api.get_workflow_for_submission(
submission["uuid"], requirements
)
del workflow_from_get['status_details']
self.assertEqual(workflow, workflow_from_get)
requirements["training"]["num_required"] = 0
workflow = workflow_api.update_from_assessments(submission["uuid"], requirements)
# New step is Peer, and a Workflow has been created.
self.assertEqual(workflow["status"], "peer")
peer_workflow = PeerWorkflow.objects.get(submission_uuid=submission["uuid"])
self.assertIsNotNone(peer_workflow)
def test_create_workflow_item_integrity_error(self, mock_create):
# Create a submission and workflow
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.create_workflow(submission['uuid'])
# Simulate a race condition in which someone creates a workflow item
# after we check if it exists.
mock.MagicMock(StudentTrainingWorkflowItem)
mock_create.side_effect = IntegrityError
# Expect that we retry and retrieve the workflow item created by someone else
self.assertEqual(workflow.next_training_example(EXAMPLES), EXAMPLES[0])
def test_create_workflow_item_integrity_error(self, mock_create):
# Create a submission and workflow
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
workflow = StudentTrainingWorkflow.create_workflow(submission['uuid'])
# Simulate a race condition in which someone creates a workflow item
# after we check if it exists.
mock_workflow_item = mock.MagicMock(StudentTrainingWorkflowItem)
mock_create.side_effect = IntegrityError
# Expect that we retry and retrieve the workflow item created by someone else
self.assertEqual(workflow.next_training_example(EXAMPLES), EXAMPLES[0])
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
If no training examples are available (the student has already assessed every example,
or no examples are defined), returns None.
Raises:
StudentTrainingInternalError
Example usage:
>>> examples = [
>>> {
>>> 'answer': u'Doler',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': u'Lorem ipsum',
'rubric': {
"prompt": "Write an essay!",
"criteria": [
{
"order_num": 0,
"name": "vocabulary",
"prompt": "How varied is the vocabulary?",
"options": options
},
{
"order_num": 1,
"name": "grammar",
"prompt": "How correct is the grammar?",
"options": options
}
],
},
'options_selected': {
'vocabulary': 'good',
'grammar': 'excellent'
}
}
"""
try:
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if len(errors) > 0:
msg = (
u"Training examples do not match the rubric (submission UUID is {uuid}): {errors}"
).format(uuid=submission_uuid, errors="\n".join(errors))
raise StudentTrainingRequestError(msg)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid)
if not workflow:
raise StudentTrainingRequestError(
u"No student training workflow found for submission {}".format(submission_uuid)
)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
next_example = workflow.next_training_example(examples)
return None if next_example is None else serialize_training_example(next_example)
except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex)
except sub_api.SubmissionNotFoundError as ex:
msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg)
except DatabaseError:
msg = (
u"Could not retrieve a training example "
u"for the student with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
If no training examples are available (the student has already assessed every example,
or no examples are defined), returns None.
Raises:
StudentTrainingInternalError
Example usage:
>>> examples = [
>>> {
>>> 'answer': {
>>> 'parts': {
>>> [
>>> {'text:' 'Answer part 1'},
>>> {'text:' 'Answer part 2'},
>>> {'text:' 'Answer part 3'}
>>> ]
>>> }
>>> },
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': {
'parts': {
[
{'text:' 'Answer part 1'},
{'text:' 'Answer part 2'},
{'text:' 'Answer part 3'}
]
}
},
'rubric': {
"prompts": [
{"description": "Prompt 1"},
{"description": "Prompt 2"},
{"description": "Prompt 3"}
],
"criteria": [
{
"order_num": 0,
"name": "vocabulary",
"prompt": "How varied is the vocabulary?",
"options": options
},
{
"order_num": 1,
"name": "grammar",
"prompt": "How correct is the grammar?",
"options": options
}
],
},
'options_selected': {
'vocabulary': 'good',
'grammar': 'excellent'
}
}
"""
try:
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if errors:
msg = (
"Training examples do not match the rubric (submission UUID is {uuid}): {errors}"
).format(uuid=submission_uuid, errors="\n".join(errors))
raise StudentTrainingRequestError(msg)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid)
if not workflow:
raise StudentTrainingRequestError(
u"No learner training workflow found for submission {}".format(submission_uuid)
)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
next_example = workflow.next_training_example(examples)
return None if next_example is None else serialize_training_example(next_example)
except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex:
logger.exception(
u"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex) from ex
except sub_api.SubmissionNotFoundError as ex:
msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg) from ex
except DatabaseError as ex:
msg = (
u"Could not retrieve a training example for the learner with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg) from ex