def on_start(submission_uuid): """ Creates a new student training workflow. This function should be called to indicate that a submission has entered the student training workflow part of the assessment process. Args: submission_uuid (str): The submission UUID for the student that is initiating training. Returns: None Raises: StudentTrainingInternalError: Raised when an error occurs persisting the Student Training Workflow """ try: StudentTrainingWorkflow.create_workflow(submission_uuid) except Exception as ex: msg = ( "An internal error has occurred while creating the learner " "training workflow for submission UUID {}".format(submission_uuid) ) logger.exception(msg) raise StudentTrainingInternalError(msg) from ex
def on_start(submission_uuid): """ Creates a new student training workflow. This function should be called to indicate that a submission has entered the student training workflow part of the assessment process. Args: submission_uuid (str): The submission UUID for the student that is initiating training. Returns: None Raises: StudentTrainingInternalError: Raised when an error occurs persisting the Student Training Workflow """ try: StudentTrainingWorkflow.create_workflow(submission_uuid) except Exception: msg = ( u"An internal error has occurred while creating the student " u"training workflow for submission UUID {}".format(submission_uuid) ) logger.exception(msg) raise StudentTrainingInternalError(msg)
def test_create_workflow_integrity_error(self, mock_create, mock_get): # Simulate a race condition in which someone creates a workflow # after we check if it exists. This will violate the database uniqueness # constraints, so we need to handle this case gracefully. mock_create.side_effect = IntegrityError # The first time we check, we should see that no workflow exists. # The second time, we should get the workflow created by someone else mock_workflow = mock.MagicMock(StudentTrainingWorkflow) mock_get.side_effect = [mock_workflow] # Expect that we retry and retrieve the workflow that someone else created submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) StudentTrainingWorkflow.create_workflow(submission['uuid']) workflow = StudentTrainingWorkflow.get_workflow(submission['uuid']) self.assertEqual(workflow, mock_workflow)
def test_create_workflow_integrity_error(self, mock_create, mock_get): # Simulate a race condition in which someone creates a workflow # after we check if it exists. This will violate the database uniqueness # constraints, so we need to handle this case gracefully. mock_create.side_effect = IntegrityError # The first time we check, we should see that no workflow exists. # The second time, we should get the workflow created by someone else mock_workflow = mock.MagicMock(StudentTrainingWorkflow) mock_get.side_effect = [ mock_workflow ] # Expect that we retry and retrieve the workflow that someone else created submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) StudentTrainingWorkflow.create_workflow(submission['uuid']) workflow = StudentTrainingWorkflow.get_workflow(submission['uuid']) self.assertEqual(workflow, mock_workflow)
def test_update_peer_workflow(self): submission = sub_api.create_submission(ITEM_1, ANSWER_1) workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"]) StudentTrainingWorkflow.create_workflow( submission_uuid=submission["uuid"]) requirements = { "training": { "num_required": 2 }, "peer": { "must_grade": 5, "must_be_graded_by": 3 } } workflow_keys = set(workflow.keys()) self.assertEqual( workflow_keys, { 'submission_uuid', 'status', 'created', 'modified', 'score', 'assessment_score_priority' }) self.assertEqual(workflow["submission_uuid"], submission["uuid"]) self.assertEqual(workflow["status"], "training") peer_workflows = list( PeerWorkflow.objects.filter(submission_uuid=submission["uuid"])) self.assertFalse(peer_workflows) workflow_from_get = workflow_api.get_workflow_for_submission( submission["uuid"], requirements) del workflow_from_get['status_details'] self.assertEqual(workflow, workflow_from_get) requirements["training"]["num_required"] = 0 workflow = workflow_api.update_from_assessments( submission["uuid"], requirements) # New step is Peer, and a Workflow has been created. self.assertEqual(workflow["status"], "peer") peer_workflow = PeerWorkflow.objects.get( submission_uuid=submission["uuid"]) self.assertIsNotNone(peer_workflow)
def test_update_peer_workflow(self): submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod") workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"], ON_INIT_PARAMS) StudentTrainingWorkflow.create_workflow(submission_uuid=submission["uuid"]) requirements = { "training": { "num_required": 2 }, "peer": { "must_grade": 5, "must_be_graded_by": 3 } } workflow_keys = set(workflow.keys()) self.assertEqual( workflow_keys, { 'override_score', 'submission_uuid', 'uuid', 'status', 'created', 'modified', 'score' } ) self.assertEqual(workflow["submission_uuid"], submission["uuid"]) self.assertEqual(workflow["status"], "training") peer_workflows = list(PeerWorkflow.objects.filter(submission_uuid=submission["uuid"])) self.assertFalse(peer_workflows) workflow_from_get = workflow_api.get_workflow_for_submission( submission["uuid"], requirements ) del workflow_from_get['status_details'] self.assertEqual(workflow, workflow_from_get) requirements["training"]["num_required"] = 0 workflow = workflow_api.update_from_assessments(submission["uuid"], requirements) # New step is Peer, and a Workflow has been created. self.assertEqual(workflow["status"], "peer") peer_workflow = PeerWorkflow.objects.get(submission_uuid=submission["uuid"]) self.assertIsNotNone(peer_workflow)
def test_create_workflow_item_integrity_error(self, mock_create): # Create a submission and workflow submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) workflow = StudentTrainingWorkflow.create_workflow(submission['uuid']) # Simulate a race condition in which someone creates a workflow item # after we check if it exists. mock.MagicMock(StudentTrainingWorkflowItem) mock_create.side_effect = IntegrityError # Expect that we retry and retrieve the workflow item created by someone else self.assertEqual(workflow.next_training_example(EXAMPLES), EXAMPLES[0])
def test_create_workflow_item_integrity_error(self, mock_create): # Create a submission and workflow submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) workflow = StudentTrainingWorkflow.create_workflow(submission['uuid']) # Simulate a race condition in which someone creates a workflow item # after we check if it exists. mock_workflow_item = mock.MagicMock(StudentTrainingWorkflowItem) mock_create.side_effect = IntegrityError # Expect that we retry and retrieve the workflow item created by someone else self.assertEqual(workflow.next_training_example(EXAMPLES), EXAMPLES[0])
def get_training_example(submission_uuid, rubric, examples): """ Retrieve a training example for the student to assess. This will implicitly create a workflow for the student if one does not yet exist. NOTE: We include the rubric in the returned dictionary to handle the case in which the instructor changes the rubric definition while the student is assessing the training example. Once a student starts on a training example, the student should see the same training example consistently. However, the next training example the student retrieves will use the updated rubric. Args: submission_uuid (str): The UUID of the student's submission. rubric (dict): Serialized rubric model. examples (list): List of serialized training examples. Returns: dict: The training example with keys "answer", "rubric", and "options_selected". If no training examples are available (the student has already assessed every example, or no examples are defined), returns None. Raises: StudentTrainingInternalError Example usage: >>> examples = [ >>> { >>> 'answer': u'Doler', >>> 'options_selected': { >>> 'vocabulary': 'good', >>> 'grammar': 'poor' >>> } >>> } >>> ] >>> >>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples) { 'answer': u'Lorem ipsum', 'rubric': { "prompt": "Write an essay!", "criteria": [ { "order_num": 0, "name": "vocabulary", "prompt": "How varied is the vocabulary?", "options": options }, { "order_num": 1, "name": "grammar", "prompt": "How correct is the grammar?", "options": options } ], }, 'options_selected': { 'vocabulary': 'good', 'grammar': 'excellent' } } """ try: # Validate the training examples errors = validate_training_examples(rubric, examples) if len(errors) > 0: msg = ( u"Training examples do not match the rubric (submission UUID is {uuid}): {errors}" ).format(uuid=submission_uuid, errors="\n".join(errors)) raise StudentTrainingRequestError(msg) # Get or create the workflow workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid) if not workflow: raise StudentTrainingRequestError( u"No student training workflow found for submission {}".format(submission_uuid) ) # Get or create the training examples examples = deserialize_training_examples(examples, rubric) # Pick a training example that the student has not yet completed # If the student already started a training example, then return that instead. next_example = workflow.next_training_example(examples) return None if next_example is None else serialize_training_example(next_example) except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex: logger.exception( "Could not deserialize training examples for submission UUID {}".format(submission_uuid) ) raise StudentTrainingRequestError(ex) except sub_api.SubmissionNotFoundError as ex: msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid) logger.exception(msg) raise StudentTrainingRequestError(msg) except DatabaseError: msg = ( u"Could not retrieve a training example " u"for the student with submission UUID {}" ).format(submission_uuid) logger.exception(msg) raise StudentTrainingInternalError(msg)
def get_training_example(submission_uuid, rubric, examples): """ Retrieve a training example for the student to assess. This will implicitly create a workflow for the student if one does not yet exist. NOTE: We include the rubric in the returned dictionary to handle the case in which the instructor changes the rubric definition while the student is assessing the training example. Once a student starts on a training example, the student should see the same training example consistently. However, the next training example the student retrieves will use the updated rubric. Args: submission_uuid (str): The UUID of the student's submission. rubric (dict): Serialized rubric model. examples (list): List of serialized training examples. Returns: dict: The training example with keys "answer", "rubric", and "options_selected". If no training examples are available (the student has already assessed every example, or no examples are defined), returns None. Raises: StudentTrainingInternalError Example usage: >>> examples = [ >>> { >>> 'answer': { >>> 'parts': { >>> [ >>> {'text:' 'Answer part 1'}, >>> {'text:' 'Answer part 2'}, >>> {'text:' 'Answer part 3'} >>> ] >>> } >>> }, >>> 'options_selected': { >>> 'vocabulary': 'good', >>> 'grammar': 'poor' >>> } >>> } >>> ] >>> >>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples) { 'answer': { 'parts': { [ {'text:' 'Answer part 1'}, {'text:' 'Answer part 2'}, {'text:' 'Answer part 3'} ] } }, 'rubric': { "prompts": [ {"description": "Prompt 1"}, {"description": "Prompt 2"}, {"description": "Prompt 3"} ], "criteria": [ { "order_num": 0, "name": "vocabulary", "prompt": "How varied is the vocabulary?", "options": options }, { "order_num": 1, "name": "grammar", "prompt": "How correct is the grammar?", "options": options } ], }, 'options_selected': { 'vocabulary': 'good', 'grammar': 'excellent' } } """ try: # Validate the training examples errors = validate_training_examples(rubric, examples) if errors: msg = ( "Training examples do not match the rubric (submission UUID is {uuid}): {errors}" ).format(uuid=submission_uuid, errors="\n".join(errors)) raise StudentTrainingRequestError(msg) # Get or create the workflow workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid) if not workflow: raise StudentTrainingRequestError( u"No learner training workflow found for submission {}".format(submission_uuid) ) # Get or create the training examples examples = deserialize_training_examples(examples, rubric) # Pick a training example that the student has not yet completed # If the student already started a training example, then return that instead. next_example = workflow.next_training_example(examples) return None if next_example is None else serialize_training_example(next_example) except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex: logger.exception( u"Could not deserialize training examples for submission UUID {}".format(submission_uuid) ) raise StudentTrainingRequestError(ex) from ex except sub_api.SubmissionNotFoundError as ex: msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid) logger.exception(msg) raise StudentTrainingRequestError(msg) from ex except DatabaseError as ex: msg = ( u"Could not retrieve a training example for the learner with submission UUID {}" ).format(submission_uuid) logger.exception(msg) raise StudentTrainingInternalError(msg) from ex