Ejemplo n.º 1
0
def on_start(submission_uuid):
    """
    Creates a new student training workflow.

    This function should be called to indicate that a submission has entered the
    student training workflow part of the assessment process.

    Args:
        submission_uuid (str): The submission UUID for the student that is
            initiating training.

    Returns:
        None

    Raises:
        StudentTrainingInternalError: Raised when an error occurs persisting the
            Student Training Workflow
    """
    try:
        StudentTrainingWorkflow.create_workflow(submission_uuid)
    except Exception as ex:
        msg = (
            "An internal error has occurred while creating the learner "
            "training workflow for submission UUID {}".format(submission_uuid)
        )
        logger.exception(msg)
        raise StudentTrainingInternalError(msg) from ex
Ejemplo n.º 2
0
def on_start(submission_uuid):
    """
    Creates a new student training workflow.

    This function should be called to indicate that a submission has entered the
    student training workflow part of the assessment process.

    Args:
        submission_uuid (str): The submission UUID for the student that is
            initiating training.

    Returns:
        None

    Raises:
        StudentTrainingInternalError: Raised when an error occurs persisting the
            Student Training Workflow
    """
    try:
        StudentTrainingWorkflow.create_workflow(submission_uuid)
    except Exception:
        msg = (
            u"An internal error has occurred while creating the student "
            u"training workflow for submission UUID {}".format(submission_uuid)
        )
        logger.exception(msg)
        raise StudentTrainingInternalError(msg)
    def test_create_workflow_integrity_error(self, mock_create, mock_get):
        # Simulate a race condition in which someone creates a workflow
        # after we check if it exists.  This will violate the database uniqueness
        # constraints, so we need to handle this case gracefully.
        mock_create.side_effect = IntegrityError

        # The first time we check, we should see that no workflow exists.
        # The second time, we should get the workflow created by someone else
        mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
        mock_get.side_effect = [mock_workflow]

        # Expect that we retry and retrieve the workflow that someone else created
        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
        StudentTrainingWorkflow.create_workflow(submission['uuid'])
        workflow = StudentTrainingWorkflow.get_workflow(submission['uuid'])
        self.assertEqual(workflow, mock_workflow)
    def test_create_workflow_integrity_error(self, mock_create, mock_get):
        # Simulate a race condition in which someone creates a workflow
        # after we check if it exists.  This will violate the database uniqueness
        # constraints, so we need to handle this case gracefully.
        mock_create.side_effect = IntegrityError

        # The first time we check, we should see that no workflow exists.
        # The second time, we should get the workflow created by someone else
        mock_workflow = mock.MagicMock(StudentTrainingWorkflow)
        mock_get.side_effect = [
            mock_workflow
        ]

        # Expect that we retry and retrieve the workflow that someone else created
        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
        StudentTrainingWorkflow.create_workflow(submission['uuid'])
        workflow = StudentTrainingWorkflow.get_workflow(submission['uuid'])
        self.assertEqual(workflow, mock_workflow)
Ejemplo n.º 5
0
    def test_update_peer_workflow(self):
        submission = sub_api.create_submission(ITEM_1, ANSWER_1)
        workflow = workflow_api.create_workflow(submission["uuid"],
                                                ["training", "peer"])
        StudentTrainingWorkflow.create_workflow(
            submission_uuid=submission["uuid"])
        requirements = {
            "training": {
                "num_required": 2
            },
            "peer": {
                "must_grade": 5,
                "must_be_graded_by": 3
            }
        }
        workflow_keys = set(workflow.keys())
        self.assertEqual(
            workflow_keys, {
                'submission_uuid', 'status', 'created', 'modified', 'score',
                'assessment_score_priority'
            })
        self.assertEqual(workflow["submission_uuid"], submission["uuid"])
        self.assertEqual(workflow["status"], "training")

        peer_workflows = list(
            PeerWorkflow.objects.filter(submission_uuid=submission["uuid"]))
        self.assertFalse(peer_workflows)

        workflow_from_get = workflow_api.get_workflow_for_submission(
            submission["uuid"], requirements)

        del workflow_from_get['status_details']
        self.assertEqual(workflow, workflow_from_get)

        requirements["training"]["num_required"] = 0
        workflow = workflow_api.update_from_assessments(
            submission["uuid"], requirements)

        # New step is Peer, and a Workflow has been created.
        self.assertEqual(workflow["status"], "peer")
        peer_workflow = PeerWorkflow.objects.get(
            submission_uuid=submission["uuid"])
        self.assertIsNotNone(peer_workflow)
Ejemplo n.º 6
0
    def test_update_peer_workflow(self):
        submission = sub_api.create_submission(ITEM_1, "Shoot Hot Rod")
        workflow = workflow_api.create_workflow(submission["uuid"], ["training", "peer"], ON_INIT_PARAMS)
        StudentTrainingWorkflow.create_workflow(submission_uuid=submission["uuid"])
        requirements = {
            "training": {
                "num_required": 2
            },
            "peer": {
                "must_grade": 5,
                "must_be_graded_by": 3
            }
        }
        workflow_keys = set(workflow.keys())
        self.assertEqual(
            workflow_keys,
            {
                'override_score',
                'submission_uuid', 'uuid', 'status', 'created', 'modified', 'score'
            }
        )
        self.assertEqual(workflow["submission_uuid"], submission["uuid"])
        self.assertEqual(workflow["status"], "training")

        peer_workflows = list(PeerWorkflow.objects.filter(submission_uuid=submission["uuid"]))
        self.assertFalse(peer_workflows)

        workflow_from_get = workflow_api.get_workflow_for_submission(
            submission["uuid"], requirements
        )

        del workflow_from_get['status_details']
        self.assertEqual(workflow, workflow_from_get)

        requirements["training"]["num_required"] = 0
        workflow = workflow_api.update_from_assessments(submission["uuid"], requirements)

        # New step is Peer, and a Workflow has been created.
        self.assertEqual(workflow["status"], "peer")
        peer_workflow = PeerWorkflow.objects.get(submission_uuid=submission["uuid"])
        self.assertIsNotNone(peer_workflow)
    def test_create_workflow_item_integrity_error(self, mock_create):
        # Create a submission and workflow
        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
        workflow = StudentTrainingWorkflow.create_workflow(submission['uuid'])

        # Simulate a race condition in which someone creates a workflow item
        # after we check if it exists.
        mock.MagicMock(StudentTrainingWorkflowItem)
        mock_create.side_effect = IntegrityError

        # Expect that we retry and retrieve the workflow item created by someone else
        self.assertEqual(workflow.next_training_example(EXAMPLES), EXAMPLES[0])
    def test_create_workflow_item_integrity_error(self, mock_create):
        # Create a submission and workflow
        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
        workflow = StudentTrainingWorkflow.create_workflow(submission['uuid'])

        # Simulate a race condition in which someone creates a workflow item
        # after we check if it exists.
        mock_workflow_item = mock.MagicMock(StudentTrainingWorkflowItem)
        mock_create.side_effect = IntegrityError

        # Expect that we retry and retrieve the workflow item created by someone else
        self.assertEqual(workflow.next_training_example(EXAMPLES), EXAMPLES[0])
Ejemplo n.º 9
0
def get_training_example(submission_uuid, rubric, examples):
    """
    Retrieve a training example for the student to assess.
    This will implicitly create a workflow for the student if one does not yet exist.

    NOTE: We include the rubric in the returned dictionary to handle
    the case in which the instructor changes the rubric definition
    while the student is assessing the training example.  Once a student
    starts on a training example, the student should see the same training
    example consistently.  However, the next training example the student
    retrieves will use the updated rubric.

    Args:
        submission_uuid (str): The UUID of the student's submission.
        rubric (dict): Serialized rubric model.
        examples (list): List of serialized training examples.

    Returns:
        dict: The training example with keys "answer", "rubric", and "options_selected".
        If no training examples are available (the student has already assessed every example,
            or no examples are defined), returns None.

    Raises:
        StudentTrainingInternalError

    Example usage:

        >>> examples = [
        >>>     {
        >>>         'answer': u'Doler',
        >>>         'options_selected': {
        >>>             'vocabulary': 'good',
        >>>             'grammar': 'poor'
        >>>         }
        >>>     }
        >>> ]
        >>>
        >>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
        {
            'answer': u'Lorem ipsum',
            'rubric': {
                "prompt": "Write an essay!",
                "criteria": [
                    {
                        "order_num": 0,
                        "name": "vocabulary",
                        "prompt": "How varied is the vocabulary?",
                        "options": options
                    },
                    {
                        "order_num": 1,
                        "name": "grammar",
                        "prompt": "How correct is the grammar?",
                        "options": options
                    }
                ],
            },
            'options_selected': {
                'vocabulary': 'good',
                'grammar': 'excellent'
            }
        }

    """
    try:
        # Validate the training examples
        errors = validate_training_examples(rubric, examples)
        if len(errors) > 0:
            msg = (
                u"Training examples do not match the rubric (submission UUID is {uuid}): {errors}"
            ).format(uuid=submission_uuid, errors="\n".join(errors))
            raise StudentTrainingRequestError(msg)

        # Get or create the workflow
        workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid)
        if not workflow:
            raise StudentTrainingRequestError(
                u"No student training workflow found for submission {}".format(submission_uuid)
            )

        # Get or create the training examples
        examples = deserialize_training_examples(examples, rubric)

        # Pick a training example that the student has not yet completed
        # If the student already started a training example, then return that instead.
        next_example = workflow.next_training_example(examples)
        return None if next_example is None else serialize_training_example(next_example)
    except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex:
        logger.exception(
            "Could not deserialize training examples for submission UUID {}".format(submission_uuid)
        )
        raise StudentTrainingRequestError(ex)
    except sub_api.SubmissionNotFoundError as ex:
        msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid)
        logger.exception(msg)
        raise StudentTrainingRequestError(msg)
    except DatabaseError:
        msg = (
            u"Could not retrieve a training example "
            u"for the student with submission UUID {}"
        ).format(submission_uuid)
        logger.exception(msg)
        raise StudentTrainingInternalError(msg)
Ejemplo n.º 10
0
def get_training_example(submission_uuid, rubric, examples):
    """
    Retrieve a training example for the student to assess.
    This will implicitly create a workflow for the student if one does not yet exist.

    NOTE: We include the rubric in the returned dictionary to handle
    the case in which the instructor changes the rubric definition
    while the student is assessing the training example.  Once a student
    starts on a training example, the student should see the same training
    example consistently.  However, the next training example the student
    retrieves will use the updated rubric.

    Args:
        submission_uuid (str): The UUID of the student's submission.
        rubric (dict): Serialized rubric model.
        examples (list): List of serialized training examples.

    Returns:
        dict: The training example with keys "answer", "rubric", and "options_selected".
        If no training examples are available (the student has already assessed every example,
            or no examples are defined), returns None.

    Raises:
        StudentTrainingInternalError

    Example usage:

        >>> examples = [
        >>>     {
        >>>         'answer': {
        >>>             'parts': {
        >>>                 [
        >>>                     {'text:' 'Answer part 1'},
        >>>                     {'text:' 'Answer part 2'},
        >>>                     {'text:' 'Answer part 3'}
        >>>                 ]
        >>>             }
        >>>         },
        >>>         'options_selected': {
        >>>             'vocabulary': 'good',
        >>>             'grammar': 'poor'
        >>>         }
        >>>     }
        >>> ]
        >>>
        >>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
        {
             'answer': {
                 'parts': {
                     [
                         {'text:' 'Answer part 1'},
                         {'text:' 'Answer part 2'},
                         {'text:' 'Answer part 3'}
                     ]
                 }
             },
            'rubric': {
                "prompts": [
                    {"description": "Prompt 1"},
                    {"description": "Prompt 2"},
                    {"description": "Prompt 3"}
                ],
                "criteria": [
                    {
                        "order_num": 0,
                        "name": "vocabulary",
                        "prompt": "How varied is the vocabulary?",
                        "options": options
                    },
                    {
                        "order_num": 1,
                        "name": "grammar",
                        "prompt": "How correct is the grammar?",
                        "options": options
                    }
                ],
            },
            'options_selected': {
                'vocabulary': 'good',
                'grammar': 'excellent'
            }
        }

    """
    try:
        # Validate the training examples
        errors = validate_training_examples(rubric, examples)
        if errors:
            msg = (
                "Training examples do not match the rubric (submission UUID is {uuid}): {errors}"
            ).format(uuid=submission_uuid, errors="\n".join(errors))
            raise StudentTrainingRequestError(msg)

        # Get or create the workflow
        workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid)
        if not workflow:
            raise StudentTrainingRequestError(
                u"No learner training workflow found for submission {}".format(submission_uuid)
            )

        # Get or create the training examples
        examples = deserialize_training_examples(examples, rubric)

        # Pick a training example that the student has not yet completed
        # If the student already started a training example, then return that instead.
        next_example = workflow.next_training_example(examples)
        return None if next_example is None else serialize_training_example(next_example)
    except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex:
        logger.exception(
            u"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
        )
        raise StudentTrainingRequestError(ex) from ex
    except sub_api.SubmissionNotFoundError as ex:
        msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid)
        logger.exception(msg)
        raise StudentTrainingRequestError(msg) from ex
    except DatabaseError as ex:
        msg = (
            u"Could not retrieve a training example for the learner with submission UUID {}"
        ).format(submission_uuid)
        logger.exception(msg)
        raise StudentTrainingInternalError(msg) from ex