def test_get_training_example_invalid_rubric(self): # Rubric is missing a very important key! invalid_rubric = copy.deepcopy(RUBRIC) del invalid_rubric['criteria'] with self.assertRaises(StudentTrainingRequestError): training_api.get_training_example(self.submission_uuid, invalid_rubric, EXAMPLES)
def test_assess_training_example_database_error(self, mock_db): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) mock_db.side_effect = DatabaseError("Kaboom!") with self.assertRaises(StudentTrainingInternalError): training_api.assess_training_example( self.submission_uuid, EXAMPLES[0]['options_selected'])
def test_get_num_completed_num_queries(self): # Complete the first training example training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected']) # Check the number completed with self.assertNumQueries(2): training_api.get_num_completed(self.submission_uuid)
def test_get_same_example(self): # Retrieve a training example retrieved = training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) # If we retrieve an example without completing the current example, # we should get the same one. next_retrieved = training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) self.assertEqual(retrieved, next_retrieved)
def test_assess_training_example_num_queries(self): # Populate the cache with training examples and rubrics self._warm_cache(RUBRIC, EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) with self.assertNumQueries(3): training_api.assess_training_example( self.submission_uuid, EXAMPLES[0]['options_selected'])
def test_submitter_is_finished_num_queries(self): # Complete the first training example training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected']) # Check whether we've completed the requirements requirements = {'num_required': 2} with self.assertNumQueries(2): training_api.submitter_is_finished(self.submission_uuid, requirements)
def test_assess_training_example_completed_workflow(self): for example in EXAMPLES: training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) training_api.assess_training_example(self.submission_uuid, example['options_selected']) # Try to assess again, and expect an error with self.assertRaises(StudentTrainingRequestError): training_api.assess_training_example( self.submission_uuid, EXAMPLES[0]['options_selected'] )
def test_submitter_is_finished_num_queries(self): # Complete the first training example training_api.on_start(self.submission_uuid) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected']) # Check whether we've completed the requirements requirements = {'num_required': 2} with self.assertNumQueries(2): training_api.submitter_is_finished(self.submission_uuid, requirements)
def test_assess_training_example_database_error(self): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) with patch( 'openassessment.assessment.models.student_training.StudentTrainingWorkflow.objects.get' ) as mock_db: mock_db.side_effect = DatabaseError("Kaboom!") with self.assertRaises(StudentTrainingInternalError): training_api.assess_training_example( self.submission_uuid, EXAMPLES[0]['options_selected'])
def test_get_training_example_none_available(self): for example in EXAMPLES: training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) training_api.assess_training_example(self.submission_uuid, example['options_selected']) # Now we should be complete self._assert_workflow_status(self.submission_uuid, 2, 2) # ... and if we try to get another example, we should get None self.assertIs( training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES), None )
def test_assess_without_update(self): # Assess the first training example the same way the instructor did # but do NOT update the workflow training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) corrections = training_api.assess_training_example( self.submission_uuid, EXAMPLES[0]['options_selected'], update_workflow=False) # Expect that we're still on the first step self.assertEqual(corrections, dict()) self._assert_workflow_status(self.submission_uuid, 0, 2)
def test_assess_without_update(self): # Assess the first training example the same way the instructor did # but do NOT update the workflow training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) corrections = training_api.assess_training_example( self.submission_uuid, EXAMPLES[0]['options_selected'], update_workflow=False ) # Expect that we're still on the first step self.assertEqual(corrections, dict()) self._assert_workflow_status(self.submission_uuid, 0, 2)
def _warm_cache(self, rubric, examples): """ Create a submission and complete student training. This will populate the cache with training examples and rubrics, which are immutable and shared for all students training on a particular problem. Args: rubric (dict): Serialized rubric model. examples (list of dict): Serialized training examples Returns: None """ pre_submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) for example in examples: training_api.get_training_example(pre_submission['uuid'], rubric, examples) training_api.assess_training_example(pre_submission['uuid'], example['options_selected'])
def _warm_cache(self, rubric, examples): """ Create a submission and complete student training. This will populate the cache with training examples and rubrics, which are immutable and shared for all students training on a particular problem. Args: rubric (dict): Serialized rubric model. examples (list of dict): Serialized training examples Returns: None """ pre_submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) training_api.on_start(pre_submission['uuid']) for example in examples: training_api.get_training_example(pre_submission['uuid'], rubric, examples) training_api.assess_training_example(pre_submission['uuid'], example['options_selected'])
def test_get_training_example_num_queries(self): # Run through the training example once using a different submission # Training examples and rubrics will be cached and shared for other # students working on the same problem. self._warm_cache(RUBRIC, EXAMPLES) # First training example # This will need to create the student training workflow and the first item # NOTE: we *could* cache the rubric model to reduce the number of queries here, # but we're selecting it by content hash, which is indexed and should be plenty fast. with self.assertNumQueries(8): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) # Without assessing the first training example, try to retrieve a training example. # This should return the same example as before, so we won't need to create # any workflows or workflow items. with self.assertNumQueries(5): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) # Assess the current training example training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected']) # Retrieve the next training example, which requires us to create # a new workflow item (but not a new workflow). with self.assertNumQueries(8): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
def test_get_training_example_num_queries(self): # Run through the training example once using a different submission # Training examples and rubrics will be cached and shared for other # students working on the same problem. self._warm_cache(RUBRIC, EXAMPLES) # First training example # This will need to create the student training workflow and the first item # NOTE: we *could* cache the rubric model to reduce the number of queries here, # but we're selecting it by content hash, which is indexed and should be plenty fast. with self.assertNumQueries(6): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) # Without assessing the first training example, try to retrieve a training example. # This should return the same example as before, so we won't need to create # any workflows or workflow items. with self.assertNumQueries(3): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) # Assess the current training example training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected']) # Retrieve the next training example, which requires us to create # a new workflow item (but not a new workflow). with self.assertNumQueries(4): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES)
def _assert_get_example(self, submission_uuid, order_num, input_examples, input_rubric): """ Check the training example we get from the API. Args: submission_uuid (str): The submission UUID associated with the student being trained. order_num (int): The order number of the example we expect to retrieve. input_examples (list of dict): The examples we used to configure the training workflow. input_rubric (dict): The rubric we used to configure the training workflow. Returns: None Raises: AssertionError """ example = training_api.get_training_example(submission_uuid, input_rubric, input_examples) expected_example = self._expected_example(input_examples[order_num], input_rubric) self.assertItemsEqual(example, expected_example)
def _assert_get_example(self, submission_uuid, order_num, input_examples, input_rubric): """ Check the training example we get from the API. Args: submission_uuid (str): The submission UUID associated with the student being trained. order_num (int): The order number of the example we expect to retrieve. input_examples (list of dict): The examples we used to configure the training workflow. input_rubric (dict): The rubric we used to configure the training workflow. Returns: None Raises: AssertionError """ example = training_api.get_training_example(submission_uuid, input_rubric, input_examples) expected_example = self._expected_example(input_examples[order_num], input_rubric) six.assertCountEqual(self, example, expected_example)
def training_path_and_context(self): """ Return the template path and context used to render the student training step. Returns: tuple of `(path, context)` where `path` is the path to the template and `context` is a dict. """ # Retrieve the status of the workflow. # If no submissions have been created yet, the status will be None. workflow_status = self.get_workflow_info().get('status') problem_closed, reason, start_date, due_date = self.is_closed( step="student-training") context = {} template = 'openassessmentblock/student_training/student_training_unavailable.html' # add allow_latex field to the context context['allow_latex'] = self.allow_latex if not workflow_status: return template, context # If the student has completed the training step, then show that the step is complete. # We put this condition first so that if a student has completed the step, it *always* # shows as complete. # We're assuming here that the training step always precedes the other assessment steps # (peer/self) -- we may need to make this more flexible later. if workflow_status == 'cancelled': template = 'openassessmentblock/student_training/student_training_cancelled.html' elif workflow_status and workflow_status != "training": template = 'openassessmentblock/student_training/student_training_complete.html' # If the problem is closed, then do not allow students to access the training step elif problem_closed and reason == 'start': context['training_start'] = start_date template = 'openassessmentblock/student_training/student_training_unavailable.html' elif problem_closed and reason == 'due': context['training_due'] = due_date template = 'openassessmentblock/student_training/student_training_closed.html' # If we're on the training step, show the student an example # We do this last so we can avoid querying the student training API if possible. else: training_module = self.get_assessment_module('student-training') if not training_module: return template, context if due_date < DISTANT_FUTURE: context['training_due'] = due_date # Report progress in the student training workflow (completed X out of Y) context['training_num_available'] = len( training_module["examples"]) context[ 'training_num_completed'] = student_training.get_num_completed( self.submission_uuid) context[ 'training_num_current'] = context['training_num_completed'] + 1 # Retrieve the example essay for the student to submit # This will contain the essay text, the rubric, and the options the instructor selected. examples = convert_training_examples_list_to_dict( training_module["examples"]) example = student_training.get_training_example( self.submission_uuid, { 'prompt': self.prompt, 'criteria': self.rubric_criteria_with_labels }, examples) if example: context['training_essay'] = create_submission_dict( {'answer': example['answer']}, self.prompts) context['training_rubric'] = { 'criteria': example['rubric']['criteria'], 'points_possible': example['rubric']['points_possible'] } template = 'openassessmentblock/student_training/student_training.html' else: logger.error( "No training example was returned from the API for student " "with Submission UUID {}".format(self.submission_uuid)) template = "openassessmentblock/student_training/student_training_error.html" return template, context
def training_path_and_context(self): """ Return the template path and context used to render the student training step. Returns: tuple of `(path, context)` where `path` is the path to the template and `context` is a dict. """ # Retrieve the status of the workflow. # If no submissions have been created yet, the status will be None. workflow_status = self.get_workflow_info().get('status') problem_closed, reason, start_date, due_date = self.is_closed(step="student-training") context = {} template = 'openassessmentblock/student_training/student_training_unavailable.html' # add allow_latex field to the context context['allow_latex'] = self.allow_latex if not workflow_status: return template, context # If the student has completed the training step, then show that the step is complete. # We put this condition first so that if a student has completed the step, it *always* # shows as complete. # We're assuming here that the training step always precedes the other assessment steps # (peer/self) -- we may need to make this more flexible later. if workflow_status == 'cancelled': template = 'openassessmentblock/student_training/student_training_cancelled.html' elif workflow_status and workflow_status != "training": template = 'openassessmentblock/student_training/student_training_complete.html' # If the problem is closed, then do not allow students to access the training step elif problem_closed and reason == 'start': context['training_start'] = start_date template = 'openassessmentblock/student_training/student_training_unavailable.html' elif problem_closed and reason == 'due': context['training_due'] = due_date template = 'openassessmentblock/student_training/student_training_closed.html' # If we're on the training step, show the student an example # We do this last so we can avoid querying the student training API if possible. else: training_module = self.get_assessment_module('student-training') if not training_module: return template, context if due_date < DISTANT_FUTURE: context['training_due'] = due_date # Report progress in the student training workflow (completed X out of Y) context['training_num_available'] = len(training_module["examples"]) context['training_num_completed'] = student_training.get_num_completed(self.submission_uuid) context['training_num_current'] = context['training_num_completed'] + 1 # Retrieve the example essay for the student to submit # This will contain the essay text, the rubric, and the options the instructor selected. examples = convert_training_examples_list_to_dict(training_module["examples"]) example = student_training.get_training_example( self.submission_uuid, { 'prompt': self.prompt, 'criteria': self.rubric_criteria_with_labels }, examples ) if example: context['training_essay'] = create_submission_dict({'answer': example['answer']}, self.prompts) context['training_rubric'] = { 'criteria': example['rubric']['criteria'], 'points_possible': example['rubric']['points_possible'] } template = 'openassessmentblock/student_training/student_training.html' else: logger.error( "No training example was returned from the API for student " "with Submission UUID {}".format(self.submission_uuid) ) template = "openassessmentblock/student_training/student_training_error.html" return template, context
def test_get_training_example_no_submission(self): with self.assertRaises(StudentTrainingRequestError): training_api.get_training_example("no_such_submission", RUBRIC, EXAMPLES)
def test_assess_training_example_num_queries(self): # Populate the cache with training examples and rubrics self._warm_cache(RUBRIC, EXAMPLES) training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) with self.assertNumQueries(4): training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])
def test_assess_training_example_database_error(self, mock_db): training_api.get_training_example(self.submission_uuid, RUBRIC, EXAMPLES) mock_db.side_effect = DatabaseError("Kaboom!") with self.assertRaises(StudentTrainingInternalError): training_api.assess_training_example(self.submission_uuid, EXAMPLES[0]['options_selected'])