def test_train_classifiers_invalid_examples(self): # Mutate an example so it does not match the rubric mutated_examples = copy.deepcopy(EXAMPLES) mutated_examples[0]['options_selected'] = {'invalid': 'invalid'} # Expect a request error with self.assertRaises(AITrainingRequestError): ai_api.train_classifiers(RUBRIC, mutated_examples, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_train_classifiers_celery_error(self): with mock.patch( 'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async' ) as mock_train: mock_train.side_effect = NotConfigured with self.assertRaises(AITrainingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_automatic_grade_error(self): # Create some submissions which will not succeed. No classifiers yet exist. for _ in range(0, 10): submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) ai_api.on_init(submission['uuid'], rubric=RUBRIC, algorithm_id=ALGORITHM_ID) # Check that there are unresolved grading workflows self._assert_complete(training_done=True, grading_done=False) patched_method = 'openassessment.assessment.worker.training.reschedule_grading_tasks.apply_async' with mock.patch(patched_method) as mocked_reschedule_grading: mocked_reschedule_grading.side_effect = AIGradingInternalError("Kablewey.") with self.assertRaises(AIGradingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_automatic_grade(self): # Create some submissions which will not succeed. No classifiers yet exist. for _ in range(0, 10): submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) ai_api.on_init(submission['uuid'], rubric=RUBRIC, algorithm_id=ALGORITHM_ID) # Check that there are unresolved grading workflows self._assert_complete(training_done=True, grading_done=False) # Create and train a classifier set. This should set off automatic grading. ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID) # Check to make sure that all work is done. self._assert_complete(training_done=True, grading_done=True)
def test_multiple_classifier_sets(self): # Train multiple classifiers ai_api.train_classifiers(RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID) second_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID) # Expect that we get the info for the second classifier classifier_info = ai_api.get_classifier_set_info( RUBRIC, ALGORITHM_ID, 'test_course', 'test_item') workflow = AITrainingWorkflow.objects.get(uuid=second_uuid) classifier_set = workflow.classifier_set self.assertEqual(classifier_info['created_at'], classifier_set.created_at)
def test_multiple_classifier_sets(self): # Train multiple classifiers ai_api.train_classifiers( RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID ) second_uuid = ai_api.train_classifiers( RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID ) # Expect that we get the info for the second classifier classifier_info = ai_api.get_classifier_set_info( RUBRIC, ALGORITHM_ID, 'test_course', 'test_item' ) workflow = AITrainingWorkflow.objects.get(uuid=second_uuid) classifier_set = workflow.classifier_set self.assertEqual(classifier_info['created_at'], classifier_set.created_at)
def schedule_training(self, data, suffix=''): # pylint: disable=W0613 """ Schedule a new training task for example-based grading. """ assessment = self.get_assessment_module('example-based-assessment') student_item_dict = self.get_student_item_dict() if assessment: examples = assessment["examples"] try: workflow_uuid = ai_api.train_classifiers( create_rubric_dict(self.prompts, self.rubric_criteria_with_labels), convert_training_examples_list_to_dict(examples), student_item_dict.get('course_id'), student_item_dict.get('item_id'), assessment["algorithm_id"] ) return { 'success': True, 'workflow_uuid': workflow_uuid, 'msg': self._(u"Training scheduled with new Workflow UUID: {uuid}".format(uuid=workflow_uuid)) } except AIError as err: return { 'success': False, 'msg': self._(u"An error occurred scheduling classifier training: {error}".format(error=err)) } else: return { 'success': False, 'msg': self._(u"Example Based Assessment is not configured for this location.") }
def test_automatic_grade_error(self): # Create some submissions which will not succeed. No classifiers yet exist. for _ in range(0, 10): submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) ai_api.on_init(submission['uuid'], rubric=RUBRIC, algorithm_id=ALGORITHM_ID) # Check that there are unresolved grading workflows self._assert_complete(training_done=True, grading_done=False) patched_method = 'openassessment.assessment.worker.training.reschedule_grading_tasks.apply_async' with mock.patch(patched_method) as mocked_reschedule_grading: mocked_reschedule_grading.side_effect = AIGradingInternalError( "Kablewey.") with self.assertRaises(AIGradingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def setUp(self): """ Sets up each test so that it will have unfinished tasks of both types """ # 1) Schedule Grading, have the scheduling succeeed but the grading fail because no classifiers exist for _ in range(0, 10): submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) self.submission_uuid = submission['uuid'] ai_api.on_init(self.submission_uuid, rubric=RUBRIC, algorithm_id=ALGORITHM_ID) # 2) Schedule Training, have it INTENTIONALLY fail. Now we are a point where both parts need to be rescheduled patched_method = 'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async' with mock.patch(patched_method) as mock_train_classifiers: mock_train_classifiers.side_effect = AITrainingInternalError('Training Classifiers Failed for some Reason.') with self.assertRaises(AITrainingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID) self._assert_complete(training_done=False, grading_done=False)
def setUp(self): """ Sets up each test so that it will have unfinished tasks of both types """ # 1) Schedule Grading, have the scheduling succeeed but the grading fail because no classifiers exist for _ in range(0, 10): submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) self.submission_uuid = submission['uuid'] ai_api.on_init(self.submission_uuid, rubric=RUBRIC, algorithm_id=ALGORITHM_ID) # 2) Schedule Training, have it INTENTIONALLY fail. Now we are a point where both parts need to be rescheduled patched_method = 'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async' with mock.patch(patched_method) as mock_train_classifiers: mock_train_classifiers.side_effect = AITrainingInternalError( 'Training Classifiers Failed for some Reason.') with self.assertRaises(AITrainingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID) self._assert_complete(training_done=False, grading_done=False)
def test_classifier_set_info(self): workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID) classifier_info = ai_api.get_classifier_set_info( RUBRIC, ALGORITHM_ID, 'test_course', 'test_item') # Retrieve the classifier set so we can get its actual creation date workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid) classifier_set = workflow.classifier_set expected_info = { 'created_at': classifier_set.created_at, 'algorithm_id': ALGORITHM_ID, 'course_id': 'test_course', 'item_id': 'test_item' } self.assertEqual(classifier_info, expected_info)
def test_classifier_set_info(self): workflow_uuid = ai_api.train_classifiers( RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID ) classifier_info = ai_api.get_classifier_set_info( RUBRIC, ALGORITHM_ID, 'test_course', 'test_item' ) # Retrieve the classifier set so we can get its actual creation date workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid) classifier_set = workflow.classifier_set expected_info = { 'created_at': classifier_set.created_at, 'algorithm_id': ALGORITHM_ID, 'course_id': 'test_course', 'item_id': 'test_item' } self.assertEqual(classifier_info, expected_info)
def test_train_classifiers_all_feedback_only_criteria(self): # Modify the rubric to include only feedback-only criteria # (a criterion with no options, just written feedback) rubric = copy.deepcopy(RUBRIC) for criterion in rubric['criteria']: criterion['options'] = [] # Modify the training examples to provide no scores examples = copy.deepcopy(EXAMPLES) for example in examples: example['options_selected'] = {} # Schedule a training task # Our training examples have no options workflow_uuid = ai_api.train_classifiers(rubric, examples, COURSE_ID, ITEM_ID, ALGORITHM_ID) # Verify that no classifier was created for the feedback-only criteria workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid) classifier_data = workflow.classifier_set.classifier_data_by_criterion self.assertEqual(classifier_data, {})
def test_train_classifiers_feedback_only_criterion(self): # Modify the rubric to include a feedback-only criterion # (a criterion with no options, just written feedback) rubric = copy.deepcopy(RUBRIC) rubric['criteria'].append({ 'name': 'feedback only', 'prompt': 'feedback', 'options': [] }) # Schedule a training task # (we use training examples that do NOT include the feedback-only criterion) workflow_uuid = ai_api.train_classifiers(rubric, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID) # Verify that no classifier was created for the feedback-only criterion # Since there's no points associated with that criterion, # there's no way for the AI algorithm to score it anyway. workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid) classifier_data = workflow.classifier_set.classifier_data_by_criterion self.assertNotIn('feedback only', classifier_data)
def test_train_classifiers(self): # Schedule a training task # Because Celery is configured in "always eager" mode, # expect the task to be executed synchronously. workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID) # Retrieve the classifier set from the database workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid) classifier_set = workflow.classifier_set self.assertIsNot(classifier_set, None) # Retrieve a dictionary mapping criteria names to deserialized classifiers classifiers = classifier_set.classifier_data_by_criterion # Check that we have classifiers for all criteria in the rubric criteria = set(criterion['name'] for criterion in RUBRIC['criteria']) self.assertEqual(set(classifiers.keys()), criteria) # Check that the classifier data matches the data from our stub AI algorithm # Since the stub data includes the training examples, we also verify # that the classifier was trained using the correct examples. for criterion in RUBRIC['criteria']: classifier = classifiers[criterion['name']] self.assertEqual(classifier['name'], StubAIAlgorithm.FAKE_CLASSIFIER['name']) self.assertEqual(classifier['binary_content'], StubAIAlgorithm.FAKE_CLASSIFIER['binary_content']) # Verify that the correct essays and scores were used to create the classifier # Our stub AI algorithm provides these for us, but they would not ordinarily # be included in the trained classifier. self.assertEqual(len(classifier['examples']), len(EXAMPLES)) expected_scores = self.EXPECTED_INPUT_SCORES[criterion['name']] for data in zip(EXAMPLES, classifier['examples'], expected_scores): sent_example, received_example, expected_score = data received_example = AIAlgorithm.ExampleEssay(*received_example) self.assertEqual(received_example.text, sent_example['answer']) self.assertEqual(received_example.score, expected_score)
def test_train_classifiers_no_examples(self): # Empty list of training examples with self.assertRaises(AITrainingRequestError): ai_api.train_classifiers(RUBRIC, [], COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_start_workflow_database_error(self, mock_create): # Simulate a database error when creating the training workflow mock_create.side_effect = DatabaseError("KABOOM!") with self.assertRaises(AITrainingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_train_classifiers_celery_error(self): with mock.patch('openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async') as mock_train: mock_train.side_effect = NotConfigured with self.assertRaises(AITrainingInternalError): ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)