def test_train_classifiers_invalid_examples(self):
# Mutate an example so it does not match the rubric
mutated_examples = copy.deepcopy(EXAMPLES)
mutated_examples[0]['options_selected'] = {'invalid': 'invalid'}
# Expect a request error
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, mutated_examples, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_train_classifiers_celery_error(self):
with mock.patch(
'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async'
) as mock_train:
mock_train.side_effect = NotConfigured
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID,
ALGORITHM_ID)
def test_train_classifiers_invalid_examples(self):
# Mutate an example so it does not match the rubric
mutated_examples = copy.deepcopy(EXAMPLES)
mutated_examples[0]['options_selected'] = {'invalid': 'invalid'}
# Expect a request error
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, mutated_examples, COURSE_ID,
ITEM_ID, ALGORITHM_ID)
def test_automatic_grade_error(self):
# Create some submissions which will not succeed. No classifiers yet exist.
for _ in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
ai_api.on_init(submission['uuid'], rubric=RUBRIC, algorithm_id=ALGORITHM_ID)
# Check that there are unresolved grading workflows
self._assert_complete(training_done=True, grading_done=False)
patched_method = 'openassessment.assessment.worker.training.reschedule_grading_tasks.apply_async'
with mock.patch(patched_method) as mocked_reschedule_grading:
mocked_reschedule_grading.side_effect = AIGradingInternalError("Kablewey.")
with self.assertRaises(AIGradingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_automatic_grade(self):
# Create some submissions which will not succeed. No classifiers yet exist.
for _ in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
ai_api.on_init(submission['uuid'], rubric=RUBRIC, algorithm_id=ALGORITHM_ID)
# Check that there are unresolved grading workflows
self._assert_complete(training_done=True, grading_done=False)
# Create and train a classifier set. This should set off automatic grading.
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
# Check to make sure that all work is done.
self._assert_complete(training_done=True, grading_done=True)
def test_multiple_classifier_sets(self):
# Train multiple classifiers
ai_api.train_classifiers(RUBRIC, EXAMPLES, 'test_course', 'test_item',
ALGORITHM_ID)
second_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, 'test_course',
'test_item', ALGORITHM_ID)
# Expect that we get the info for the second classifier
classifier_info = ai_api.get_classifier_set_info(
RUBRIC, ALGORITHM_ID, 'test_course', 'test_item')
workflow = AITrainingWorkflow.objects.get(uuid=second_uuid)
classifier_set = workflow.classifier_set
self.assertEqual(classifier_info['created_at'],
classifier_set.created_at)
def test_multiple_classifier_sets(self):
# Train multiple classifiers
ai_api.train_classifiers(
RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID
)
second_uuid = ai_api.train_classifiers(
RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID
)
# Expect that we get the info for the second classifier
classifier_info = ai_api.get_classifier_set_info(
RUBRIC, ALGORITHM_ID, 'test_course', 'test_item'
)
workflow = AITrainingWorkflow.objects.get(uuid=second_uuid)
classifier_set = workflow.classifier_set
self.assertEqual(classifier_info['created_at'], classifier_set.created_at)
def schedule_training(self, data, suffix=''): # pylint: disable=W0613
"""
Schedule a new training task for example-based grading.
"""
assessment = self.get_assessment_module('example-based-assessment')
student_item_dict = self.get_student_item_dict()
if assessment:
examples = assessment["examples"]
try:
workflow_uuid = ai_api.train_classifiers(
create_rubric_dict(self.prompts, self.rubric_criteria_with_labels),
convert_training_examples_list_to_dict(examples),
student_item_dict.get('course_id'),
student_item_dict.get('item_id'),
assessment["algorithm_id"]
)
return {
'success': True,
'workflow_uuid': workflow_uuid,
'msg': self._(u"Training scheduled with new Workflow UUID: {uuid}".format(uuid=workflow_uuid))
}
except AIError as err:
return {
'success': False,
'msg': self._(u"An error occurred scheduling classifier training: {error}".format(error=err))
}
else:
return {
'success': False,
'msg': self._(u"Example Based Assessment is not configured for this location.")
}
def schedule_training(self, data, suffix=''): # pylint: disable=W0613
"""
Schedule a new training task for example-based grading.
"""
assessment = self.get_assessment_module('example-based-assessment')
student_item_dict = self.get_student_item_dict()
if assessment:
examples = assessment["examples"]
try:
workflow_uuid = ai_api.train_classifiers(
create_rubric_dict(self.prompts, self.rubric_criteria_with_labels),
convert_training_examples_list_to_dict(examples),
student_item_dict.get('course_id'),
student_item_dict.get('item_id'),
assessment["algorithm_id"]
)
return {
'success': True,
'workflow_uuid': workflow_uuid,
'msg': self._(u"Training scheduled with new Workflow UUID: {uuid}".format(uuid=workflow_uuid))
}
except AIError as err:
return {
'success': False,
'msg': self._(u"An error occurred scheduling classifier training: {error}".format(error=err))
}
else:
return {
'success': False,
'msg': self._(u"Example Based Assessment is not configured for this location.")
}
def test_automatic_grade(self):
# Create some submissions which will not succeed. No classifiers yet exist.
for _ in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
ai_api.on_init(submission['uuid'],
rubric=RUBRIC,
algorithm_id=ALGORITHM_ID)
# Check that there are unresolved grading workflows
self._assert_complete(training_done=True, grading_done=False)
# Create and train a classifier set. This should set off automatic grading.
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID,
ALGORITHM_ID)
# Check to make sure that all work is done.
self._assert_complete(training_done=True, grading_done=True)
def test_automatic_grade_error(self):
# Create some submissions which will not succeed. No classifiers yet exist.
for _ in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
ai_api.on_init(submission['uuid'],
rubric=RUBRIC,
algorithm_id=ALGORITHM_ID)
# Check that there are unresolved grading workflows
self._assert_complete(training_done=True, grading_done=False)
patched_method = 'openassessment.assessment.worker.training.reschedule_grading_tasks.apply_async'
with mock.patch(patched_method) as mocked_reschedule_grading:
mocked_reschedule_grading.side_effect = AIGradingInternalError(
"Kablewey.")
with self.assertRaises(AIGradingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID,
ALGORITHM_ID)
def setUp(self):
"""
Sets up each test so that it will have unfinished tasks of both types
"""
# 1) Schedule Grading, have the scheduling succeeed but the grading fail because no classifiers exist
for _ in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
ai_api.on_init(self.submission_uuid, rubric=RUBRIC, algorithm_id=ALGORITHM_ID)
# 2) Schedule Training, have it INTENTIONALLY fail. Now we are a point where both parts need to be rescheduled
patched_method = 'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async'
with mock.patch(patched_method) as mock_train_classifiers:
mock_train_classifiers.side_effect = AITrainingInternalError('Training Classifiers Failed for some Reason.')
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
self._assert_complete(training_done=False, grading_done=False)
def setUp(self):
"""
Sets up each test so that it will have unfinished tasks of both types
"""
# 1) Schedule Grading, have the scheduling succeeed but the grading fail because no classifiers exist
for _ in range(0, 10):
submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
self.submission_uuid = submission['uuid']
ai_api.on_init(self.submission_uuid,
rubric=RUBRIC,
algorithm_id=ALGORITHM_ID)
# 2) Schedule Training, have it INTENTIONALLY fail. Now we are a point where both parts need to be rescheduled
patched_method = 'openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async'
with mock.patch(patched_method) as mock_train_classifiers:
mock_train_classifiers.side_effect = AITrainingInternalError(
'Training Classifiers Failed for some Reason.')
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID,
ALGORITHM_ID)
self._assert_complete(training_done=False, grading_done=False)
def test_classifier_set_info(self):
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES,
'test_course', 'test_item',
ALGORITHM_ID)
classifier_info = ai_api.get_classifier_set_info(
RUBRIC, ALGORITHM_ID, 'test_course', 'test_item')
# Retrieve the classifier set so we can get its actual creation date
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_set = workflow.classifier_set
expected_info = {
'created_at': classifier_set.created_at,
'algorithm_id': ALGORITHM_ID,
'course_id': 'test_course',
'item_id': 'test_item'
}
self.assertEqual(classifier_info, expected_info)
def test_classifier_set_info(self):
workflow_uuid = ai_api.train_classifiers(
RUBRIC, EXAMPLES, 'test_course', 'test_item', ALGORITHM_ID
)
classifier_info = ai_api.get_classifier_set_info(
RUBRIC, ALGORITHM_ID, 'test_course', 'test_item'
)
# Retrieve the classifier set so we can get its actual creation date
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_set = workflow.classifier_set
expected_info = {
'created_at': classifier_set.created_at,
'algorithm_id': ALGORITHM_ID,
'course_id': 'test_course',
'item_id': 'test_item'
}
self.assertEqual(classifier_info, expected_info)
def test_train_classifiers_all_feedback_only_criteria(self):
# Modify the rubric to include only feedback-only criteria
# (a criterion with no options, just written feedback)
rubric = copy.deepcopy(RUBRIC)
for criterion in rubric['criteria']:
criterion['options'] = []
# Modify the training examples to provide no scores
examples = copy.deepcopy(EXAMPLES)
for example in examples:
example['options_selected'] = {}
# Schedule a training task
# Our training examples have no options
workflow_uuid = ai_api.train_classifiers(rubric, examples, COURSE_ID, ITEM_ID, ALGORITHM_ID)
# Verify that no classifier was created for the feedback-only criteria
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_data = workflow.classifier_set.classifier_data_by_criterion
self.assertEqual(classifier_data, {})
def test_train_classifiers_feedback_only_criterion(self):
# Modify the rubric to include a feedback-only criterion
# (a criterion with no options, just written feedback)
rubric = copy.deepcopy(RUBRIC)
rubric['criteria'].append({
'name': 'feedback only',
'prompt': 'feedback',
'options': []
})
# Schedule a training task
# (we use training examples that do NOT include the feedback-only criterion)
workflow_uuid = ai_api.train_classifiers(rubric, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
# Verify that no classifier was created for the feedback-only criterion
# Since there's no points associated with that criterion,
# there's no way for the AI algorithm to score it anyway.
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_data = workflow.classifier_set.classifier_data_by_criterion
self.assertNotIn('feedback only', classifier_data)
def test_train_classifiers_feedback_only_criterion(self):
# Modify the rubric to include a feedback-only criterion
# (a criterion with no options, just written feedback)
rubric = copy.deepcopy(RUBRIC)
rubric['criteria'].append({
'name': 'feedback only',
'prompt': 'feedback',
'options': []
})
# Schedule a training task
# (we use training examples that do NOT include the feedback-only criterion)
workflow_uuid = ai_api.train_classifiers(rubric, EXAMPLES, COURSE_ID,
ITEM_ID, ALGORITHM_ID)
# Verify that no classifier was created for the feedback-only criterion
# Since there's no points associated with that criterion,
# there's no way for the AI algorithm to score it anyway.
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_data = workflow.classifier_set.classifier_data_by_criterion
self.assertNotIn('feedback only', classifier_data)
def test_train_classifiers_all_feedback_only_criteria(self):
# Modify the rubric to include only feedback-only criteria
# (a criterion with no options, just written feedback)
rubric = copy.deepcopy(RUBRIC)
for criterion in rubric['criteria']:
criterion['options'] = []
# Modify the training examples to provide no scores
examples = copy.deepcopy(EXAMPLES)
for example in examples:
example['options_selected'] = {}
# Schedule a training task
# Our training examples have no options
workflow_uuid = ai_api.train_classifiers(rubric, examples, COURSE_ID,
ITEM_ID, ALGORITHM_ID)
# Verify that no classifier was created for the feedback-only criteria
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_data = workflow.classifier_set.classifier_data_by_criterion
self.assertEqual(classifier_data, {})
def test_train_classifiers(self):
# Schedule a training task
# Because Celery is configured in "always eager" mode,
# expect the task to be executed synchronously.
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID,
ITEM_ID, ALGORITHM_ID)
# Retrieve the classifier set from the database
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_set = workflow.classifier_set
self.assertIsNot(classifier_set, None)
# Retrieve a dictionary mapping criteria names to deserialized classifiers
classifiers = classifier_set.classifier_data_by_criterion
# Check that we have classifiers for all criteria in the rubric
criteria = set(criterion['name'] for criterion in RUBRIC['criteria'])
self.assertEqual(set(classifiers.keys()), criteria)
# Check that the classifier data matches the data from our stub AI algorithm
# Since the stub data includes the training examples, we also verify
# that the classifier was trained using the correct examples.
for criterion in RUBRIC['criteria']:
classifier = classifiers[criterion['name']]
self.assertEqual(classifier['name'],
StubAIAlgorithm.FAKE_CLASSIFIER['name'])
self.assertEqual(classifier['binary_content'],
StubAIAlgorithm.FAKE_CLASSIFIER['binary_content'])
# Verify that the correct essays and scores were used to create the classifier
# Our stub AI algorithm provides these for us, but they would not ordinarily
# be included in the trained classifier.
self.assertEqual(len(classifier['examples']), len(EXAMPLES))
expected_scores = self.EXPECTED_INPUT_SCORES[criterion['name']]
for data in zip(EXAMPLES, classifier['examples'], expected_scores):
sent_example, received_example, expected_score = data
received_example = AIAlgorithm.ExampleEssay(*received_example)
self.assertEqual(received_example.text, sent_example['answer'])
self.assertEqual(received_example.score, expected_score)
def test_train_classifiers(self):
# Schedule a training task
# Because Celery is configured in "always eager" mode,
# expect the task to be executed synchronously.
workflow_uuid = ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
# Retrieve the classifier set from the database
workflow = AITrainingWorkflow.objects.get(uuid=workflow_uuid)
classifier_set = workflow.classifier_set
self.assertIsNot(classifier_set, None)
# Retrieve a dictionary mapping criteria names to deserialized classifiers
classifiers = classifier_set.classifier_data_by_criterion
# Check that we have classifiers for all criteria in the rubric
criteria = set(criterion['name'] for criterion in RUBRIC['criteria'])
self.assertEqual(set(classifiers.keys()), criteria)
# Check that the classifier data matches the data from our stub AI algorithm
# Since the stub data includes the training examples, we also verify
# that the classifier was trained using the correct examples.
for criterion in RUBRIC['criteria']:
classifier = classifiers[criterion['name']]
self.assertEqual(classifier['name'], StubAIAlgorithm.FAKE_CLASSIFIER['name'])
self.assertEqual(classifier['binary_content'], StubAIAlgorithm.FAKE_CLASSIFIER['binary_content'])
# Verify that the correct essays and scores were used to create the classifier
# Our stub AI algorithm provides these for us, but they would not ordinarily
# be included in the trained classifier.
self.assertEqual(len(classifier['examples']), len(EXAMPLES))
expected_scores = self.EXPECTED_INPUT_SCORES[criterion['name']]
for data in zip(EXAMPLES, classifier['examples'], expected_scores):
sent_example, received_example, expected_score = data
received_example = AIAlgorithm.ExampleEssay(*received_example)
self.assertEqual(received_example.text, sent_example['answer'])
self.assertEqual(received_example.score, expected_score)
def test_train_classifiers_no_examples(self):
# Empty list of training examples
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, [], COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_start_workflow_database_error(self, mock_create):
# Simulate a database error when creating the training workflow
mock_create.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_train_classifiers_celery_error(self):
with mock.patch('openassessment.assessment.api.ai.training_tasks.train_classifiers.apply_async') as mock_train:
mock_train.side_effect = NotConfigured
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID, ALGORITHM_ID)
def test_train_classifiers_no_examples(self):
# Empty list of training examples
with self.assertRaises(AITrainingRequestError):
ai_api.train_classifiers(RUBRIC, [], COURSE_ID, ITEM_ID,
ALGORITHM_ID)
def test_start_workflow_database_error(self, mock_create):
# Simulate a database error when creating the training workflow
mock_create.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AITrainingInternalError):
ai_api.train_classifiers(RUBRIC, EXAMPLES, COURSE_ID, ITEM_ID,
ALGORITHM_ID)
