def test_duplicate_training_example(self):
# Deserialize some examples for a rubric
deserialize_training_examples(self.EXAMPLES[0:2], self.RUBRIC)
# Deserialize some more examples, of which two are duplicates
examples = deserialize_training_examples(self.EXAMPLES, self.RUBRIC)
# Check that only three examples were created in the database
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 3)
# Check that the examples match what we got from the deserializer
self.assertItemsEqual(examples, db_examples)
def test_duplicate_training_example(self):
# Deserialize some examples for a rubric
deserialize_training_examples(self.EXAMPLES[0:2], self.RUBRIC)
# Deserialize some more examples, of which two are duplicates
examples = deserialize_training_examples(self.EXAMPLES, self.RUBRIC)
# Check that only three examples were created in the database
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 3)
# Check that the examples match what we got from the deserializer
self.assertItemsEqual(examples, db_examples)
def setUp(self):
"""
Create a training workflow in the database.
"""
examples = deserialize_training_examples(EXAMPLES, RUBRIC)
workflow = AITrainingWorkflow.start_workflow(examples, self.COURSE_ID, self.ITEM_ID, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def test_similar_training_examples_different_rubric(self):
# Deserialize some examples
first_examples = deserialize_training_examples(self.EXAMPLES, self.RUBRIC)
# Deserialize one more example with the rubric mutated slightly
mutated_rubric = copy.deepcopy(self.RUBRIC)
mutated_rubric['criteria'][0]['options'][0]['points'] = 5
second_examples = deserialize_training_examples(self.EXAMPLES[0:2], mutated_rubric)
# There should be a total of 5 examples (3 for the first rubric + 2 for the second)
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 5)
# Check that each of the examples from the deserializer are in the database
for example in (first_examples + second_examples):
self.assertIn(example, db_examples)
def train_classifiers(rubric_dict, examples, course_id, item_id, algorithm_id):
"""
Schedule a task to train classifiers.
All training examples must match the rubric!
After training of classifiers completes successfully, all AIGradingWorkflows that are incomplete will be
automatically rescheduled to complete.
Args:
rubric_dict (dict): The rubric used to assess the classifiers.
examples (list of dict): Serialized training examples.
algorithm_id (unicode): The ID of the algorithm used to train the classifiers.
Returns:
training_workflow_uuid (str): The UUID of the training workflow.
Usually the caller will not need this (since the workers
are parametrized by training workflow UUID), but it's
useful for testing.
Raises:
AITrainingRequestError
AITrainingInternalError
Example usage:
>>> train_classifiers(rubric, examples, 'ease')
'10df7db776686822e501b05f452dc1e4b9141fe5'
"""
# Get or create the rubric and training examples
try:
examples = deserialize_training_examples(examples, rubric_dict)
except (InvalidRubric, InvalidTrainingExample, InvalidRubricSelection) as ex:
msg = u"Could not parse rubric and/or training examples: {ex}".format(ex=ex)
raise AITrainingRequestError(msg)
# Create the workflow model
try:
workflow = AITrainingWorkflow.start_workflow(examples, course_id, item_id, algorithm_id)
except NoTrainingExamples as ex:
raise AITrainingRequestError(ex)
except:
msg = (
u"An unexpected error occurred while creating "
u"the AI training workflow"
)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Schedule the task, parametrized by the workflow UUID
try:
training_tasks.train_classifiers.apply_async(args=[workflow.uuid])
except ANTICIPATED_CELERY_ERRORS as ex:
msg = (
u"An unexpected error occurred while scheduling incomplete training workflows with"
u" course_id={cid} and item_id={iid}: {ex}"
).format(cid=course_id, iid=item_id, ex=ex)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Return the workflow UUID
return workflow.uuid
def test_similar_training_examples_different_options(self):
# Deserialize some examples
first_examples = deserialize_training_examples(self.EXAMPLES, self.RUBRIC)
# Deserialize another example that's identical to the first example,
# with one option changed
mutated_examples = copy.deepcopy(self.EXAMPLES)
mutated_examples[0]['options_selected'][u'vøȼȺƀᵾłȺɍɏ'] = u"єχ¢єℓℓєηт"
second_examples = deserialize_training_examples(mutated_examples, self.RUBRIC)
# Expect that a total of 4 examples (3 for the first call, plus one new example in the second call)
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 4)
# Check that all the examples are in the database
for example in (first_examples + second_examples):
self.assertIn(example, db_examples)
def test_similar_training_examples_different_answer(self):
# Deserialize some examples
first_examples = deserialize_training_examples(self.EXAMPLES, self.RUBRIC)
# Deserialize another example that's identical to the first example,
# with a different answer
mutated_examples = copy.deepcopy(self.EXAMPLES)
mutated_examples[0]['answer'] = u"MUTATED!"
second_examples = deserialize_training_examples(mutated_examples, self.RUBRIC)
# Expect that a total of 4 examples (3 for the first call, plus one new example in the second call)
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 4)
# Check that all the examples are in the database
for example in (first_examples + second_examples):
self.assertIn(example, db_examples)
def setUp(self):
"""
Create a training workflow in the database.
"""
examples = deserialize_training_examples(EXAMPLES, RUBRIC)
workflow = AITrainingWorkflow.start_workflow(examples, self.COURSE_ID,
self.ITEM_ID,
self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def test_similar_training_examples_different_rubric(self):
# Deserialize some examples
first_examples = deserialize_training_examples(self.EXAMPLES,
self.RUBRIC)
# Deserialize one more example with the rubric mutated slightly
mutated_rubric = copy.deepcopy(self.RUBRIC)
mutated_rubric['criteria'][0]['options'][0]['points'] = 5
second_examples = deserialize_training_examples(
self.EXAMPLES[0:2], mutated_rubric)
# There should be a total of 5 examples (3 for the first rubric + 2 for the second)
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 5)
# Check that each of the examples from the deserializer are in the database
for example in (first_examples + second_examples):
self.assertIn(example, db_examples)
def test_serialize_training_example_with_legacy_answer(self):
"""Test that legacy answer format in training example serialized correctly"""
training_examples = deserialize_training_examples(self.EXAMPLES, self.RUBRIC)
for example in training_examples:
self.assertIsInstance(example.answer, unicode)
serialized_example = serialize_training_example(example)
self.assertIsInstance(serialized_example["answer"], dict)
expected_answer_dict = {'parts': [{'text': example.answer}]}
self.assertEqual(serialized_example["answer"], expected_answer_dict)
def test_similar_training_examples_different_answer(self):
# Deserialize some examples
first_examples = deserialize_training_examples(self.EXAMPLES,
self.RUBRIC)
# Deserialize another example that's identical to the first example,
# with a different answer
mutated_examples = copy.deepcopy(self.EXAMPLES)
mutated_examples[0]['answer'] = u"MUTATED!"
second_examples = deserialize_training_examples(
mutated_examples, self.RUBRIC)
# Expect that a total of 4 examples (3 for the first call, plus one new example in the second call)
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 4)
# Check that all the examples are in the database
for example in (first_examples + second_examples):
self.assertIn(example, db_examples)
def test_similar_training_examples_different_options(self):
# Deserialize some examples
first_examples = deserialize_training_examples(self.EXAMPLES,
self.RUBRIC)
# Deserialize another example that's identical to the first example,
# with one option changed
mutated_examples = copy.deepcopy(self.EXAMPLES)
mutated_examples[0]['options_selected'][u'vøȼȺƀᵾłȺɍɏ'] = u"єχ¢єℓℓєηт"
second_examples = deserialize_training_examples(
mutated_examples, self.RUBRIC)
# Expect that a total of 4 examples (3 for the first call, plus one new example in the second call)
db_examples = TrainingExample.objects.all()
self.assertEqual(len(db_examples), 4)
# Check that all the examples are in the database
for example in (first_examples + second_examples):
self.assertIn(example, db_examples)
def test_serialize_training_example_with_legacy_answer(self):
"""Test that legacy answer format in training example serialized correctly"""
training_examples = deserialize_training_examples(
self.EXAMPLES, self.RUBRIC)
for example in training_examples:
self.assertIsInstance(example.answer, unicode)
serialized_example = serialize_training_example(example)
self.assertIsInstance(serialized_example["answer"], dict)
expected_answer_dict = {'parts': [{'text': example.answer}]}
self.assertEqual(serialized_example["answer"],
expected_answer_dict)
def test_deserialize_integrity_error(self, mock_create, mock_get):
# Simulate an integrity error when creating the training example
# This can occur when using repeatable-read isolation mode.
mock_example = mock.MagicMock(TrainingExample)
mock_get.side_effect = [TrainingExample.DoesNotExist, mock_example]
mock_create.side_effect = IntegrityError
# Expect that we get the mock example back
# (proves that the function tried to retrieve the object again after
# catching the integrity error)
examples = deserialize_training_examples(self.EXAMPLES[:1], self.RUBRIC)
self.assertEqual(examples, [mock_example])
def test_deserialize_integrity_error(self):
"""
Simulate an integrity error when creating the training example
This can occur when using repeatable-read isolation mode.
"""
example = deserialize_training_examples(self.EXAMPLES[:1],
self.RUBRIC)[0]
with mock.patch(
'openassessment.assessment.models.TrainingExample.objects.get'
) as mock_get:
with mock.patch(
'openassessment.assessment.models.TrainingExample.create_example'
) as mock_create:
mock_get.side_effect = [TrainingExample.DoesNotExist, example]
mock_create.side_effect = IntegrityError
# Expect that we get the mock example back
# (proves that the function tried to retrieve the object again after
# catching the integrity error)
examples = deserialize_training_examples(
self.EXAMPLES[:1], self.RUBRIC)
self.assertEqual(examples, [example])
def test_deserialize_integrity_error(self, mock_create, mock_get):
# Simulate an integrity error when creating the training example
# This can occur when using repeatable-read isolation mode.
mock_example = mock.MagicMock(TrainingExample)
mock_get.side_effect = [TrainingExample.DoesNotExist, mock_example]
mock_create.side_effect = IntegrityError
# Expect that we get the mock example back
# (proves that the function tried to retrieve the object again after
# catching the integrity error)
examples = deserialize_training_examples(self.EXAMPLES[:1],
self.RUBRIC)
self.assertEqual(examples, [mock_example])
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
If no training examples are available (the student has already assessed every example,
or no examples are defined), returns None.
Raises:
StudentTrainingInternalError
Example usage:
>>> examples = [
>>> {
>>> 'answer': u'Doler',
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': u'Lorem ipsum',
'rubric': {
"prompt": "Write an essay!",
"criteria": [
{
"order_num": 0,
"name": "vocabulary",
"prompt": "How varied is the vocabulary?",
"options": options
},
{
"order_num": 1,
"name": "grammar",
"prompt": "How correct is the grammar?",
"options": options
}
],
},
'options_selected': {
'vocabulary': 'good',
'grammar': 'excellent'
}
}
"""
try:
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if len(errors) > 0:
msg = (
u"Training examples do not match the rubric (submission UUID is {uuid}): {errors}"
).format(uuid=submission_uuid, errors="\n".join(errors))
raise StudentTrainingRequestError(msg)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid)
if not workflow:
raise StudentTrainingRequestError(
u"No student training workflow found for submission {}".format(submission_uuid)
)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
next_example = workflow.next_training_example(examples)
return None if next_example is None else serialize_training_example(next_example)
except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex:
logger.exception(
"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex)
except sub_api.SubmissionNotFoundError as ex:
msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg)
except DatabaseError:
msg = (
u"Could not retrieve a training example "
u"for the student with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg)
def get_training_example(submission_uuid, rubric, examples):
"""
Retrieve a training example for the student to assess.
This will implicitly create a workflow for the student if one does not yet exist.
NOTE: We include the rubric in the returned dictionary to handle
the case in which the instructor changes the rubric definition
while the student is assessing the training example. Once a student
starts on a training example, the student should see the same training
example consistently. However, the next training example the student
retrieves will use the updated rubric.
Args:
submission_uuid (str): The UUID of the student's submission.
rubric (dict): Serialized rubric model.
examples (list): List of serialized training examples.
Returns:
dict: The training example with keys "answer", "rubric", and "options_selected".
If no training examples are available (the student has already assessed every example,
or no examples are defined), returns None.
Raises:
StudentTrainingInternalError
Example usage:
>>> examples = [
>>> {
>>> 'answer': {
>>> 'parts': {
>>> [
>>> {'text:' 'Answer part 1'},
>>> {'text:' 'Answer part 2'},
>>> {'text:' 'Answer part 3'}
>>> ]
>>> }
>>> },
>>> 'options_selected': {
>>> 'vocabulary': 'good',
>>> 'grammar': 'poor'
>>> }
>>> }
>>> ]
>>>
>>> get_training_example("5443ebbbe2297b30f503736e26be84f6c7303c57", rubric, examples)
{
'answer': {
'parts': {
[
{'text:' 'Answer part 1'},
{'text:' 'Answer part 2'},
{'text:' 'Answer part 3'}
]
}
},
'rubric': {
"prompts": [
{"description": "Prompt 1"},
{"description": "Prompt 2"},
{"description": "Prompt 3"}
],
"criteria": [
{
"order_num": 0,
"name": "vocabulary",
"prompt": "How varied is the vocabulary?",
"options": options
},
{
"order_num": 1,
"name": "grammar",
"prompt": "How correct is the grammar?",
"options": options
}
],
},
'options_selected': {
'vocabulary': 'good',
'grammar': 'excellent'
}
}
"""
try:
# Validate the training examples
errors = validate_training_examples(rubric, examples)
if errors:
msg = (
"Training examples do not match the rubric (submission UUID is {uuid}): {errors}"
).format(uuid=submission_uuid, errors="\n".join(errors))
raise StudentTrainingRequestError(msg)
# Get or create the workflow
workflow = StudentTrainingWorkflow.get_workflow(submission_uuid=submission_uuid)
if not workflow:
raise StudentTrainingRequestError(
u"No learner training workflow found for submission {}".format(submission_uuid)
)
# Get or create the training examples
examples = deserialize_training_examples(examples, rubric)
# Pick a training example that the student has not yet completed
# If the student already started a training example, then return that instead.
next_example = workflow.next_training_example(examples)
return None if next_example is None else serialize_training_example(next_example)
except (InvalidRubric, InvalidRubricSelection, InvalidTrainingExample) as ex:
logger.exception(
u"Could not deserialize training examples for submission UUID {}".format(submission_uuid)
)
raise StudentTrainingRequestError(ex) from ex
except sub_api.SubmissionNotFoundError as ex:
msg = u"Could not retrieve the submission with UUID {}".format(submission_uuid)
logger.exception(msg)
raise StudentTrainingRequestError(msg) from ex
except DatabaseError as ex:
msg = (
u"Could not retrieve a training example for the learner with submission UUID {}"
).format(submission_uuid)
logger.exception(msg)
raise StudentTrainingInternalError(msg) from ex
def train_classifiers(rubric_dict, examples, course_id, item_id, algorithm_id):
"""
Schedule a task to train classifiers.
All training examples must match the rubric!
After training of classifiers completes successfully, all AIGradingWorkflows that are incomplete will be
automatically rescheduled to complete.
Args:
rubric_dict (dict): The rubric used to assess the classifiers.
examples (list of dict): Serialized training examples.
algorithm_id (unicode): The ID of the algorithm used to train the classifiers.
Returns:
training_workflow_uuid (str): The UUID of the training workflow.
Usually the caller will not need this (since the workers
are parametrized by training workflow UUID), but it's
useful for testing.
Raises:
AITrainingRequestError
AITrainingInternalError
Example usage:
>>> train_classifiers(rubric, examples, 'ease')
'10df7db776686822e501b05f452dc1e4b9141fe5'
"""
# Get or create the rubric and training examples
try:
examples = deserialize_training_examples(examples, rubric_dict)
except (InvalidRubric, InvalidTrainingExample, InvalidRubricSelection) as ex:
msg = u"Could not parse rubric and/or training examples: {ex}".format(ex=ex)
raise AITrainingRequestError(msg)
# Create the workflow model
try:
workflow = AITrainingWorkflow.start_workflow(examples, course_id, item_id, algorithm_id)
except NoTrainingExamples as ex:
raise AITrainingRequestError(ex)
except:
msg = (
u"An unexpected error occurred while creating "
u"the AI training workflow"
)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Schedule the task, parametrized by the workflow UUID
try:
training_tasks.train_classifiers.apply_async(args=[workflow.uuid])
except ANTICIPATED_CELERY_ERRORS as ex:
msg = (
u"An unexpected error occurred while scheduling incomplete training workflows with"
u" course_id={cid} and item_id={iid}: {ex}"
).format(cid=course_id, iid=item_id, ex=ex)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Return the workflow UUID
return workflow.uuid
