def test_get_grading_task_params_num_queries(self):
with self.assertNumQueries(6):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
# The second time through we should be caching the queries
# to determine the valid scores for a classifier
with self.assertNumQueries(2):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
def test_get_grading_task_params_num_queries(self):
with self.assertNumQueries(6):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
# The second time through we should be caching the queries
# to determine the valid scores for a classifier
with self.assertNumQueries(2):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
def test_invalid_classifier_data(self):
# Modify the classifier data so it is not valid JSON
invalid_json = "{"
for classifier in AIClassifier.objects.all():
classifier.classifier_data.save(uuid4().hex, ContentFile(invalid_json))
# Should get an error when retrieving task params
with self.assertRaises(AIGradingInternalError):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
def test_get_grading_task_params_no_classifiers(self):
# Remove the classifiers from the workflow
workflow = AIGradingWorkflow.objects.get(uuid=self.workflow_uuid)
workflow.classifier_set = None
workflow.save()
# Should get an error when retrieving task params
with self.assertRaises(AIGradingInternalError):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
def test_get_grading_task_params_no_classifiers(self):
# Remove the classifiers from the workflow
workflow = AIGradingWorkflow.objects.get(uuid=self.workflow_uuid)
workflow.classifier_set = None
workflow.save()
# Should get an error when retrieving task params
with self.assertRaises(AIGradingInternalError):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
def test_invalid_classifier_data(self):
# Modify the classifier data so it is not valid JSON
invalid_json = "{"
for classifier in AIClassifier.objects.all():
classifier.classifier_data.save(uuid4().hex,
ContentFile(invalid_json))
# Should get an error when retrieving task params
with self.assertRaises(AIGradingInternalError):
ai_worker_api.get_grading_task_params(self.workflow_uuid)
def test_get_grading_task_params(self):
params = ai_worker_api.get_grading_task_params(self.workflow_uuid)
expected_params = {
'essay_text': ANSWER,
'classifier_set': CLASSIFIERS,
'algorithm_id': ALGORITHM_ID,
'valid_scores': {
u"vøȼȺƀᵾłȺɍɏ": [0, 1, 2],
u"ﻭɼค๓๓คɼ": [0, 1, 2]
}
}
self.assertItemsEqual(params, expected_params)
def test_get_grading_task_params(self):
params = ai_worker_api.get_grading_task_params(self.workflow_uuid)
expected_params = {
'essay_text': ANSWER,
'classifier_set': CLASSIFIERS,
'algorithm_id': ALGORITHM_ID,
'valid_scores': {
u"vøȼȺƀᵾłȺɍɏ": [0, 1, 2],
u"ﻭɼค๓๓คɼ": [0, 1, 2]
}
}
self.assertItemsEqual(params, expected_params)
def grade_essay(workflow_uuid):
"""
Asynchronous task to grade an essay using a text classifier
(trained using a supervised ML algorithm).
If the task could not be completed successfully,
it will be retried a few times; if it continues to fail,
it is left incomplete. Incomplate tasks can be rescheduled
manually through the AI API.
Args:
workflow_uuid (str): The UUID of the workflow associated
with this grading task.
Returns:
None
Raises:
AIError: An error occurred while making an AI worker API call.
AIAlgorithmError: An error occurred while retrieving or using an AI algorithm.
"""
# Short-circuit if the workflow is already marked complete
# This is an optimization, but grading tasks could still
# execute multiple times depending on when they get picked
# up by workers and marked complete.
try:
if ai_worker_api.is_grading_workflow_complete(workflow_uuid):
return
except AIError:
msg = (
u"An unexpected error occurred while checking the "
u"completion of grading workflow with UUID {uuid}"
).format(uuid=workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Retrieve the task parameters
try:
params = ai_worker_api.get_grading_task_params(workflow_uuid)
essay_text = params['essay_text']
classifier_set = params['classifier_set']
algorithm_id = params['algorithm_id']
valid_scores = params['valid_scores']
except (AIError, KeyError):
msg = (
u"An error occurred while retrieving the AI grading task "
u"parameters for the workflow with UUID {}"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Validate that the we have valid scores for each criterion
for criterion_name in classifier_set.keys():
msg = None
if criterion_name not in valid_scores:
msg = (
u"Could not find {criterion} in the list of valid scores "
u"for grading workflow with UUID {uuid}"
).format(criterion=criterion_name, uuid=workflow_uuid)
elif len(valid_scores[criterion_name]) == 0:
msg = (
u"Valid scores for {criterion} is empty for "
u"grading workflow with UUID {uuid}"
).format(criterion=criterion_name, uuid=workflow_uuid)
if msg:
logger.exception(msg)
raise AIGradingInternalError(msg)
# Retrieve the AI algorithm
try:
algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
except AIAlgorithmError:
msg = (
u"An error occurred while retrieving "
u"the algorithm ID (grading workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Use the algorithm to evaluate the essay for each criterion
# Provide an in-memory cache so the algorithm can re-use
# results for multiple rubric criteria.
try:
cache = dict()
scores_by_criterion = {
criterion_name: _closest_valid_score(
algorithm.score(essay_text, classifier, cache),
valid_scores[criterion_name]
)
for criterion_name, classifier in classifier_set.iteritems()
}
except AIAlgorithmError:
msg = (
u"An error occurred while scoring essays using "
u"an AI algorithm (worker workflow UUID {})"
).format(workflow_uuid)
logger.exception(msg)
raise grade_essay.retry()
# Create the assessment and mark the workflow complete
try:
ai_worker_api.create_assessment(workflow_uuid, scores_by_criterion)
except AIError:
msg = (
u"An error occurred while creating assessments "
u"for the AI grading workflow with UUID {uuid}. "
u"The assessment scores were: {scores}"
).format(uuid=workflow_uuid, scores=scores_by_criterion)
logger.exception(msg)
raise grade_essay.retry()
def test_get_grading_task_params_database_error(self, mock_call):
mock_call.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AIGradingInternalError):
ai_worker_api.get_grading_task_params(self.submission_uuid)
def test_get_grading_task_params_no_workflow(self):
with self.assertRaises(AIGradingRequestError):
ai_worker_api.get_grading_task_params("invalid_uuid")
def test_get_grading_task_params_database_error(self, mock_call):
mock_call.side_effect = DatabaseError("KABOOM!")
with self.assertRaises(AIGradingInternalError):
ai_worker_api.get_grading_task_params(self.submission_uuid)
def test_get_grading_task_params_no_workflow(self):
with self.assertRaises(AIGradingRequestError):
ai_worker_api.get_grading_task_params("invalid_uuid")
