def test_simulate_ai_grading_error(self):
# Run the command
cmd = simulate_ai_grading_error.Command()
cmd.handle(self.COURSE_ID.encode('utf-8'),
self.ITEM_ID.encode('utf-8'), self.NUM_SUBMISSIONS, "fake")
# Check that the correct number of incomplete workflows
# were created. These workflows should still have
# a classifier set, though, because otherwise they
# wouldn't have been scheduled for grading
# (that is, the submissions were made before classifier
# training completed).
incomplete_workflows = AIGradingWorkflow.objects.filter(
classifier_set__isnull=False, completed_at__isnull=True)
num_errors = incomplete_workflows.count()
self.assertEqual(self.NUM_SUBMISSIONS, num_errors)
# Verify that we can complete the workflows successfully
# (that is, make sure the classifier data is valid)
# We're calling a Celery task method here,
# but we're NOT using `apply_async`, so this will
# execute synchronously.
for workflow in incomplete_workflows:
grade_essay(workflow.uuid)
# Now there should be no incomplete workflows
remaining_incomplete = AIGradingWorkflow.objects.filter(
classifier_set__isnull=False, completed_at__isnull=True).count()
self.assertEqual(remaining_incomplete, 0)
def test_params_missing_criterion_for_valid_scores(self, mock_call):
mock_call.return_value = {
'essay_text': 'test',
'classifier_set': {
u"vøȼȺƀᵾłȺɍɏ": {},
u"ﻭɼค๓๓คɼ": {}
},
'algorithm_id': ALGORITHM_ID,
'valid_scores': {}
}
with self.assert_retry(grade_essay, AIGradingInternalError):
grade_essay(self.workflow_uuid)
def test_params_missing_criterion_for_valid_scores(self, mock_call):
mock_call.return_value = {
'essay_text': 'test',
'classifier_set': {
u"vøȼȺƀᵾłȺɍɏ": {},
u"ﻭɼค๓๓คɼ": {}
},
'algorithm_id': ALGORITHM_ID,
'valid_scores': {}
}
with self.assert_retry(grade_essay, AIGradingInternalError):
grade_essay(self.workflow_uuid)
def test_skip_completed_workflow(self):
# Mark the grading workflow as complete
workflow = AIGradingWorkflow.objects.get(uuid=self.workflow_uuid)
workflow.mark_complete_and_save()
# The grading task should short-circuit immediately, skipping calls
# to get parameters for the task.
actual_call = ai_worker_api.get_grading_task_params
patched = 'openassessment.assessment.worker.grading.ai_worker_api.get_grading_task_params'
with mock.patch(patched) as mock_call:
mock_call.side_effect = actual_call
grade_essay(self.workflow_uuid)
self.assertFalse(mock_call.called)
def test_skip_completed_workflow(self):
# Mark the grading workflow as complete
workflow = AIGradingWorkflow.objects.get(uuid=self.workflow_uuid)
workflow.mark_complete_and_save()
# The grading task should short-circuit immediately, skipping calls
# to get parameters for the task.
actual_call = ai_worker_api.get_grading_task_params
patched = 'openassessment.assessment.worker.grading.ai_worker_api.get_grading_task_params'
with mock.patch(patched) as mock_call:
mock_call.side_effect = actual_call
grade_essay(self.workflow_uuid)
self.assertFalse(mock_call.called)
def test_algorithm_gives_invalid_score(self, mock_create_assessment):
# If an algorithm provides a score that isn't in the rubric,
# we should choose the closest valid score.
self._set_algorithm_id(INVALID_SCORE_ALGORITHM_ID)
# The first score given by the algorithm should be below the minimum valid score
# The second score will be between two valid scores (0 and 1), rounding up
grade_essay(self.workflow_uuid)
expected_scores = {u"vøȼȺƀᵾłȺɍɏ": 0, u"ﻭɼค๓๓คɼ": 1}
mock_create_assessment.assert_called_with(self.workflow_uuid,
expected_scores)
# The third score will be between two valid scores (1 and 2), rounding down
# The final score will be greater than the maximum score
self._reset_workflow()
grade_essay(self.workflow_uuid)
expected_scores = {u"vøȼȺƀᵾłȺɍɏ": 1, u"ﻭɼค๓๓คɼ": 2}
mock_create_assessment.assert_called_with(self.workflow_uuid,
expected_scores)
def test_algorithm_gives_invalid_score(self, mock_create_assessment):
# If an algorithm provides a score that isn't in the rubric,
# we should choose the closest valid score.
self._set_algorithm_id(INVALID_SCORE_ALGORITHM_ID)
# The first score given by the algorithm should be below the minimum valid score
# The second score will be between two valid scores (0 and 1), rounding up
grade_essay(self.workflow_uuid)
expected_scores = {
u"vøȼȺƀᵾłȺɍɏ": 0,
u"ﻭɼค๓๓คɼ": 1
}
mock_create_assessment.assert_called_with(self.workflow_uuid, expected_scores)
# The third score will be between two valid scores (1 and 2), rounding down
# The final score will be greater than the maximum score
self._reset_workflow()
grade_essay(self.workflow_uuid)
expected_scores = {
u"vøȼȺƀᵾłȺɍɏ": 1,
u"ﻭɼค๓๓คɼ": 2
}
mock_create_assessment.assert_called_with(self.workflow_uuid, expected_scores)
def test_simulate_ai_grading_error(self):
# Run the command
cmd = simulate_ai_grading_error.Command()
cmd.handle(
self.COURSE_ID.encode('utf-8'),
self.ITEM_ID.encode('utf-8'),
self.NUM_SUBMISSIONS,
"fake"
)
# Check that the correct number of incomplete workflows
# were created. These workflows should still have
# a classifier set, though, because otherwise they
# wouldn't have been scheduled for grading
# (that is, the submissions were made before classifier
# training completed).
incomplete_workflows = AIGradingWorkflow.objects.filter(
classifier_set__isnull=False,
completed_at__isnull=True
)
num_errors = incomplete_workflows.count()
self.assertEqual(self.NUM_SUBMISSIONS, num_errors)
# Verify that we can complete the workflows successfully
# (that is, make sure the classifier data is valid)
# We're calling a Celery task method here,
# but we're NOT using `apply_async`, so this will
# execute synchronously.
for workflow in incomplete_workflows:
grade_essay(workflow.uuid)
# Now there should be no incomplete workflows
remaining_incomplete = AIGradingWorkflow.objects.filter(
classifier_set__isnull=False,
completed_at__isnull=True
).count()
self.assertEqual(remaining_incomplete, 0)
def test_create_assessment_error(self, mock_call):
mock_call.side_effect = AIGradingInternalError
with self.assert_retry(grade_essay, AIGradingInternalError):
grade_essay(self.workflow_uuid)
def test_algorithm_score_error(self):
self._set_algorithm_id(ERROR_STUB_ALGORITHM_ID)
with self.assert_retry(grade_essay, ScoreError):
grade_essay(self.workflow_uuid)
def test_unknown_algorithm_id_error(self):
# Since we're not overriding settings, the algorithm ID won't be recognized
with self.assert_retry(grade_essay, UnknownAlgorithm):
grade_essay(self.workflow_uuid)
def test_retrieve_params_error(self, mock_call):
mock_call.side_effect = AIGradingInternalError("Test error")
with self.assert_retry(grade_essay, AIGradingInternalError):
grade_essay(self.workflow_uuid)
def test_check_complete_error(self):
with self.assert_retry(grade_essay, AIGradingRequestError):
grade_essay("no such workflow uuid")
def test_check_complete_error(self):
with self.assert_retry(grade_essay, AIGradingRequestError):
grade_essay("no such workflow uuid")
def test_create_assessment_error(self, mock_call):
mock_call.side_effect = AIGradingInternalError
with self.assert_retry(grade_essay, AIGradingInternalError):
grade_essay(self.workflow_uuid)
def test_algorithm_score_error(self):
self._set_algorithm_id(ERROR_STUB_ALGORITHM_ID)
with self.assert_retry(grade_essay, ScoreError):
grade_essay(self.workflow_uuid)
def test_unknown_algorithm_id_error(self):
# Since we're not overriding settings, the algorithm ID won't be recognized
with self.assert_retry(grade_essay, UnknownAlgorithm):
grade_essay(self.workflow_uuid)
def test_retrieve_params_error(self, mock_call):
mock_call.side_effect = AIGradingInternalError("Test error")
with self.assert_retry(grade_essay, AIGradingInternalError):
grade_essay(self.workflow_uuid)
