def test_skip_completed_workflow(self):
# Mark the grading workflow as complete
workflow = AITrainingWorkflow.objects.get(uuid=self.workflow_uuid)
workflow.mark_complete_and_save()
# The training task should short-circuit immediately, skipping calls
# to get parameters for the task.
actual_call = ai_worker_api.get_training_task_params
patched = 'openassessment.assessment.worker.grading.ai_worker_api.get_training_task_params'
with mock.patch(patched) as mock_call:
mock_call.side_effect = actual_call
train_classifiers(self.workflow_uuid)
self.assertFalse(mock_call.called)
def test_skip_completed_workflow(self):
# Mark the grading workflow as complete
workflow = AITrainingWorkflow.objects.get(uuid=self.workflow_uuid)
workflow.mark_complete_and_save()
# The training task should short-circuit immediately, skipping calls
# to get parameters for the task.
actual_call = ai_worker_api.get_training_task_params
patched = 'openassessment.assessment.worker.grading.ai_worker_api.get_training_task_params'
with mock.patch(patched) as mock_call:
mock_call.side_effect = actual_call
train_classifiers(self.workflow_uuid)
self.assertFalse(mock_call.called)
def _assert_mutated_examples(self, mutate_func):
"""
Mutate the training examples returned by the API,
then check that we get the expected error.
This *may* be a little paranoid :)
Args:
mutate_func (callable): Function that accepts a single argument,
the list of example dictionaries.
Raises:
AssertionError
"""
params = ai_worker_api.get_training_task_params(self.workflow_uuid)
mutate_func(params['training_examples'])
call_signature = 'openassessment.assessment.worker.training.ai_worker_api.get_training_task_params'
with mock.patch(call_signature) as mock_call:
mock_call.return_value = params
with self.assert_retry(train_classifiers, InvalidExample):
train_classifiers(self.workflow_uuid)
def _assert_mutated_examples(self, mutate_func):
"""
Mutate the training examples returned by the API,
then check that we get the expected error.
This *may* be a little paranoid :)
Args:
mutate_func (callable): Function that accepts a single argument,
the list of example dictionaries.
Raises:
AssertionError
"""
params = ai_worker_api.get_training_task_params(self.workflow_uuid)
mutate_func(params['training_examples'])
call_signature = 'openassessment.assessment.worker.training.ai_worker_api.get_training_task_params'
with mock.patch(call_signature) as mock_call:
mock_call.return_value = params
with self.assert_retry(train_classifiers, InvalidExample):
train_classifiers(self.workflow_uuid)
def test_create_classifiers_api_error(self, mock_call):
mock_call.side_effect = AITrainingRequestError("Test error!")
with self.assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers(self.workflow_uuid)
def test_training_algorithm_error(self):
# Use a stub algorithm implementation that raises an exception during training
self._set_algorithm_id(ERROR_STUB_ALGORITHM_ID)
with self.assert_retry(train_classifiers, TrainingError):
train_classifiers(self.workflow_uuid)
def test_unable_to_find_algorithm_module(self):
# The algorithm is defined in the settings, but the module can't be loaded
self._set_algorithm_id(UNDEFINED_MODULE_ALGORITHM_ID)
with self.assert_retry(train_classifiers, AlgorithmLoadError):
train_classifiers(self.workflow_uuid)
def test_unable_to_load_algorithm_class(self):
# The algorithm is defined in the settings, but the class does not exist.
self._set_algorithm_id(UNDEFINED_CLASS_ALGORITHM_ID)
with self.assert_retry(train_classifiers, AlgorithmLoadError):
train_classifiers(self.workflow_uuid)
def test_check_complete_error(self):
with self.assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers("no such workflow uuid")
def test_unknown_algorithm(self):
# Since we haven't overridden settings to configure the algorithms,
# the worker will not recognize the workflow's algorithm ID.
with self.assert_retry(train_classifiers, UnknownAlgorithm):
train_classifiers(self.workflow_uuid)
def test_create_classifiers_api_error(self, mock_call):
mock_call.side_effect = AITrainingRequestError("Test error!")
with self.assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers(self.workflow_uuid)
def test_training_algorithm_error(self):
# Use a stub algorithm implementation that raises an exception during training
self._set_algorithm_id(ERROR_STUB_ALGORITHM_ID)
with self.assert_retry(train_classifiers, TrainingError):
train_classifiers(self.workflow_uuid)
def test_unable_to_find_algorithm_module(self):
# The algorithm is defined in the settings, but the module can't be loaded
self._set_algorithm_id(UNDEFINED_MODULE_ALGORITHM_ID)
with self.assert_retry(train_classifiers, AlgorithmLoadError):
train_classifiers(self.workflow_uuid)
def test_unable_to_load_algorithm_class(self):
# The algorithm is defined in the settings, but the class does not exist.
self._set_algorithm_id(UNDEFINED_CLASS_ALGORITHM_ID)
with self.assert_retry(train_classifiers, AlgorithmLoadError):
train_classifiers(self.workflow_uuid)
def test_check_complete_error(self):
with self.assert_retry(train_classifiers, AITrainingRequestError):
train_classifiers("no such workflow uuid")
def test_unknown_algorithm(self):
# Since we haven't overridden settings to configure the algorithms,
# the worker will not recognize the workflow's algorithm ID.
with self.assert_retry(train_classifiers, UnknownAlgorithm):
train_classifiers(self.workflow_uuid)
