def setUp(self):
"""
Create a training workflow in the database.
"""
examples = deserialize_training_examples(EXAMPLES, RUBRIC)
workflow = AITrainingWorkflow.start_workflow(examples, self.COURSE_ID, self.ITEM_ID, self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def is_training_workflow_complete(workflow_uuid):
"""
Check whether the training workflow is complete.
Args:
workflow_uuid (str): The UUID of the training workflow
Returns:
bool
Raises:
AITrainingRequestError
AITrainingInternalError
"""
try:
return AITrainingWorkflow.is_workflow_complete(workflow_uuid)
except AITrainingWorkflow.DoesNotExist:
msg = (
u"Could not retrieve training workflow "
u"with uuid {uuid} to check whether it's complete."
).format(uuid=workflow_uuid)
raise AITrainingRequestError(msg)
except DatabaseError:
msg = (
u"An unexpected error occurred while checking "
u"the training workflow with uuid {uuid} for completeness"
).format(uuid=workflow_uuid)
raise AITrainingInternalError(msg)
def is_training_workflow_complete(workflow_uuid):
"""
Check whether the training workflow is complete.
Args:
workflow_uuid (str): The UUID of the training workflow
Returns:
bool
Raises:
AITrainingRequestError
AITrainingInternalError
"""
try:
return AITrainingWorkflow.is_workflow_complete(workflow_uuid)
except AITrainingWorkflow.DoesNotExist:
msg = (u"Could not retrieve training workflow "
u"with uuid {uuid} to check whether it's complete.").format(
uuid=workflow_uuid)
raise AITrainingRequestError(msg)
except DatabaseError:
msg = (
u"An unexpected error occurred while checking "
u"the training workflow with uuid {uuid} for completeness").format(
uuid=workflow_uuid)
raise AITrainingInternalError(msg)
def train_classifiers(rubric_dict, examples, course_id, item_id, algorithm_id):
"""
Schedule a task to train classifiers.
All training examples must match the rubric!
After training of classifiers completes successfully, all AIGradingWorkflows that are incomplete will be
automatically rescheduled to complete.
Args:
rubric_dict (dict): The rubric used to assess the classifiers.
examples (list of dict): Serialized training examples.
algorithm_id (unicode): The ID of the algorithm used to train the classifiers.
Returns:
training_workflow_uuid (str): The UUID of the training workflow.
Usually the caller will not need this (since the workers
are parametrized by training workflow UUID), but it's
useful for testing.
Raises:
AITrainingRequestError
AITrainingInternalError
Example usage:
>>> train_classifiers(rubric, examples, 'ease')
'10df7db776686822e501b05f452dc1e4b9141fe5'
"""
# Get or create the rubric and training examples
try:
examples = deserialize_training_examples(examples, rubric_dict)
except (InvalidRubric, InvalidTrainingExample, InvalidRubricSelection) as ex:
msg = u"Could not parse rubric and/or training examples: {ex}".format(ex=ex)
raise AITrainingRequestError(msg)
# Create the workflow model
try:
workflow = AITrainingWorkflow.start_workflow(examples, course_id, item_id, algorithm_id)
except NoTrainingExamples as ex:
raise AITrainingRequestError(ex)
except:
msg = (
u"An unexpected error occurred while creating "
u"the AI training workflow"
)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Schedule the task, parametrized by the workflow UUID
try:
training_tasks.train_classifiers.apply_async(args=[workflow.uuid])
except ANTICIPATED_CELERY_ERRORS as ex:
msg = (
u"An unexpected error occurred while scheduling incomplete training workflows with"
u" course_id={cid} and item_id={iid}: {ex}"
).format(cid=course_id, iid=item_id, ex=ex)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Return the workflow UUID
return workflow.uuid
def setUp(self):
"""
Create a training workflow in the database.
"""
examples = deserialize_training_examples(EXAMPLES, RUBRIC)
workflow = AITrainingWorkflow.start_workflow(examples, self.COURSE_ID,
self.ITEM_ID,
self.ALGORITHM_ID)
self.workflow_uuid = workflow.uuid
def _assert_complete(self, training_done=None, grading_done=None):
"""
Asserts that the Training and Grading are of a given completion status
Serves as an assertion for a number of unit tests.
Args:
training_done (bool): whether the user expects there to be unfinished training workflows
grading_done (bool): whether the user expects there to be unfinished grading workflows
"""
incomplete_training_workflows = AITrainingWorkflow.get_incomplete_workflows(course_id=COURSE_ID, item_id=ITEM_ID)
incomplete_grading_workflows = AIGradingWorkflow.get_incomplete_workflows(course_id=COURSE_ID, item_id=ITEM_ID)
if training_done is not None:
self.assertEqual(self._is_empty_generator(incomplete_training_workflows), training_done)
if grading_done is not None:
self.assertEqual(self._is_empty_generator(incomplete_grading_workflows), grading_done)
def _assert_complete(self, training_done=None, grading_done=None):
"""
Asserts that the Training and Grading are of a given completion status
Serves as an assertion for a number of unit tests.
Args:
training_done (bool): whether the user expects there to be unfinished training workflows
grading_done (bool): whether the user expects there to be unfinished grading workflows
"""
incomplete_training_workflows = AITrainingWorkflow.get_incomplete_workflows(
course_id=COURSE_ID, item_id=ITEM_ID)
incomplete_grading_workflows = AIGradingWorkflow.get_incomplete_workflows(
course_id=COURSE_ID, item_id=ITEM_ID)
if training_done is not None:
self.assertEqual(
self._is_empty_generator(incomplete_training_workflows),
training_done)
if grading_done is not None:
self.assertEqual(
self._is_empty_generator(incomplete_grading_workflows),
grading_done)
def train_classifiers(rubric_dict, examples, course_id, item_id, algorithm_id):
"""
Schedule a task to train classifiers.
All training examples must match the rubric!
After training of classifiers completes successfully, all AIGradingWorkflows that are incomplete will be
automatically rescheduled to complete.
Args:
rubric_dict (dict): The rubric used to assess the classifiers.
examples (list of dict): Serialized training examples.
algorithm_id (unicode): The ID of the algorithm used to train the classifiers.
Returns:
training_workflow_uuid (str): The UUID of the training workflow.
Usually the caller will not need this (since the workers
are parametrized by training workflow UUID), but it's
useful for testing.
Raises:
AITrainingRequestError
AITrainingInternalError
Example usage:
>>> train_classifiers(rubric, examples, 'ease')
'10df7db776686822e501b05f452dc1e4b9141fe5'
"""
# Get or create the rubric and training examples
try:
examples = deserialize_training_examples(examples, rubric_dict)
except (InvalidRubric, InvalidTrainingExample, InvalidRubricSelection) as ex:
msg = u"Could not parse rubric and/or training examples: {ex}".format(ex=ex)
raise AITrainingRequestError(msg)
# Create the workflow model
try:
workflow = AITrainingWorkflow.start_workflow(examples, course_id, item_id, algorithm_id)
except NoTrainingExamples as ex:
raise AITrainingRequestError(ex)
except:
msg = (
u"An unexpected error occurred while creating "
u"the AI training workflow"
)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Schedule the task, parametrized by the workflow UUID
try:
training_tasks.train_classifiers.apply_async(args=[workflow.uuid])
except ANTICIPATED_CELERY_ERRORS as ex:
msg = (
u"An unexpected error occurred while scheduling incomplete training workflows with"
u" course_id={cid} and item_id={iid}: {ex}"
).format(cid=course_id, iid=item_id, ex=ex)
logger.exception(msg)
raise AITrainingInternalError(msg)
# Return the workflow UUID
return workflow.uuid