def create_classifiers(training_workflow_uuid, classifier_set): """ Upload trained classifiers and mark the workflow complete. If grading tasks were submitted before any classifiers were trained, this call will automatically reschedule those tasks. Args: training_workflow_uuid (str): The UUID of the training workflow. classifier_set (dict): Mapping of criteria names to serialized classifiers. Returns: None Raises: AITrainingRequestError AITrainingInternalError """ try: workflow = AITrainingWorkflow.objects.get(uuid=training_workflow_uuid) # If the task is executed multiple times, the classifier set may already # have been created. If so, log it, then return immediately. if workflow.is_complete: msg = u"AI training workflow with UUID {} already has trained classifiers.".format( workflow.uuid) logger.info(msg) else: workflow.complete(classifier_set) logger.info(( u"Created trained classifiers for the AI training workflow with UUID {workflow_uuid} " u"(using algorithm ID {algorithm_id})").format( workflow_uuid=workflow.uuid, algorithm_id=workflow.algorithm_id)) except AITrainingWorkflow.DoesNotExist: msg = (u"Could not retrieve AI training workflow with UUID {}" ).format(training_workflow_uuid) raise AITrainingRequestError(msg) except NoTrainingExamples as ex: logger.exception(ex) raise AITrainingInternalError(ex) except (IncompleteClassifierSet, InvalidRubricSelection) as ex: msg = (u"An error occurred while creating the classifier set " u"for the training workflow with UUID {uuid}: {ex}").format( uuid=training_workflow_uuid, ex=ex) raise AITrainingRequestError(msg) except (ClassifierSerializeError, ClassifierUploadError, DatabaseError) as ex: msg = (u"An unexpected error occurred while creating the classifier " u"set for training workflow UUID {uuid}: {ex}").format( uuid=training_workflow_uuid, ex=ex) logger.exception(msg) raise AITrainingInternalError(msg)
def is_training_workflow_complete(workflow_uuid): """ Check whether the training workflow is complete. Args: workflow_uuid (str): The UUID of the training workflow Returns: bool Raises: AITrainingRequestError AITrainingInternalError """ try: return AITrainingWorkflow.is_workflow_complete(workflow_uuid) except AITrainingWorkflow.DoesNotExist: msg = (u"Could not retrieve training workflow " u"with uuid {uuid} to check whether it's complete.").format( uuid=workflow_uuid) raise AITrainingRequestError(msg) except DatabaseError: msg = ( u"An unexpected error occurred while checking " u"the training workflow with uuid {uuid} for completeness").format( uuid=workflow_uuid) raise AITrainingInternalError(msg)
def train_classifiers(rubric_dict, examples, course_id, item_id, algorithm_id): """ Schedule a task to train classifiers. All training examples must match the rubric! After training of classifiers completes successfully, all AIGradingWorkflows that are incomplete will be automatically rescheduled to complete. Args: rubric_dict (dict): The rubric used to assess the classifiers. examples (list of dict): Serialized training examples. algorithm_id (unicode): The ID of the algorithm used to train the classifiers. Returns: training_workflow_uuid (str): The UUID of the training workflow. Usually the caller will not need this (since the workers are parametrized by training workflow UUID), but it's useful for testing. Raises: AITrainingRequestError AITrainingInternalError Example usage: >>> train_classifiers(rubric, examples, 'ease') '10df7db776686822e501b05f452dc1e4b9141fe5' """ # Get or create the rubric and training examples try: examples = deserialize_training_examples(examples, rubric_dict) except (InvalidRubric, InvalidTrainingExample, InvalidRubricSelection) as ex: msg = u"Could not parse rubric and/or training examples: {ex}".format(ex=ex) raise AITrainingRequestError(msg) # Create the workflow model try: workflow = AITrainingWorkflow.start_workflow(examples, course_id, item_id, algorithm_id) except NoTrainingExamples as ex: raise AITrainingRequestError(ex) except: msg = ( u"An unexpected error occurred while creating " u"the AI training workflow" ) logger.exception(msg) raise AITrainingInternalError(msg) # Schedule the task, parametrized by the workflow UUID try: training_tasks.train_classifiers.apply_async(args=[workflow.uuid]) except ANTICIPATED_CELERY_ERRORS as ex: msg = ( u"An unexpected error occurred while scheduling incomplete training workflows with" u" course_id={cid} and item_id={iid}: {ex}" ).format(cid=course_id, iid=item_id, ex=ex) logger.exception(msg) raise AITrainingInternalError(msg) # Return the workflow UUID return workflow.uuid
def test_create_classifiers_api_error(self, mock_call): mock_call.side_effect = AITrainingRequestError("Test error!") with self.assert_retry(train_classifiers, AITrainingRequestError): train_classifiers(self.workflow_uuid)
def get_training_task_params(training_workflow_uuid): """ Retrieve the training examples and algorithm ID associated with a training task. Args: training_workflow_uuid (str): The UUID of the training workflow. Returns: dict with keys: * training_examples (list of dict): The examples used to train the classifiers. * course_id (unicode): The course ID that the training task is associated with. * item_id (unicode): Identifies the item that the AI will be training to grade. * algorithm_id (unicode): The ID of the algorithm to use for training. Raises: AITrainingRequestError AITrainingInternalError Example usage: >>> params = get_training_task_params('abcd1234') >>> params['algorithm_id'] u'ease' >>> params['training_examples'] [ { "text": u"Example answer number one", "scores": { "vocabulary": 1, "grammar": 2 } }, { "text": u"Example answer number two", "scores": { "vocabulary": 3, "grammar": 1 } } ] """ try: workflow = AITrainingWorkflow.objects.get(uuid=training_workflow_uuid) returned_examples = [] for example in workflow.training_examples.all(): scores = { option.criterion.name: option.points for option in example.options_selected.all() } returned_examples.append({ 'text': essay_text_from_submission({'answer': example.answer}), 'scores': scores }) return { 'training_examples': returned_examples, 'algorithm_id': workflow.algorithm_id, 'course_id': workflow.course_id, 'item_id': workflow.item_id } except AITrainingWorkflow.DoesNotExist: msg = (u"Could not retrieve AI training workflow with UUID {}" ).format(training_workflow_uuid) raise AITrainingRequestError(msg) except DatabaseError: msg = (u"An unexpected error occurred while retrieving " u"training examples for the AI training workflow with UUID {}" ).format(training_workflow_uuid) logger.exception(msg) raise AITrainingInternalError(msg)