def is_grading_workflow_complete(workflow_uuid): """ Check whether the grading workflow is complete. Args: workflow_uuid (str): The UUID of the grading workflow Returns: bool Raises: AIGradingRequestError AIGradingInternalError """ try: return AIGradingWorkflow.is_workflow_complete(workflow_uuid) except AIGradingWorkflow.DoesNotExist: msg = (u"Could not retrieve grading workflow " u"with uuid {uuid} to check whether it's complete.").format( uuid=workflow_uuid) raise AIGradingRequestError(msg) except DatabaseError: msg = ( u"An unexpected error occurred while checking " u"the grading workflow with uuid {uuid} for completeness").format( uuid=workflow_uuid) raise AIGradingInternalError(msg)
def create_assessment(grading_workflow_uuid, criterion_scores): """ Create an AI assessment (complete the AI grading task). Args: grading_workflow_uuid (str): The UUID of the grading workflow. criterion_scores (dict): Dictionary mapping criteria names to integer scores. Returns: None Raises: AIGradingRequestError AIGradingInternalError """ try: workflow = AIGradingWorkflow.objects.get(uuid=grading_workflow_uuid) except AIGradingWorkflow.DoesNotExist: msg = (u"Could not retrieve the AI grading workflow with uuid {}" ).format(grading_workflow_uuid) raise AIGradingRequestError(msg) except DatabaseError as ex: msg = (u"An unexpected error occurred while retrieving the " u"AI grading workflow with uuid {uuid}: {ex}").format( uuid=grading_workflow_uuid, ex=ex) logger.exception(msg) raise AIGradingInternalError(msg) # Optimization: if the workflow has already been marked complete # (perhaps the task was picked up by multiple workers), # then we don't need to do anything. # Otherwise, create the assessment mark the workflow complete. try: if not workflow.is_complete: workflow.complete(criterion_scores) logger.info(( u"Created assessment for AI grading workflow with UUID {workflow_uuid} " u"(algorithm ID {algorithm_id})").format( workflow_uuid=workflow.uuid, algorithm_id=workflow.algorithm_id)) else: msg = u"Grading workflow with UUID {} is already marked complete".format( workflow.uuid) logger.info(msg) except DatabaseError as ex: msg = (u"An unexpected error occurred while creating the assessment " u"for AI grading workflow with uuid {uuid}: {ex}").format( uuid=grading_workflow_uuid, ex=ex) logger.exception(msg) raise AIGradingInternalError(msg) # Fire a signal to update the workflow API # This will allow students to receive a score if they're # waiting on an AI assessment. # The signal receiver is responsible for catching and logging # all exceptions that may occur when updating the workflow. from openassessment.assessment.signals import assessment_complete_signal assessment_complete_signal.send(sender=None, submission_uuid=workflow.submission_uuid)
def get_grading_task_params(grading_workflow_uuid): """ Retrieve the classifier set and algorithm ID associated with a particular grading workflow. Args: grading_workflow_uuid (str): The UUID of the grading workflow. Returns: dict with keys: * essay_text (unicode): The text of the essay submission. * classifier_set (dict): Maps criterion names to serialized classifiers. * valid_scores (dict): Maps criterion names to a list of valid scores for that criterion. * algorithm_id (unicode): ID of the algorithm used to perform training. Raises: AIGradingRequestError AIGradingInternalError """ try: workflow = AIGradingWorkflow.objects.get(uuid=grading_workflow_uuid) except AIGradingWorkflow.DoesNotExist: msg = (u"Could not retrieve the AI grading workflow with uuid {}" ).format(grading_workflow_uuid) raise AIGradingRequestError(msg) except DatabaseError as ex: msg = (u"An unexpected error occurred while retrieving the " u"AI grading workflow with uuid {uuid}: {ex}").format( uuid=grading_workflow_uuid, ex=ex) logger.exception(msg) raise AIGradingInternalError(msg) classifier_set = workflow.classifier_set # Though tasks shouldn't be scheduled until classifer set(s) exist, off of the happy path this is a likely # occurrence. Our response is to log this lack of compliance to dependency as an exception, and then thrown # an error with the purpose of killing the celery task running this code. if classifier_set is None: msg = ( u"AI grading workflow with UUID {} has no classifier set, but was scheduled for grading" ).format(grading_workflow_uuid) logger.exception(msg) raise AIGradingInternalError(msg) try: return { 'essay_text': workflow.essay_text, 'classifier_set': workflow.classifier_set.classifier_data_by_criterion, 'algorithm_id': workflow.algorithm_id, 'valid_scores': workflow.classifier_set.valid_scores_by_criterion, } except (DatabaseError, ClassifierSerializeError, IncompleteClassifierSet, ValueError, IOError, HTTPException) as ex: msg = (u"An unexpected error occurred while retrieving " u"classifiers for the grading workflow with UUID {uuid}: {ex}" ).format(uuid=grading_workflow_uuid, ex=ex) logger.exception(msg) raise AIGradingInternalError(msg)
def get_classifier_set_info(rubric_dict, algorithm_id, course_id, item_id): """ Get information about the classifier available for a particular problem. This is the classifier that would be selected to grade essays for the problem. Args: rubric_dict (dict): The serialized rubric model. algorithm_id (unicode): The algorithm to use for classification. course_id (unicode): The course identifier for the current problem. item_id (unicode): The item identifier for the current problem. Returns: dict with keys 'created_at', 'algorithm_id', 'course_id', and 'item_id' Note that course ID and item ID might be different than the current problem if a classifier from a different problem with a similar rubric is the best available match. """ try: rubric = rubric_from_dict(rubric_dict) classifier_set = AIClassifierSet.most_recent_classifier_set( rubric, algorithm_id, course_id, item_id ) if classifier_set is not None: return { 'created_at': classifier_set.created_at, 'algorithm_id': classifier_set.algorithm_id, 'course_id': classifier_set.course_id, 'item_id': classifier_set.item_id } else: return None except InvalidRubric: msg = u"Could not retrieve classifier set info: the rubric definition was not valid." logger.exception(msg) raise AIGradingRequestError(msg) except DatabaseError as ex: msg = u"An unexpected error occurred while retrieving classifier set info: {ex}".format(ex=ex) logger.exception(msg) raise AIGradingInternalError(msg)
def on_init(submission_uuid, rubric=None, algorithm_id=None): """ Submit a response for AI assessment. This will: (a) create a workflow (database record) to track the grading task (b) if classifiers exist for the rubric, schedule an asynchronous grading task. Args: submission_uuid (str): The UUID of the submission to assess. Keyword Arguments: rubric (dict): Serialized rubric model. algorithm_id (unicode): Use only classifiers trained with the specified algorithm. Returns: grading_workflow_uuid (str): The UUID of the grading workflow. Usually the caller of `submit()` won't need this (since the workers are parameterized by grading workflow UUID), but it's useful for testing. Raises: AIGradingRequestError AIGradingInternalError Example Usage: >>> on_init('74a9d63e8a5fea369fd391d07befbd86ae4dc6e2', rubric, 'ease') '10df7db776686822e501b05f452dc1e4b9141fe5' """ if rubric is None: raise AIGradingRequestError(u'No rubric provided') if algorithm_id is None: raise AIGradingRequestError(u'No algorithm ID provided') try: workflow = AIGradingWorkflow.start_workflow(submission_uuid, rubric, algorithm_id) except (sub_api.SubmissionNotFoundError, sub_api.SubmissionRequestError) as ex: msg = ( u"An error occurred while retrieving the " u"submission with UUID {uuid}: {ex}" ).format(uuid=submission_uuid, ex=ex) raise AIGradingRequestError(msg) except InvalidRubric as ex: msg = ( u"An error occurred while parsing the serialized " u"rubric {rubric}: {ex}" ).format(rubric=rubric, ex=ex) raise AIGradingRequestError(msg) except (sub_api.SubmissionInternalError, DatabaseError) as ex: msg = ( u"An unexpected error occurred while submitting an " u"essay for AI grading: {ex}" ).format(ex=ex) logger.exception(msg) raise AIGradingInternalError(msg) # If we find classifiers for this rubric/algorithm # then associate the classifiers with the workflow # and schedule a grading task. # Otherwise, the task will need to be scheduled later, # once the classifiers have been trained. if workflow.classifier_set is not None: try: grading_tasks.grade_essay.apply_async(args=[workflow.uuid]) logger.info(( u"Scheduled grading task for AI grading workflow with UUID {workflow_uuid} " u"(submission UUID = {sub_uuid}, algorithm ID = {algorithm_id})" ).format(workflow_uuid=workflow.uuid, sub_uuid=submission_uuid, algorithm_id=algorithm_id)) return workflow.uuid except (DatabaseError,) + ANTICIPATED_CELERY_ERRORS as ex: msg = ( u"An unexpected error occurred while scheduling the " u"AI grading task for the submission with UUID {uuid}: {ex}" ).format(uuid=submission_uuid, ex=ex) logger.exception(msg) raise AIGradingInternalError(msg) else: logger.info(( u"Cannot schedule a grading task for AI grading workflow with UUID {workflow_uuid} " u"because no classifiers are available for the rubric associated with submission {sub_uuid} " u"for the algorithm {algorithm_id}" ).format(workflow_uuid=workflow.uuid, sub_uuid=submission_uuid, algorithm_id=algorithm_id))