def test_missing_options(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data("data/rubric/missing_options.json")) self.assertEqual( cm.exception.errors, {"criteria": [{"options": [u"This field is required."]}, {}]}, # No errors in second criterion )
def test_empty_criteria(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data('data/rubric/empty_criteria.json')) self.assertEqual( cm.exception.errors, {'criteria': [u'Must have at least one criterion']} )
def test_empty_criteria(self): with self.assertRaises(InvalidRubric) as criteria_exception_message: rubric_from_dict(json_data('data/rubric/empty_criteria.json')) self.assertEqual( criteria_exception_message.exception.errors, {'criteria': ['Must have at least one criterion']} )
def test_missing_criteria(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data('data/rubric/missing_criteria.json')) self.assertEqual( cm.exception.errors, {'criteria': [u'This field is required.']} )
def test_missing_criteria(self): with self.assertRaises(InvalidRubric) as criteria_exception_message: rubric_from_dict(json_data('data/rubric/missing_criteria.json')) self.assertEqual( criteria_exception_message.exception.errors, {'criteria': ['This field is required.']} )
def validate_rubric(rubric_dict, current_rubric, is_released): """ Check that the rubric is semantically valid. Args: rubric_dict (dict): Serialized Rubric model representing the updated state of the rubric. current_rubric (dict): Serialized Rubric model representing the current state of the rubric. is_released (bool): True if and only if the problem has been released. Returns: tuple (is_valid, msg) where is_valid is a boolean indicating whether the assessment is semantically valid and msg describes any validation errors found. """ try: rubric_from_dict(rubric_dict) except InvalidRubric: return (False, u'This rubric definition is not valid.') # No duplicate criteria names duplicates = _duplicates([criterion['name'] for criterion in rubric_dict['criteria']]) if len(duplicates) > 0: msg = u"Criteria duplicate name(s): {duplicates}".format( duplicates=", ".join(duplicates) ) return (False, msg) # No duplicate option names within a criterion for criterion in rubric_dict['criteria']: duplicates = _duplicates([option['name'] for option in criterion['options']]) if len(duplicates) > 0: msg = u"Options in '{criterion}' have duplicate name(s): {duplicates}".format( criterion=criterion['name'], duplicates=", ".join(duplicates) ) return (False, msg) # After a problem is released, authors are allowed to change text, # but nothing that would change the point value of a rubric. if is_released: # Number of criteria must be the same if len(rubric_dict['criteria']) != len(current_rubric['criteria']): return (False, u'The number of criteria cannot be changed after a problem is released.') # Number of options for each criterion must be the same for new_criterion, old_criterion in _match_by_order(rubric_dict['criteria'], current_rubric['criteria']): if len(new_criterion['options']) != len(old_criterion['options']): return (False, u'The number of options cannot be changed after a problem is released.') else: for new_option, old_option in _match_by_order(new_criterion['options'], old_criterion['options']): if new_option['points'] != old_option['points']: return (False, u'Point values cannot be changed after a problem is released.') return (True, u'')
def test_empty_options(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data("data/rubric/empty_options.json")) self.assertEqual( cm.exception.errors, { "criteria": [ {}, # There are no errors in the first criterion {"options": [u"Criterion must have at least one option."]}, ] }, )
def test_missing_options(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data('rubric_data/missing_options.json')) self.assertEqual( cm.exception.errors, { 'criteria': [ {'options': [u'This field is required.']}, {} # No errors in second criterion ] } )
def test_empty_options(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data('rubric_data/empty_options.json')) self.assertEqual( cm.exception.errors, { 'criteria': [ {}, # There are no errors in the first criterion {'options': [u'Criterion must have at least one option.']} ] } )
def test_rubric_only_created_once(self): # Make sure sending the same Rubric data twice only creates one Rubric, # and returns a reference to it the next time. rubric_data = json_data('data/rubric/project_plan_rubric.json') r1 = rubric_from_dict(rubric_data) with self.assertNumQueries(1): # Just the select -- shouldn't need the create queries r2 = rubric_from_dict(rubric_data) self.assertEqual(r1.id, r2.id) r1.delete()
def test_missing_options(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data('data/rubric/missing_options.json')) self.assertEqual( cm.exception.errors, { 'criteria': [ { 'options': [u'This field is required.'] }, {} # No errors in second criterion ] })
def test_rubric_only_created_once(self): # Make sure sending the same Rubric data twice only creates one Rubric, # and returns a reference to it the next time. rubric_data = json_data("data/rubric/project_plan_rubric.json") r1 = rubric_from_dict(rubric_data) with self.assertNumQueries(1): # Just the select -- shouldn't need the create queries r2 = rubric_from_dict(rubric_data) self.assertEqual(r1.id, r2.id) r1.delete()
def test_empty_options(self): with self.assertRaises(InvalidRubric) as cm: rubric_from_dict(json_data('data/rubric/empty_options.json')) self.assertEqual( cm.exception.errors, { 'criteria': [ {}, # There are no errors in the first criterion { 'options': [u'Criterion must have at least one option.'] } ] })
def setUp(self): """ Create a new grading workflow. """ self.rubric = rubric_from_dict(RUBRIC) self.workflow = AIGradingWorkflow.objects.create( submission_uuid='test', essay_text='test', rubric=self.rubric, algorithm_id=self.ALGORITHM_ID, item_id=self.ITEM_ID, course_id=self.COURSE_ID ) # Create a rubric with a similar structure, but different prompt similar_rubric_dict = copy.deepcopy(RUBRIC) similar_rubric_dict['prompt'] = 'Different prompt!' self.similar_rubric = rubric_from_dict(similar_rubric_dict)
def test_full_assessment_dict_criteria_no_options(self): # Create a rubric with a criterion that has no options (just feedback) rubric_dict = copy.deepcopy(RUBRIC) rubric_dict['criteria'].append({ 'order_num': 2, 'name': 'feedback only', 'prompt': 'feedback only', 'options': [] }) rubric = rubric_from_dict(rubric_dict) # Create an assessment for the rubric assessment = Assessment.create(rubric, "Bob", "submission UUID", "PE") selected = { u"vøȼȺƀᵾłȺɍɏ": u"𝓰𝓸𝓸𝓭", u"ﻭɼค๓๓คɼ": u"єχ¢єℓℓєηт", } feedback = { u"feedback only": u"enjoy the feedback!" } AssessmentPart.create_from_option_names(assessment, selected, feedback=feedback) # Serialize the assessment serialized = full_assessment_dict(assessment) # Verify that the assessment dict correctly serialized the criterion with options. self.assertEqual(serialized['parts'][0]['criterion']['name'], u"vøȼȺƀᵾłȺɍɏ") self.assertEqual(serialized['parts'][0]['option']['name'], u"𝓰𝓸𝓸𝓭") self.assertEqual(serialized['parts'][1]['criterion']['name'], u"ﻭɼค๓๓คɼ") self.assertEqual(serialized['parts'][1]['option']['name'], u"єχ¢єℓℓєηт") # Verify that the assessment dict correctly serialized the criterion with no options. self.assertIs(serialized['parts'][2]['option'], None) self.assertEqual(serialized['parts'][2]['criterion']['name'], u"feedback only")
def test_full_assessment_dict_criteria_no_options(self): # Create a rubric with a criterion that has no options (just feedback) rubric_dict = copy.deepcopy(RUBRIC) rubric_dict['criteria'].append({ 'order_num': 2, 'name': 'feedback only', 'prompt': 'feedback only', 'options': [] }) rubric = rubric_from_dict(rubric_dict) # Create an assessment for the rubric assessment = Assessment.create(rubric, "Bob", "submission-UUID", "PE") selected = { "vøȼȺƀᵾłȺɍɏ": "𝓰𝓸𝓸𝓭", "ﻭɼค๓๓คɼ": "єχ¢єℓℓєηт", } feedback = { "feedback only": "enjoy the feedback!" } AssessmentPart.create_from_option_names(assessment, selected, feedback=feedback) # Serialize the assessment serialized = full_assessment_dict(assessment) # Verify that the assessment dict correctly serialized the criterion with options. self.assertEqual(serialized['parts'][0]['criterion']['name'], "vøȼȺƀᵾłȺɍɏ") self.assertEqual(serialized['parts'][0]['option']['name'], "𝓰𝓸𝓸𝓭") self.assertEqual(serialized['parts'][1]['criterion']['name'], "ﻭɼค๓๓คɼ") self.assertEqual(serialized['parts'][1]['option']['name'], "єχ¢єℓℓєηт") # Verify that the assessment dict correctly serialized the criterion with no options. self.assertIs(serialized['parts'][2]['option'], None) self.assertEqual(serialized['parts'][2]['criterion']['name'], "feedback only")
def get_data_for_override_score(submission_uuid, student_item, rubric_dict): """ Gets the data needed to render the score override section of the student data page. Args: submission_uuid (string): UUID of the student's submission student_item (dict): The dictionary representation of a student item. rubric_dict (dict): The dictionary representation of a rubric Returns: scores (dict): points_earned, pointed_possible, points_override """ scores = {} score_data = sub_api.get_latest_score_for_submission(submission_uuid) if score_data: scores["points_earned"] = score_data.get("points_earned") scores["points_possible"] = score_data.get("points_possible") # If there is no Score, retrieve points_possible from the rubric as there may not be any assessments yet. if not scores.get("points_possible"): try: rubric = rubric_from_dict(rubric_dict) except InvalidRubric: points_possible = None else: points_possible = rubric.points_possible scores["points_possible"] = points_possible latest_override_score = sub_api.get_score_override(student_item) if latest_override_score: scores["points_override"] = latest_override_score["points_earned"] return scores
def handle(self, *args, **options): """ Execute the command. Args: course_id (unicode): The ID of the course to create submissions/workflows in. item_id (unicode): The ID of the problem in the course. num_submissions (int): The number of submissions/workflows to create. algorithm_id (unicode): The ID of the ML algorithm to use ("fake" or "ease") Raises: CommandError """ if len(args) < 4: raise CommandError(u"Usage: simulate_ai_grading_error {}".format( self.args)) # Parse arguments course_id = args[0].decode('utf-8') item_id = args[1].decode('utf-8') num_submissions = int(args[2]) algorithm_id = args[3].decode('utf-8') # Create the rubric model rubric = rubric_from_dict(self.RUBRIC) # Train classifiers print u"Training classifiers using {algorithm_id}...".format( algorithm_id=algorithm_id) algorithm = AIAlgorithm.algorithm_for_id(algorithm_id) classifier_data = { criterion_name: algorithm.train_classifier(example) for criterion_name, example in self.EXAMPLES.iteritems() } print u"Successfully trained classifiers." # Create the classifier set classifier_set = AIClassifierSet.create_classifier_set( classifier_data, rubric, algorithm_id, course_id, item_id) print u"Successfully created classifier set with id {}".format( classifier_set.pk) # Create submissions and grading workflows for num in range(num_submissions): student_item = { 'course_id': course_id, 'item_id': item_id, 'item_type': 'openassessment', 'student_id': "{base}_{num}".format(base=self.STUDENT_ID, num=num) } submission = sub_api.create_submission(student_item, self.ANSWER) workflow = AIGradingWorkflow.start_workflow( submission['uuid'], self.RUBRIC, algorithm_id) workflow.classifier_set = classifier_set workflow.save() print u"{num}: Created incomplete grading workflow with UUID {uuid}".format( num=num, uuid=workflow.uuid)
def _create_classifier(self): """ Create and return an AIClassifier. """ rubric = rubric_from_dict(RUBRIC) classifier_set = AIClassifierSet.create_classifier_set( CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID) return AIClassifier.objects.filter(classifier_set=classifier_set)[0]
def setUp(self): """ Create a new grading workflow. """ self.rubric = rubric_from_dict(RUBRIC) self.workflow = AIGradingWorkflow.objects.create( submission_uuid='test', essay_text='test', rubric=self.rubric, algorithm_id=self.ALGORITHM_ID, item_id=self.ITEM_ID, course_id=self.COURSE_ID) # Create a rubric with a similar structure, but different prompt similar_rubric_dict = copy.deepcopy(RUBRIC) similar_rubric_dict['prompt'] = 'Different prompt!' self.similar_rubric = rubric_from_dict(similar_rubric_dict)
def handle(self, *args, **options): """ Execute the command. Args: course_id (unicode): The ID of the course to create submissions/workflows in. item_id (unicode): The ID of the problem in the course. num_submissions (int): The number of submissions/workflows to create. algorithm_id (unicode): The ID of the ML algorithm to use ("fake" or "ease") Raises: CommandError """ if len(args) < 4: raise CommandError(u"Usage: simulate_ai_grading_error {}".format(self.args)) # Parse arguments course_id = args[0].decode('utf-8') item_id = args[1].decode('utf-8') num_submissions = int(args[2]) algorithm_id = args[3].decode('utf-8') # Create the rubric model rubric = rubric_from_dict(self.RUBRIC) # Train classifiers print u"Training classifiers using {algorithm_id}...".format(algorithm_id=algorithm_id) algorithm = AIAlgorithm.algorithm_for_id(algorithm_id) classifier_data = { criterion_name: algorithm.train_classifier(example) for criterion_name, example in self.EXAMPLES.iteritems() } print u"Successfully trained classifiers." # Create the classifier set classifier_set = AIClassifierSet.create_classifier_set( classifier_data, rubric, algorithm_id, course_id, item_id ) print u"Successfully created classifier set with id {}".format(classifier_set.pk) # Create submissions and grading workflows for num in range(num_submissions): student_item = { 'course_id': course_id, 'item_id': item_id, 'item_type': 'openassessment', 'student_id': "{base}_{num}".format(base=self.STUDENT_ID, num=num) } submission = sub_api.create_submission(student_item, self.ANSWER) workflow = AIGradingWorkflow.start_workflow( submission['uuid'], self.RUBRIC, algorithm_id ) workflow.classifier_set = classifier_set workflow.save() print u"{num}: Created incomplete grading workflow with UUID {uuid}".format( num=num, uuid=workflow.uuid )
def _complete_assessment( rubric_dict, scorer_id, peer_submission_uuid, options_selected, criterion_feedback, scorer_workflow, overall_feedback, num_required_grades, scored_at, ): """ Internal function for atomic assessment creation. Creates a peer assessment and closes the associated peer workflow item in a single transaction. Args: rubric_dict (dict): The rubric model associated with this assessment scorer_id (str): The user ID for the user giving this assessment. This is required to create an assessment on a submission. peer_submission_uuid (str): The submission uuid for the submission being assessed. options_selected (dict): Dictionary mapping criterion names to the option names the user selected for that criterion. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. scorer_workflow (PeerWorkflow): The PeerWorkflow associated with the scorer. Updates the workflow item associated with this assessment. overall_feedback (unicode): Free-form text feedback on the submission overall. num_required_grades (int): The required number of assessments a submission requires before it is completed. If this number of assessments is reached, the grading_completed_at timestamp is set for the Workflow. scored_at (datetime): Optional argument to override the time in which the assessment took place. If not specified, scored_at is set to now. Returns: The Assessment model """ # Get or create the rubric rubric = rubric_from_dict(rubric_dict) # Create the peer assessment assessment = Assessment.create( rubric, scorer_id, peer_submission_uuid, PEER_TYPE, scored_at=scored_at, feedback=overall_feedback ) # Create assessment parts for each criterion in the rubric # This will raise an `InvalidRubricSelection` if the selected options do not # match the rubric. AssessmentPart.create_from_option_names(assessment, options_selected, feedback=criterion_feedback) # Close the active assessment scorer_workflow.close_active_assessment(peer_submission_uuid, assessment, num_required_grades) return assessment
def _complete_assessment( submission_uuid, scorer_id, options_selected, criterion_feedback, overall_feedback, rubric_dict, scored_at, scorer_workflow ): """ Internal function for atomic assessment creation. Creates a staff assessment in a single transaction. Args: submission_uuid (str): The submission uuid for the submission being assessed. scorer_id (str): The user ID for the user giving this assessment. This is required to create an assessment on a submission. options_selected (dict): Dictionary mapping criterion names to the option names the user selected for that criterion. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. overall_feedback (unicode): Free-form text feedback on the submission overall. rubric_dict (dict): The rubric model associated with this assessment scored_at (datetime): Optional argument to override the time in which the assessment took place. If not specified, scored_at is set to now. Returns: The Assessment model """ # Get or create the rubric rubric = rubric_from_dict(rubric_dict) # Create the staff assessment assessment = Assessment.create( rubric, scorer_id, submission_uuid, STAFF_TYPE, scored_at=scored_at, feedback=overall_feedback ) # Create assessment parts for each criterion in the rubric # This will raise an `InvalidRubricSelection` if the selected options do not # match the rubric. AssessmentPart.create_from_option_names(assessment, options_selected, feedback=criterion_feedback) # Close the active assessment if scorer_workflow is not None: scorer_workflow.close_active_assessment(assessment, scorer_id) return assessment
def _create_classifier(self): """ Create and return an AIClassifier. """ rubric = rubric_from_dict(RUBRIC) classifier_set = AIClassifierSet.create_classifier_set( CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID ) return AIClassifier.objects.filter(classifier_set=classifier_set)[0]
def _complete_assessment(rubric_dict, scorer_id, peer_submission_uuid, options_selected, criterion_feedback, scorer_workflow, overall_feedback, num_required_grades, scored_at): """ Internal function for atomic assessment creation. Creates a peer assessment and closes the associated peer workflow item in a single transaction. Args: rubric_dict (dict): The rubric model associated with this assessment scorer_id (str): The user ID for the user giving this assessment. This is required to create an assessment on a submission. peer_submission_uuid (str): The submission uuid for the submission being assessed. options_selected (dict): Dictionary mapping criterion names to the option names the user selected for that criterion. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. scorer_workflow (PeerWorkflow): The PeerWorkflow associated with the scorer. Updates the workflow item associated with this assessment. overall_feedback (unicode): Free-form text feedback on the submission overall. num_required_grades (int): The required number of assessments a submission requires before it is completed. If this number of assessments is reached, the grading_completed_at timestamp is set for the Workflow. scored_at (datetime): Optional argument to override the time in which the assessment took place. If not specified, scored_at is set to now. Returns: The Assessment model """ # Get or create the rubric rubric = rubric_from_dict(rubric_dict) # Create the peer assessment assessment = Assessment.create(rubric, scorer_id, peer_submission_uuid, PEER_TYPE, scored_at=scored_at, feedback=overall_feedback) # Create assessment parts for each criterion in the rubric # This will raise an `InvalidRubricSelection` if the selected options do not # match the rubric. AssessmentPart.create_from_option_names(assessment, options_selected, feedback=criterion_feedback) # Close the active assessment scorer_workflow.close_active_assessment(peer_submission_uuid, assessment, num_required_grades) return assessment
def _rubric_with_one_feedback_only_criterion(self): """Create a rubric with one feedback-only criterion.""" rubric_dict = copy.deepcopy(RUBRIC) rubric_dict['criteria'].append({ "order_num": 2, "name": u"feedback", "prompt": u"only feedback, no points", "options": [] }) return rubric_from_dict(rubric_dict)
def start_workflow(cls, submission_uuid, rubric_dict, algorithm_id): """ Start a grading workflow. Args: submission_uuid (str): The UUID of the submission to grade. rubric_dict (dict): The serialized rubric model. algorithm_id (unicode): The ID of the algorithm to use for grading. Returns: AIGradingWorkflow Raises: SubmissionNotFoundError SubmissionRequestError SubmissionInternalError InvalidRubric DatabaseError """ # Retrieve info about the submission submission = sub_api.get_submission_and_student(submission_uuid) # Get or create the rubric from openassessment.assessment.serializers import rubric_from_dict rubric = rubric_from_dict(rubric_dict) # Retrieve the submission text # Submissions are arbitrary JSON-blobs, which *should* # contain a single key, "answer", containing the essay # submission text. If not, though, assume we've been # given the essay text directly (convenient for testing). if isinstance(submission, dict): essay_text = submission.get('answer') else: essay_text = unicode(submission) # Create the workflow workflow = cls.objects.create( submission_uuid=submission_uuid, essay_text=essay_text, algorithm_id=algorithm_id, student_id=submission['student_item']['student_id'], item_id=submission['student_item']['item_id'], course_id=submission['student_item']['course_id'], rubric=rubric ) # Retrieve and assign classifier set candidates workflow.assign_most_recent_classifier_set() workflow._log_start_workflow() return workflow
def start_workflow(cls, submission_uuid, rubric_dict, algorithm_id): """ Start a grading workflow. Args: submission_uuid (str): The UUID of the submission to grade. rubric_dict (dict): The serialized rubric model. algorithm_id (unicode): The ID of the algorithm to use for grading. Returns: AIGradingWorkflow Raises: SubmissionNotFoundError SubmissionRequestError SubmissionInternalError InvalidRubric DatabaseError """ # Retrieve info about the submission submission = sub_api.get_submission_and_student(submission_uuid) # Get or create the rubric from openassessment.assessment.serializers import rubric_from_dict rubric = rubric_from_dict(rubric_dict) # Retrieve the submission text # Submissions are arbitrary JSON-blobs, which *should* # contain a single key, "answer", containing the essay # submission text. If not, though, assume we've been # given the essay text directly (convenient for testing). if isinstance(submission, dict): essay_text = submission.get('answer') else: essay_text = unicode(submission) # Create the workflow workflow = cls.objects.create( submission_uuid=submission_uuid, essay_text=essay_text, algorithm_id=algorithm_id, student_id=submission['student_item']['student_id'], item_id=submission['student_item']['item_id'], course_id=submission['student_item']['course_id'], rubric=rubric) # Retrieve and assign classifier set candidates workflow.assign_most_recent_classifier_set() workflow._log_start_workflow() return workflow
def train_classifiers(rubric_dict, classifier_score_overrides): """ Simple utility function to train classifiers. Args: rubric_dict (dict): The rubric to train the classifiers on. classifier_score_overrides (dict): A dictionary of classifier overrides to set the scores for the given submission. """ rubric = rubric_from_dict(rubric_dict) AIClassifierSet.create_classifier_set(classifier_score_overrides, rubric, ALGORITHM_ID, COURSE_ID, ITEM_ID)
def train_classifiers(rubric_dict, classifier_score_overrides): """ Simple utility function to train classifiers. Args: rubric_dict (dict): The rubric to train the classifiers on. classifier_score_overrides (dict): A dictionary of classifier overrides to set the scores for the given submission. """ rubric = rubric_from_dict(rubric_dict) AIClassifierSet.create_classifier_set( classifier_score_overrides, rubric, ALGORITHM_ID, COURSE_ID, ITEM_ID )
def _complete_assessment( submission_uuid, user_id, options_selected, criterion_feedback, overall_feedback, rubric_dict, scored_at ): """ Internal function for creating an assessment and its parts atomically. Args: submission_uuid (str): The unique identifier for the submission being assessed. user_id (str): The ID of the user creating the assessment. This must match the ID of the user who made the submission. options_selected (dict): Mapping of rubric criterion names to option values selected. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. overall_feedback (unicode): Free-form text feedback on the submission overall. rubric_dict (dict): Serialized Rubric model. scored_at (datetime): The timestamp of the assessment. Returns: Assessment model """ # Get or create the rubric rubric = rubric_from_dict(rubric_dict) # Create the self assessment assessment = Assessment.create( rubric, user_id, submission_uuid, SELF_TYPE, scored_at=scored_at, feedback=overall_feedback ) # This will raise an `InvalidRubricSelection` if the selected options do not match the rubric. AssessmentPart.create_from_option_names(assessment, options_selected, feedback=criterion_feedback) return assessment
def start_workflow(cls, submission_uuid, rubric_dict, algorithm_id): """ Start a grading workflow. Args: submission_uuid (str): The UUID of the submission to grade. rubric_dict (dict): The serialized rubric model. algorithm_id (unicode): The ID of the algorithm to use for grading. Returns: AIGradingWorkflow Raises: SubmissionNotFoundError SubmissionRequestError SubmissionInternalError InvalidRubric DatabaseError """ # Retrieve info about the submission submission = sub_api.get_submission_and_student(submission_uuid) # Get or create the rubric from openassessment.assessment.serializers import rubric_from_dict rubric = rubric_from_dict(rubric_dict) # Create the workflow workflow = cls.objects.create( submission_uuid=submission_uuid, essay_text=essay_text_from_submission(submission), algorithm_id=algorithm_id, student_id=submission['student_item']['student_id'], item_id=submission['student_item']['item_id'], course_id=submission['student_item']['course_id'], rubric=rubric ) # Retrieve and assign classifier set candidates workflow.assign_most_recent_classifier_set() workflow._log_start_workflow() return workflow
def setUp(self): """ Create a submission and grading workflow. """ # Create a submission submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) self.submission_uuid = submission['uuid'] # Create a workflow for the submission workflow = AIGradingWorkflow.start_workflow(self.submission_uuid, RUBRIC, ALGORITHM_ID) self.workflow_uuid = workflow.uuid # Associate the workflow with classifiers rubric = rubric_from_dict(RUBRIC) classifier_set = AIClassifierSet.create_classifier_set( self.CLASSIFIERS, rubric, ALGORITHM_ID, STUDENT_ITEM.get('course_id'), STUDENT_ITEM.get('item_id') ) workflow.classifier_set = classifier_set workflow.save()
def start_workflow(cls, submission_uuid, rubric_dict, algorithm_id): """ Start a grading workflow. Args: submission_uuid (str): The UUID of the submission to grade. rubric_dict (dict): The serialized rubric model. algorithm_id (unicode): The ID of the algorithm to use for grading. Returns: AIGradingWorkflow Raises: SubmissionNotFoundError SubmissionRequestError SubmissionInternalError InvalidRubric DatabaseError """ # Retrieve info about the submission submission = sub_api.get_submission_and_student(submission_uuid) # Get or create the rubric from openassessment.assessment.serializers import rubric_from_dict rubric = rubric_from_dict(rubric_dict) # Create the workflow workflow = cls.objects.create( submission_uuid=submission_uuid, essay_text=essay_text_from_submission(submission), algorithm_id=algorithm_id, student_id=submission['student_item']['student_id'], item_id=submission['student_item']['item_id'], course_id=submission['student_item']['course_id'], rubric=rubric) # Retrieve and assign classifier set candidates workflow.assign_most_recent_classifier_set() workflow._log_start_workflow() return workflow
def setUp(self): """ Create a submission and grading workflow. """ # Create a submission submission = sub_api.create_submission(STUDENT_ITEM, ANSWER) self.submission_uuid = submission['uuid'] # Create a workflow for the submission workflow = AIGradingWorkflow.start_workflow(self.submission_uuid, RUBRIC, ALGORITHM_ID) self.workflow_uuid = workflow.uuid # Associate the workflow with classifiers rubric = rubric_from_dict(RUBRIC) classifier_set = AIClassifierSet.create_classifier_set( self.CLASSIFIERS, rubric, ALGORITHM_ID, STUDENT_ITEM.get('course_id'), STUDENT_ITEM.get('item_id')) workflow.classifier_set = classifier_set workflow.save()
def get_classifier_set_info(rubric_dict, algorithm_id, course_id, item_id): """ Get information about the classifier available for a particular problem. This is the classifier that would be selected to grade essays for the problem. Args: rubric_dict (dict): The serialized rubric model. algorithm_id (unicode): The algorithm to use for classification. course_id (unicode): The course identifier for the current problem. item_id (unicode): The item identifier for the current problem. Returns: dict with keys 'created_at', 'algorithm_id', 'course_id', and 'item_id' Note that course ID and item ID might be different than the current problem if a classifier from a different problem with a similar rubric is the best available match. """ try: rubric = rubric_from_dict(rubric_dict) classifier_set = AIClassifierSet.most_recent_classifier_set( rubric, algorithm_id, course_id, item_id ) if classifier_set is not None: return { 'created_at': classifier_set.created_at, 'algorithm_id': classifier_set.algorithm_id, 'course_id': classifier_set.course_id, 'item_id': classifier_set.item_id } else: return None except InvalidRubric: msg = u"Could not retrieve classifier set info: the rubric definition was not valid." logger.exception(msg) raise AIGradingRequestError(msg) except DatabaseError as ex: msg = u"An unexpected error occurred while retrieving classifier set info: {ex}".format(ex=ex) logger.exception(msg) raise AIGradingInternalError(msg)
def validate_rubric(rubric_dict, current_rubric, is_released, _): """ Check that the rubric is semantically valid. Args: rubric_dict (dict): Serialized Rubric model representing the updated state of the rubric. current_rubric (dict): Serialized Rubric model representing the current state of the rubric. is_released (bool): True if and only if the problem has been released. _ (function): The service function used to get the appropriate i18n text Returns: tuple (is_valid, msg) where is_valid is a boolean indicating whether the assessment is semantically valid and msg describes any validation errors found. """ try: rubric_from_dict(rubric_dict) except InvalidRubric: return False, _(u'This rubric definition is not valid.') for criterion in rubric_dict['criteria']: # No duplicate option names within a criterion duplicates = _duplicates( [option['name'] for option in criterion['options']]) if len(duplicates) > 0: msg = _( u"Options in '{criterion}' have duplicate name(s): {duplicates}" ).format(criterion=criterion['name'], duplicates=", ".join(duplicates)) return False, msg # Some criteria may have no options, just written feedback. # In this case, written feedback must be required (not optional or disabled). if len(criterion['options']) == 0 and criterion.get( 'feedback', 'disabled') != 'required': msg = _(u'Criteria with no options must require written feedback.') return False, msg # After a problem is released, authors are allowed to change text, # but nothing that would change the point value of a rubric. if is_released: # Number of prompts must be the same if len(rubric_dict['prompts']) != len(current_rubric['prompts']): return False, _( u'Prompts cannot be created or deleted after a problem is released.' ) # Number of criteria must be the same if len(rubric_dict['criteria']) != len(current_rubric['criteria']): return False, _( u'The number of criteria cannot be changed after a problem is released.' ) # Criteria names must be the same # We use criteria names as unique identifiers (unfortunately) # throughout the system. Changing them mid-flight can cause # the grade page, for example, to raise 500 errors. # When we implement non-XML authoring, we might be able to fix this # the right way by assigning unique identifiers for criteria; # but for now, this is the safest way to avoid breaking problems # post-release. current_criterion_names = set( criterion.get('name') for criterion in current_rubric['criteria']) new_criterion_names = set( criterion.get('name') for criterion in rubric_dict['criteria']) if current_criterion_names != new_criterion_names: return False, _( u'Criteria names cannot be changed after a problem is released' ) # Number of options for each criterion must be the same for new_criterion, old_criterion in _match_by_order( rubric_dict['criteria'], current_rubric['criteria']): if len(new_criterion['options']) != len(old_criterion['options']): return False, _( u'The number of options cannot be changed after a problem is released.' ) else: for new_option, old_option in _match_by_order( new_criterion['options'], old_criterion['options']): if new_option['points'] != old_option['points']: return False, _( u'Point values cannot be changed after a problem is released.' ) return True, u''
def test_rubric_requires_positive_score(self): with self.assertRaises(InvalidRubric): rubric_from_dict(json_data("data/rubric/no_points.json"))
def create_assessment( scorer_submission_uuid, scorer_id, options_selected, criterion_feedback, overall_feedback, rubric_dict, num_required_grades, scored_at=None ): """Creates an assessment on the given submission. Assessments are created based on feedback associated with a particular rubric. Args: scorer_submission_uuid (str): The submission uuid for the Scorer's workflow. The submission being assessed can be determined via the peer workflow of the grading student. scorer_id (str): The user ID for the user giving this assessment. This is required to create an assessment on a submission. options_selected (dict): Dictionary mapping criterion names to the option names the user selected for that criterion. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. overall_feedback (unicode): Free-form text feedback on the submission overall. num_required_grades (int): The required number of assessments a submission requires before it is completed. If this number of assessments is reached, the grading_completed_at timestamp is set for the Workflow. Kwargs: scored_at (datetime): Optional argument to override the time in which the assessment took place. If not specified, scored_at is set to now. Returns: dict: the Assessment model, serialized as a dict. Raises: PeerAssessmentRequestError: Raised when the submission_id is invalid, or the assessment_dict does not contain the required values to create an assessment. PeerAssessmentInternalError: Raised when there is an internal error while creating a new assessment. Examples: >>> options_selected = {"clarity": "Very clear", "precision": "Somewhat precise"} >>> criterion_feedback = {"clarity": "I thought this essay was very clear."} >>> feedback = "Your submission was thrilling." >>> create_assessment("1", "Tim", options_selected, criterion_feedback, feedback, rubric_dict) """ try: # Retrieve workflow information scorer_workflow = PeerWorkflow.objects.get(submission_uuid=scorer_submission_uuid) peer_workflow_item = scorer_workflow.get_latest_open_workflow_item() if peer_workflow_item is None: message = ( u"There are no open assessments associated with the scorer's " u"submission UUID {}." ).format(scorer_submission_uuid) logger.warning(message) raise PeerAssessmentWorkflowError(message) peer_submission_uuid = peer_workflow_item.author.submission_uuid # Get or create the rubric rubric = rubric_from_dict(rubric_dict) # Create the peer assessment assessment = Assessment.create( rubric, scorer_id, peer_submission_uuid, PEER_TYPE, scored_at=scored_at, feedback=overall_feedback ) # Create assessment parts for each criterion in the rubric # This will raise an `InvalidRubricSelection` if the selected options do not match the rubric. AssessmentPart.create_from_option_names(assessment, options_selected, feedback=criterion_feedback) # Close the active assessment scorer_workflow.close_active_assessment(peer_submission_uuid, assessment, num_required_grades) _log_assessment(assessment, scorer_workflow) return full_assessment_dict(assessment) except PeerWorkflow.DoesNotExist: message = ( u"There is no Peer Workflow associated with the given " u"submission UUID {}." ).format(scorer_submission_uuid) logger.exception(message) raise PeerAssessmentWorkflowError(message) except InvalidRubric: msg = u"Rubric definition was not valid" logger.exception(msg) raise PeerAssessmentRequestError(msg) except InvalidRubricSelection: msg = u"Invalid options selected in the rubric" logger.warning(msg, exc_info=True) raise PeerAssessmentRequestError(msg) except DatabaseError: error_message = ( u"An error occurred while retrieving the peer workflow item by scorer with ID: {}" ).format(scorer_id) logger.exception(error_message) raise PeerAssessmentInternalError(error_message)
def create_assessment( scorer_submission_uuid, scorer_id, options_selected, criterion_feedback, overall_feedback, rubric_dict, num_required_grades, scored_at=None): """Creates an assessment on the given submission. Assessments are created based on feedback associated with a particular rubric. Args: scorer_submission_uuid (str): The submission uuid for the Scorer's workflow. The submission being assessed can be determined via the peer workflow of the grading student. scorer_id (str): The user ID for the user giving this assessment. This is required to create an assessment on a submission. options_selected (dict): Dictionary mapping criterion names to the option names the user selected for that criterion. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. overall_feedback (unicode): Free-form text feedback on the submission overall. num_required_grades (int): The required number of assessments a submission requires before it is completed. If this number of assessments is reached, the grading_completed_at timestamp is set for the Workflow. Kwargs: scored_at (datetime): Optional argument to override the time in which the assessment took place. If not specified, scored_at is set to now. Returns: dict: the Assessment model, serialized as a dict. Raises: PeerAssessmentRequestError: Raised when the submission_id is invalid, or the assessment_dict does not contain the required values to create an assessment. PeerAssessmentInternalError: Raised when there is an internal error while creating a new assessment. Examples: >>> options_selected = {"clarity": "Very clear", "precision": "Somewhat precise"} >>> criterion_feedback = {"clarity": "I thought this essay was very clear."} >>> feedback = "Your submission was thrilling." >>> create_assessment("1", "Tim", options_selected, criterion_feedback, feedback, rubric_dict) """ # Ensure that this variables is declared so if an error occurs # we don't get an error when trying to log it! assessment_dict = None try: rubric = rubric_from_dict(rubric_dict) # Validate that the selected options matched the rubric # and raise an error if this is not the case try: option_ids = rubric.options_ids(options_selected) except InvalidOptionSelection: msg = "Selected options do not match the rubric" logger.warning(msg, exc_info=True) raise PeerAssessmentRequestError(msg) scorer_workflow = PeerWorkflow.objects.get(submission_uuid=scorer_submission_uuid) peer_workflow_item = scorer_workflow.get_latest_open_workflow_item() if peer_workflow_item is None: message = ( u"There are no open assessments associated with the scorer's " u"submission UUID {}." ).format(scorer_submission_uuid) logger.warning(message) raise PeerAssessmentWorkflowError(message) peer_submission_uuid = peer_workflow_item.author.submission_uuid peer_assessment = { "rubric": rubric.id, "scorer_id": scorer_id, "submission_uuid": peer_submission_uuid, "score_type": PEER_TYPE, "feedback": overall_feedback[0:Assessment.MAXSIZE], } if scored_at is not None: peer_assessment["scored_at"] = scored_at peer_serializer = AssessmentSerializer(data=peer_assessment) if not peer_serializer.is_valid(): msg = ( u"An error occurred while serializing " u"the peer assessment associated with " u"the scorer's submission UUID {}." ).format(scorer_submission_uuid) raise PeerAssessmentRequestError(msg) assessment = peer_serializer.save() # We do this to do a run around django-rest-framework serializer # validation, which would otherwise require two DB queries per # option to do validation. We already validated these options above. AssessmentPart.add_to_assessment(assessment, option_ids, criterion_feedback=criterion_feedback) # Close the active assessment scorer_workflow.close_active_assessment(peer_submission_uuid, assessment, num_required_grades) assessment_dict = full_assessment_dict(assessment) _log_assessment(assessment, scorer_workflow) return assessment_dict except DatabaseError: error_message = ( u"An error occurred while creating assessment {} by: {}" ).format(assessment_dict, scorer_id) logger.exception(error_message) raise PeerAssessmentInternalError(error_message) except PeerWorkflow.DoesNotExist: message = ( u"There is no Peer Workflow associated with the given " u"submission UUID {}." ).format(scorer_submission_uuid) logger.error(message) raise PeerAssessmentWorkflowError(message)
def validate_rubric(rubric_dict, current_rubric, is_released): """ Check that the rubric is semantically valid. Args: rubric_dict (dict): Serialized Rubric model representing the updated state of the rubric. current_rubric (dict): Serialized Rubric model representing the current state of the rubric. is_released (bool): True if and only if the problem has been released. Returns: tuple (is_valid, msg) where is_valid is a boolean indicating whether the assessment is semantically valid and msg describes any validation errors found. """ try: rubric_from_dict(rubric_dict) except InvalidRubric: return (False, u'This rubric definition is not valid.') # No duplicate criteria names duplicates = _duplicates([criterion['name'] for criterion in rubric_dict['criteria']]) if len(duplicates) > 0: msg = u"Criteria duplicate name(s): {duplicates}".format( duplicates=", ".join(duplicates) ) return (False, msg) # No duplicate option names within a criterion for criterion in rubric_dict['criteria']: duplicates = _duplicates([option['name'] for option in criterion['options']]) if len(duplicates) > 0: msg = u"Options in '{criterion}' have duplicate name(s): {duplicates}".format( criterion=criterion['name'], duplicates=", ".join(duplicates) ) return (False, msg) # After a problem is released, authors are allowed to change text, # but nothing that would change the point value of a rubric. if is_released: # Number of criteria must be the same if len(rubric_dict['criteria']) != len(current_rubric['criteria']): return (False, u'The number of criteria cannot be changed after a problem is released.') # Criteria names must be the same # We use criteria names as unique identifiers (unfortunately) # throughout the system. Changing them mid-flight can cause # the grade page, for example, to raise 500 errors. # When we implement non-XML authoring, we might be able to fix this # the right way by assigning unique identifiers for criteria; # but for now, this is the safest way to avoid breaking problems # post-release. current_criterion_names = set(criterion.get('name') for criterion in current_rubric['criteria']) new_criterion_names = set(criterion.get('name') for criterion in rubric_dict['criteria']) if current_criterion_names != new_criterion_names: return (False, u'Criteria names cannot be changed after a problem is released') # Number of options for each criterion must be the same for new_criterion, old_criterion in _match_by_order(rubric_dict['criteria'], current_rubric['criteria']): if len(new_criterion['options']) != len(old_criterion['options']): return (False, u'The number of options cannot be changed after a problem is released.') else: for new_option, old_option in _match_by_order(new_criterion['options'], old_criterion['options']): if new_option['points'] != old_option['points']: return (False, u'Point values cannot be changed after a problem is released.') return (True, u'')
def validate_rubric(rubric_dict, current_rubric, is_released, is_example_based, _): """ Check that the rubric is semantically valid. Args: rubric_dict (dict): Serialized Rubric model representing the updated state of the rubric. current_rubric (dict): Serialized Rubric model representing the current state of the rubric. is_released (bool): True if and only if the problem has been released. is_example_based (bool): True if and only if this is an example-based assessment. _ (function): The service function used to get the appropriate i18n text Returns: tuple (is_valid, msg) where is_valid is a boolean indicating whether the assessment is semantically valid and msg describes any validation errors found. """ try: rubric_from_dict(rubric_dict) except InvalidRubric: return False, _(u'This rubric definition is not valid.') for criterion in rubric_dict['criteria']: # No duplicate option names within a criterion duplicates = _duplicates([option['name'] for option in criterion['options']]) if len(duplicates) > 0: msg = _(u"Options in '{criterion}' have duplicate name(s): {duplicates}").format( criterion=criterion['name'], duplicates=", ".join(duplicates) ) return False, msg # Some criteria may have no options, just written feedback. # In this case, written feedback must be required (not optional or disabled). if len(criterion['options']) == 0 and criterion.get('feedback', 'disabled') != 'required': msg = _(u'Criteria with no options must require written feedback.') return False, msg # Example-based assessments impose the additional restriction # that the point values for options must be unique within # a particular rubric criterion. if is_example_based: duplicates = _duplicates([option['points'] for option in criterion['options']]) if len(duplicates) > 0: msg = _(u"Example-based assessments cannot have duplicate point values.") return False, msg # After a problem is released, authors are allowed to change text, # but nothing that would change the point value of a rubric. if is_released: # Number of prompts must be the same if len(rubric_dict['prompts']) != len(current_rubric['prompts']): return False, _(u'Prompts cannot be created or deleted after a problem is released.') # Number of criteria must be the same if len(rubric_dict['criteria']) != len(current_rubric['criteria']): return False, _(u'The number of criteria cannot be changed after a problem is released.') # Criteria names must be the same # We use criteria names as unique identifiers (unfortunately) # throughout the system. Changing them mid-flight can cause # the grade page, for example, to raise 500 errors. # When we implement non-XML authoring, we might be able to fix this # the right way by assigning unique identifiers for criteria; # but for now, this is the safest way to avoid breaking problems # post-release. current_criterion_names = set(criterion.get('name') for criterion in current_rubric['criteria']) new_criterion_names = set(criterion.get('name') for criterion in rubric_dict['criteria']) if current_criterion_names != new_criterion_names: return False, _(u'Criteria names cannot be changed after a problem is released') # Number of options for each criterion must be the same for new_criterion, old_criterion in _match_by_order(rubric_dict['criteria'], current_rubric['criteria']): if len(new_criterion['options']) != len(old_criterion['options']): return False, _(u'The number of options cannot be changed after a problem is released.') else: for new_option, old_option in _match_by_order(new_criterion['options'], old_criterion['options']): if new_option['points'] != old_option['points']: return False, _(u'Point values cannot be changed after a problem is released.') return True, u''
def create_assessment( submission_uuid, user_id, options_selected, criterion_feedback, overall_feedback, rubric_dict, scored_at=None ): """ Create a self-assessment for a submission. Args: submission_uuid (str): The unique identifier for the submission being assessed. user_id (str): The ID of the user creating the assessment. This must match the ID of the user who made the submission. options_selected (dict): Mapping of rubric criterion names to option values selected. criterion_feedback (dict): Dictionary mapping criterion names to the free-form text feedback the user gave for the criterion. Since criterion feedback is optional, some criteria may not appear in the dictionary. overall_feedback (unicode): Free-form text feedback on the submission overall. rubric_dict (dict): Serialized Rubric model. Keyword Arguments: scored_at (datetime): The timestamp of the assessment; defaults to the current time. Returns: dict: serialized Assessment model Raises: SelfAssessmentRequestError: Could not retrieve a submission that the user is allowed to score. """ # Check that there are not any assessments for this submission if Assessment.objects.filter(submission_uuid=submission_uuid, score_type=SELF_TYPE).exists(): msg = ( u"Cannot submit a self-assessment for the submission {uuid} " "because another self-assessment already exists for that submission." ).format(uuid=submission_uuid) raise SelfAssessmentRequestError(msg) # Check that the student is allowed to assess this submission try: submission = get_submission_and_student(submission_uuid) if submission['student_item']['student_id'] != user_id: msg = ( u"Cannot submit a self-assessment for the submission {uuid} " u"because it was created by another student " u"(submission student ID {student_id} does not match your " u"student id {other_id})" ).format( uuid=submission_uuid, student_id=submission['student_item']['student_id'], other_id=user_id ) raise SelfAssessmentRequestError(msg) except SubmissionNotFoundError: msg = ( "Could not submit a self-assessment because no submission " "exists with UUID {uuid}" ).format(uuid=submission_uuid) raise SelfAssessmentRequestError() try: # Get or create the rubric rubric = rubric_from_dict(rubric_dict) # Create the self assessment assessment = Assessment.create( rubric, user_id, submission_uuid, SELF_TYPE, scored_at=scored_at, feedback=overall_feedback ) # This will raise an `InvalidRubricSelection` if the selected options do not match the rubric. AssessmentPart.create_from_option_names(assessment, options_selected, feedback=criterion_feedback) _log_assessment(assessment, submission) except InvalidRubric as ex: msg = "Invalid rubric definition: " + str(ex) logger.warning(msg, exc_info=True) raise SelfAssessmentRequestError(msg) except InvalidRubricSelection as ex: msg = "Selected options do not match the rubric: " + str(ex) logger.warning(msg, exc_info=True) raise SelfAssessmentRequestError(msg) # Return the serialized assessment return full_assessment_dict(assessment)
def test_empty_options(self): rubric = rubric_from_dict(json_data('data/rubric/empty_options.json')) self.assertEqual(rubric.criteria.count(), 2)
def test_rubric_requires_positive_score(self): with self.assertRaises(InvalidRubric): rubric_from_dict(json_data('data/rubric/no_points.json'))
def create_assessment(submission_uuid, user_id, options_selected, rubric_dict, scored_at=None): """ Create a self-assessment for a submission. Args: submission_uuid (str): The unique identifier for the submission being assessed. user_id (str): The ID of the user creating the assessment. This must match the ID of the user who made the submission. options_selected (dict): Mapping of rubric criterion names to option values selected. rubric_dict (dict): Serialized Rubric model. Kwargs: scored_at (datetime): The timestamp of the assessment; defaults to the current time. Returns: dict: serialized Assessment model Raises: SelfAssessmentRequestError: Could not retrieve a submission that the user is allowed to score. """ # Check that there are not any assessments for this submission if Assessment.objects.filter(submission_uuid=submission_uuid, score_type=SELF_TYPE).exists(): raise SelfAssessmentRequestError(_("You've already completed your self assessment for this response.")) # Check that the student is allowed to assess this submission try: submission = get_submission_and_student(submission_uuid) if submission['student_item']['student_id'] != user_id: raise SelfAssessmentRequestError(_("You can only complete a self assessment on your own response.")) except SubmissionNotFoundError: raise SelfAssessmentRequestError(_("Could not retrieve the response.")) # Get or create the rubric try: rubric = rubric_from_dict(rubric_dict) option_ids = rubric.options_ids(options_selected) except InvalidRubric as ex: msg = _("Invalid rubric definition: {errors}").format(errors=ex.errors) raise SelfAssessmentRequestError(msg) except InvalidOptionSelection: msg = _("Selected options do not match the rubric") raise SelfAssessmentRequestError(msg) # Create the assessment # Since we have already retrieved the submission, we can assume that # the user who created the submission exists. self_assessment = { "rubric": rubric.id, "scorer_id": user_id, "submission_uuid": submission_uuid, "score_type": SELF_TYPE, "feedback": u"", } if scored_at is not None: self_assessment['scored_at'] = scored_at # Serialize the assessment serializer = AssessmentSerializer(data=self_assessment) if not serializer.is_valid(): msg = _("Could not create self assessment: {errors}").format(errors=serializer.errors) raise SelfAssessmentRequestError(msg) assessment = serializer.save() # We do this to do a run around django-rest-framework serializer # validation, which would otherwise require two DB queries per # option to do validation. We already validated these options above. AssessmentPart.add_to_assessment(assessment, option_ids) assessment_dict = full_assessment_dict(assessment) _log_assessment(assessment, submission) # Return the serialized assessment return assessment_dict
def _rubric_with_all_feedback_only_criteria(self): """Create a rubric with all feedback-only criteria.""" rubric_dict = copy.deepcopy(RUBRIC) for criterion in rubric_dict['criteria']: criterion['options'] = [] return rubric_from_dict(rubric_dict)
def setUp(self): super(AIClassifierSetTest, self).setUp() rubric = rubric_from_dict(RUBRIC) self.classifier_set = AIClassifierSet.create_classifier_set( CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID)