예제 #1
0
파일: test_ai.py 프로젝트: zimka/edx-ora2
def train_classifiers(rubric_dict, classifier_score_overrides):
    """
    Simple utility function to train classifiers.

    Args:
        rubric_dict (dict): The rubric to train the classifiers on.
        classifier_score_overrides (dict): A dictionary of classifier overrides
            to set the scores for the given submission.

    """
    rubric = rubric_from_dict(rubric_dict)
    AIClassifierSet.create_classifier_set(classifier_score_overrides, rubric,
                                          ALGORITHM_ID, COURSE_ID, ITEM_ID)
예제 #2
0
    def test_assign_most_recent_classifier_set(self):
        # No classifier sets are available
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertFalse(found)
        self.assertIs(self.workflow.classifier_set, None)

        # Same rubric (exact), but different course id
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
            "different course!", self.ITEM_ID)
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric (exact) but different item id
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
            self.COURSE_ID, "different item!")
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric (exact), but different algorithm id
        # Shouldn't change, since the algorithm ID doesn't match
        AIClassifierSet.create_classifier_set(self.CLASSIFIERS_DICT,
                                              self.rubric,
                                              "different algorithm!",
                                              self.COURSE_ID, self.ITEM_ID)
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric *structure*, but in a different item
        # Shouldn't change, since the rubric isn't an exact match.
        AIClassifierSet.create_classifier_set(self.CLASSIFIERS_DICT,
                                              self.similar_rubric,
                                              self.ALGORITHM_ID,
                                              self.COURSE_ID,
                                              "different item!")
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric *structure* AND in the same course/item
        # This should replace our current classifier set
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.similar_rubric, self.ALGORITHM_ID,
            self.COURSE_ID, self.ITEM_ID)
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric and same course/item
        # This is the ideal, so we should always prefer it
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
            self.COURSE_ID, self.ITEM_ID)
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)
예제 #3
0
def train_classifiers(rubric_dict, classifier_score_overrides):
    """
    Simple utility function to train classifiers.

    Args:
        rubric_dict (dict): The rubric to train the classifiers on.
        classifier_score_overrides (dict): A dictionary of classifier overrides
            to set the scores for the given submission.

    """
    rubric = rubric_from_dict(rubric_dict)
    AIClassifierSet.create_classifier_set(
        classifier_score_overrides, rubric, ALGORITHM_ID, COURSE_ID, ITEM_ID
    )
예제 #4
0
    def handle(self, *args, **options):
        """
        Execute the command.

        Args:
            course_id (unicode): The ID of the course to create submissions/workflows in.
            item_id (unicode): The ID of the problem in the course.
            num_submissions (int): The number of submissions/workflows to create.
            algorithm_id (unicode): The ID of the ML algorithm to use ("fake" or "ease")

        Raises:
            CommandError

        """
        if len(args) < 4:
            raise CommandError(u"Usage: simulate_ai_grading_error {}".format(
                self.args))

        # Parse arguments
        course_id = args[0].decode('utf-8')
        item_id = args[1].decode('utf-8')
        num_submissions = int(args[2])
        algorithm_id = args[3].decode('utf-8')

        # Create the rubric model
        rubric = rubric_from_dict(self.RUBRIC)

        # Train classifiers
        print u"Training classifiers using {algorithm_id}...".format(
            algorithm_id=algorithm_id)
        algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
        classifier_data = {
            criterion_name: algorithm.train_classifier(example)
            for criterion_name, example in self.EXAMPLES.iteritems()
        }
        print u"Successfully trained classifiers."

        # Create the classifier set
        classifier_set = AIClassifierSet.create_classifier_set(
            classifier_data, rubric, algorithm_id, course_id, item_id)
        print u"Successfully created classifier set with id {}".format(
            classifier_set.pk)

        # Create submissions and grading workflows
        for num in range(num_submissions):
            student_item = {
                'course_id': course_id,
                'item_id': item_id,
                'item_type': 'openassessment',
                'student_id': "{base}_{num}".format(base=self.STUDENT_ID,
                                                    num=num)
            }
            submission = sub_api.create_submission(student_item, self.ANSWER)
            workflow = AIGradingWorkflow.start_workflow(
                submission['uuid'], self.RUBRIC, algorithm_id)
            workflow.classifier_set = classifier_set
            workflow.save()
            print u"{num}: Created incomplete grading workflow with UUID {uuid}".format(
                num=num, uuid=workflow.uuid)
예제 #5
0
 def _create_classifier(self):
     """
     Create and return an AIClassifier.
     """
     rubric = rubric_from_dict(RUBRIC)
     classifier_set = AIClassifierSet.create_classifier_set(
         CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID)
     return AIClassifier.objects.filter(classifier_set=classifier_set)[0]
    def handle(self, *args, **options):
        """
        Execute the command.

        Args:
            course_id (unicode): The ID of the course to create submissions/workflows in.
            item_id (unicode): The ID of the problem in the course.
            num_submissions (int): The number of submissions/workflows to create.
            algorithm_id (unicode): The ID of the ML algorithm to use ("fake" or "ease")

        Raises:
            CommandError

        """
        if len(args) < 4:
            raise CommandError(u"Usage: simulate_ai_grading_error {}".format(self.args))

        # Parse arguments
        course_id = args[0].decode('utf-8')
        item_id = args[1].decode('utf-8')
        num_submissions = int(args[2])
        algorithm_id = args[3].decode('utf-8')

        # Create the rubric model
        rubric = rubric_from_dict(self.RUBRIC)

        # Train classifiers
        print u"Training classifiers using {algorithm_id}...".format(algorithm_id=algorithm_id)
        algorithm = AIAlgorithm.algorithm_for_id(algorithm_id)
        classifier_data = {
            criterion_name: algorithm.train_classifier(example)
            for criterion_name, example in self.EXAMPLES.iteritems()
        }
        print u"Successfully trained classifiers."

        # Create the classifier set
        classifier_set = AIClassifierSet.create_classifier_set(
            classifier_data, rubric, algorithm_id, course_id, item_id
        )
        print u"Successfully created classifier set with id {}".format(classifier_set.pk)

        # Create submissions and grading workflows
        for num in range(num_submissions):
            student_item = {
                'course_id': course_id,
                'item_id': item_id,
                'item_type': 'openassessment',
                'student_id': "{base}_{num}".format(base=self.STUDENT_ID, num=num)
            }
            submission = sub_api.create_submission(student_item, self.ANSWER)
            workflow = AIGradingWorkflow.start_workflow(
                submission['uuid'], self.RUBRIC, algorithm_id
            )
            workflow.classifier_set = classifier_set
            workflow.save()
            print u"{num}: Created incomplete grading workflow with UUID {uuid}".format(
                num=num, uuid=workflow.uuid
            )
예제 #7
0
 def _create_classifier(self):
     """
     Create and return an AIClassifier.
     """
     rubric = rubric_from_dict(RUBRIC)
     classifier_set = AIClassifierSet.create_classifier_set(
         CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID
     )
     return AIClassifier.objects.filter(classifier_set=classifier_set)[0]
예제 #8
0
    def setUp(self):
        """
        Create a submission and grading workflow.
        """
        # Create a submission
        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
        self.submission_uuid = submission['uuid']

        # Create a workflow for the submission
        workflow = AIGradingWorkflow.start_workflow(self.submission_uuid, RUBRIC, ALGORITHM_ID)
        self.workflow_uuid = workflow.uuid

        # Associate the workflow with classifiers
        rubric = rubric_from_dict(RUBRIC)
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS, rubric, ALGORITHM_ID, STUDENT_ITEM.get('course_id'), STUDENT_ITEM.get('item_id')
        )
        workflow.classifier_set = classifier_set
        workflow.save()
예제 #9
0
    def setUp(self):
        """
        Create a submission and grading workflow.
        """
        # Create a submission
        submission = sub_api.create_submission(STUDENT_ITEM, ANSWER)
        self.submission_uuid = submission['uuid']

        # Create a workflow for the submission
        workflow = AIGradingWorkflow.start_workflow(self.submission_uuid,
                                                    RUBRIC, ALGORITHM_ID)
        self.workflow_uuid = workflow.uuid

        # Associate the workflow with classifiers
        rubric = rubric_from_dict(RUBRIC)
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS, rubric, ALGORITHM_ID,
            STUDENT_ITEM.get('course_id'), STUDENT_ITEM.get('item_id'))
        workflow.classifier_set = classifier_set
        workflow.save()
예제 #10
0
 def setUp(self):
     super(AIClassifierSetTest, self).setUp()
     rubric = rubric_from_dict(RUBRIC)
     self.classifier_set = AIClassifierSet.create_classifier_set(
         CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID)
예제 #11
0
 def setUp(self):
     super(AIClassifierSetTest, self).setUp()
     rubric = rubric_from_dict(RUBRIC)
     self.classifier_set = AIClassifierSet.create_classifier_set(
         CLASSIFIERS_DICT, rubric, "test_algorithm", COURSE_ID, ITEM_ID
     )
예제 #12
0
    def test_assign_most_recent_classifier_set(self):
        # No classifier sets are available
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertFalse(found)
        self.assertIs(self.workflow.classifier_set, None)

        # Same rubric (exact), but different course id
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
            "different course!", self.ITEM_ID
        )
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric (exact) but different item id
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
            self.COURSE_ID, "different item!"
        )
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric (exact), but different algorithm id
        # Shouldn't change, since the algorithm ID doesn't match
        AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, "different algorithm!",
            self.COURSE_ID, self.ITEM_ID
        )
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric *structure*, but in a different item
        # Shouldn't change, since the rubric isn't an exact match.
        AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.similar_rubric, self.ALGORITHM_ID,
            self.COURSE_ID, "different item!"
        )
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric *structure* AND in the same course/item
        # This should replace our current classifier set
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.similar_rubric, self.ALGORITHM_ID,
            self.COURSE_ID, self.ITEM_ID
        )
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)

        # Same rubric and same course/item
        # This is the ideal, so we should always prefer it
        classifier_set = AIClassifierSet.create_classifier_set(
            self.CLASSIFIERS_DICT, self.rubric, self.ALGORITHM_ID,
            self.COURSE_ID, self.ITEM_ID
        )
        found = self.workflow.assign_most_recent_classifier_set()
        self.assertTrue(found)
        self.assertEqual(classifier_set.pk, self.workflow.classifier_set.pk)