Esempio n. 1
0
    def test_train_robot_task(self):


        # Take at least (min number for training) images.  Preprocess,
        # feature extract, and add human annotations to the features.
        source_images = Image.objects.filter(source__pk=self.source_id)

        for img in source_images:
            preprocess_image(img.id)
            make_features(img.id)
            self.add_human_annotations(img.id)
            add_labels_to_features(img.id)

        # From now on, this variable may be out of date.
        # del it to avoid mistakes.
        del source_images


        # Create a robot.
        result = train_robot(self.source_id)
        self.assertTrue(result == 1)


        source_images = Image.objects.filter(source__pk=self.source_id)

        for img in source_images:
            self.assertTrue(img.status.usedInCurrentModel)
Esempio n. 2
0
    def test_classify_task(self):


        # Take at least (min number for training) images.
        # Preprocess, feature extract, and add human annotations to
        # the features.
        for img in Image.objects.filter(source__pk=self.source_id):
            preprocess_image(img.id)
            make_features(img.id)
            self.add_human_annotations(img.id)
            add_labels_to_features(img.id)


        # Create a robot.
        result = train_robot(self.source_id)
        self.assertTrue(result == 1)


        # Upload a new image.
        img_id = self.upload_image('006_2012-06-28_color-grid-006.png')[0]

        # Preprocess and feature extract.
        preprocess_image(img_id)
        make_features(img_id)
        # Sanity check: not classified yet
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, False)


        # Run task, attempt 1.
        result = classify_image(img_id)

        # Check that the task didn't encounter an exception
        self.assertTrue(result == 1)
        # Should have classified the image
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
        # Check that the image's actual Annotation objects are there.
        # Should have 1 annotation per point.
        self.assertEqual(
            Annotation.objects.filter(image__pk=img_id).count(),
            Point.objects.filter(image__pk=img_id).count(),
        )

        # TODO: Check that the history entries are there?


        # Run task, attempt 2.
        result = classify_image(img_id)

        # Check that the task didn't encounter an exception
        self.assertTrue(result == 1)
        # Should have exited without re-doing the classification
        # TODO: Check file ctime/mtime to check that it wasn't redone?
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
Esempio n. 3
0
    def helper_classify_does_not_overwrite_manual_annotations(self, annotator_user):
        """
        Helper function for the tests that follow.
        """


        # Take at least (min number for training) images.
        # Preprocess, feature extract, and add human annotations to
        # the features.
        for img in Image.objects.filter(source__pk=self.source_id):
            preprocess_image(img.id)
            make_features(img.id)
            self.add_human_annotations(img.id)
            add_labels_to_features(img.id)

        # Create a robot.
        result = train_robot(self.source_id)
        self.assertTrue(result == 1)

        # Upload a new image.
        img_id = self.upload_image('006_2012-06-28_color-grid-006.png')[0]

        # Preprocess and feature extract.
        preprocess_image(img_id)
        make_features(img_id)

        # Add annotations.
        source = Source.objects.get(pk=self.source_id)
        img = Image.objects.get(pk=img_id)
        points = Point.objects.filter(image=img)
        labels = source.labelset.labels.all()

        # For odd-numbered points, make an annotation by picking a
        # label randomly from the source's labelset.
        # Leave the even-numbered points alone.
        # (Assumption: the test source has at least 2 points per image)
        for pt in points:

            if pt.point_number % 2 == 0:
                continue

            label = random.choice(labels)
            anno = Annotation(
                point=pt,
                image=img,
                source=source,
                user=annotator_user,
                label=label,
            )
            anno.save()

        img.status.save()


        # Get those annotations (again, only odd-numbered points).
        num_points = Point.objects.filter(image__pk=img_id).count()
        manual_annotations = dict()

        for point_num in range(1, num_points+1, 2):

            label_id = Annotation.objects.get(image__pk=img_id, point__point_number=point_num).label.id
            manual_annotations[point_num] = label_id


        # Try to Classify.
        result = classify_image(img_id)

        # Shouldn't throw exception.
        self.assertTrue(result == 1)
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)


        # Check the Annotations.
        for point_num in range(1, num_points+1):

            anno = Annotation.objects.get(image__pk=img_id, point__point_number=point_num)
            label_id = anno.label.id

            if point_num % 2 == 0:
                # Even; should be robot
                self.assertEqual(anno.user.id, get_robot_user().id)
            else:
                # Odd; should be manual (and same as before)
                self.assertEqual(label_id, manual_annotations[point_num])
                self.assertEqual(anno.user.id, annotator_user.id)

            if settings.UNIT_TEST_VERBOSITY >= 1:
                print "Point {num} | {username} | {label_id}".format(
                    num=point_num,
                    username=anno.user.username,
                    label_id=label_id,
                )
Esempio n. 4
0
    def test_reclassify(self):
        """
        Test that we can classify an image twice with two different robots,
        and that the robot annotations are actually updated on the second
        classification.

        Note: this test is able to catch errors ONLY if the two different
        robots assign different labels to the image points.
        """


        # Take at least (min number for training) images.
        # Preprocess, feature extract, and add human annotations to
        # the features.
        for img in Image.objects.filter(source__pk=self.source_id):
            preprocess_image(img.id)
            make_features(img.id)
            self.add_human_annotations(img.id)
            add_labels_to_features(img.id)

        # Create a robot.
        result = train_robot(self.source_id)
        self.assertTrue(result == 1)


        # Upload a new image.
        img_id = self.upload_image('006_2012-06-28_color-grid-006.png')[0]

        # Preprocess and feature extract.
        preprocess_image(img_id)
        make_features(img_id)
        # Sanity check: not classified yet
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, False)

        # Classify, 1st time.
        result = classify_image(img_id)

        self.assertTrue(result == 1)
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)

        num_points = Point.objects.filter(image__pk=img_id).count()
        self.assertEqual(
            Annotation.objects.filter(image__pk=img_id).count(),
            num_points,
        )

        # Verify that the points match those in the label file output by the
        # classification task.
        label_filename = os.path.join(
            settings.PROCESSING_ROOT,
            'images',
            'classify',
            '{img_id}_{process_date}.txt'.format(
                img_id=str(img_id),
                process_date=Image.objects.get(pk=img_id).get_process_date_short_str(),
            )
        )

        label_score_dict = read_label_score_file(img_id)

        for point_num in range(num_points):

            label_id = Annotation.objects.get(image__pk=img_id, point__point_number=point_num+1).label.id # from database
            scores = np.asarray([s['score'] for s in label_score_dict[point_num]])
            label_id_file = int(label_score_dict[point_num][scores.argmax()]['label']) # from file
            self.assertEqual(label_id, label_id_file)


        # Add another training image.
        extra_training_img_id = self.upload_image(
            os.path.join('1key', '001_2011-05-28.png')
        )[0]
        preprocess_image(extra_training_img_id)
        make_features(extra_training_img_id)
        self.add_human_annotations(extra_training_img_id)
        add_labels_to_features(extra_training_img_id)

        # Create another robot.
        result = train_robot(self.source_id)
        self.assertTrue(result == 1)

        # Classify, 2nd time.
        result = classify_image(img_id)

        self.assertTrue(result == 1)
        self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)

        # We'd better still have one annotation per point
        # (not two annotations per point).
        self.assertEqual(
            Annotation.objects.filter(image__pk=img_id).count(),
            Point.objects.filter(image__pk=img_id).count(),
        )

        # Verify, again, that the points match those in the label file output
        # by the classification task.
        label_score_dict = read_label_score_file(img_id)

        for point_num in range(num_points):

            label_id = Annotation.objects.get(image__pk=img_id, point__point_number=point_num+1).label.id # from database
            scores = np.asarray([s['score'] for s in label_score_dict[point_num]])
            label_id_file = int(label_score_dict[point_num][scores.argmax()]['label']) # from file
            self.assertEqual(label_id, label_id_file)