def test_train_robot_task(self):
# Take at least (min number for training) images. Preprocess,
# feature extract, and add human annotations to the features.
source_images = Image.objects.filter(source__pk=self.source_id)
for img in source_images:
preprocess_image(img.id)
make_features(img.id)
self.add_human_annotations(img.id)
add_labels_to_features(img.id)
# From now on, this variable may be out of date.
# del it to avoid mistakes.
del source_images
# Create a robot.
result = train_robot(self.source_id)
self.assertTrue(result == 1)
source_images = Image.objects.filter(source__pk=self.source_id)
for img in source_images:
self.assertTrue(img.status.usedInCurrentModel)
def test_classify_task(self):
# Take at least (min number for training) images.
# Preprocess, feature extract, and add human annotations to
# the features.
for img in Image.objects.filter(source__pk=self.source_id):
preprocess_image(img.id)
make_features(img.id)
self.add_human_annotations(img.id)
add_labels_to_features(img.id)
# Create a robot.
result = train_robot(self.source_id)
self.assertTrue(result == 1)
# Upload a new image.
img_id = self.upload_image('006_2012-06-28_color-grid-006.png')[0]
# Preprocess and feature extract.
preprocess_image(img_id)
make_features(img_id)
# Sanity check: not classified yet
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, False)
# Run task, attempt 1.
result = classify_image(img_id)
# Check that the task didn't encounter an exception
self.assertTrue(result == 1)
# Should have classified the image
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
# Check that the image's actual Annotation objects are there.
# Should have 1 annotation per point.
self.assertEqual(
Annotation.objects.filter(image__pk=img_id).count(),
Point.objects.filter(image__pk=img_id).count(),
)
# TODO: Check that the history entries are there?
# Run task, attempt 2.
result = classify_image(img_id)
# Check that the task didn't encounter an exception
self.assertTrue(result == 1)
# Should have exited without re-doing the classification
# TODO: Check file ctime/mtime to check that it wasn't redone?
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
def helper_classify_does_not_overwrite_manual_annotations(self, annotator_user):
"""
Helper function for the tests that follow.
"""
# Take at least (min number for training) images.
# Preprocess, feature extract, and add human annotations to
# the features.
for img in Image.objects.filter(source__pk=self.source_id):
preprocess_image(img.id)
make_features(img.id)
self.add_human_annotations(img.id)
add_labels_to_features(img.id)
# Create a robot.
result = train_robot(self.source_id)
self.assertTrue(result == 1)
# Upload a new image.
img_id = self.upload_image('006_2012-06-28_color-grid-006.png')[0]
# Preprocess and feature extract.
preprocess_image(img_id)
make_features(img_id)
# Add annotations.
source = Source.objects.get(pk=self.source_id)
img = Image.objects.get(pk=img_id)
points = Point.objects.filter(image=img)
labels = source.labelset.labels.all()
# For odd-numbered points, make an annotation by picking a
# label randomly from the source's labelset.
# Leave the even-numbered points alone.
# (Assumption: the test source has at least 2 points per image)
for pt in points:
if pt.point_number % 2 == 0:
continue
label = random.choice(labels)
anno = Annotation(
point=pt,
image=img,
source=source,
user=annotator_user,
label=label,
)
anno.save()
img.status.save()
# Get those annotations (again, only odd-numbered points).
num_points = Point.objects.filter(image__pk=img_id).count()
manual_annotations = dict()
for point_num in range(1, num_points+1, 2):
label_id = Annotation.objects.get(image__pk=img_id, point__point_number=point_num).label.id
manual_annotations[point_num] = label_id
# Try to Classify.
result = classify_image(img_id)
# Shouldn't throw exception.
self.assertTrue(result == 1)
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
# Check the Annotations.
for point_num in range(1, num_points+1):
anno = Annotation.objects.get(image__pk=img_id, point__point_number=point_num)
label_id = anno.label.id
if point_num % 2 == 0:
# Even; should be robot
self.assertEqual(anno.user.id, get_robot_user().id)
else:
# Odd; should be manual (and same as before)
self.assertEqual(label_id, manual_annotations[point_num])
self.assertEqual(anno.user.id, annotator_user.id)
if settings.UNIT_TEST_VERBOSITY >= 1:
print "Point {num} | {username} | {label_id}".format(
num=point_num,
username=anno.user.username,
label_id=label_id,
)
def test_reclassify(self):
"""
Test that we can classify an image twice with two different robots,
and that the robot annotations are actually updated on the second
classification.
Note: this test is able to catch errors ONLY if the two different
robots assign different labels to the image points.
"""
# Take at least (min number for training) images.
# Preprocess, feature extract, and add human annotations to
# the features.
for img in Image.objects.filter(source__pk=self.source_id):
preprocess_image(img.id)
make_features(img.id)
self.add_human_annotations(img.id)
add_labels_to_features(img.id)
# Create a robot.
result = train_robot(self.source_id)
self.assertTrue(result == 1)
# Upload a new image.
img_id = self.upload_image('006_2012-06-28_color-grid-006.png')[0]
# Preprocess and feature extract.
preprocess_image(img_id)
make_features(img_id)
# Sanity check: not classified yet
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, False)
# Classify, 1st time.
result = classify_image(img_id)
self.assertTrue(result == 1)
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
num_points = Point.objects.filter(image__pk=img_id).count()
self.assertEqual(
Annotation.objects.filter(image__pk=img_id).count(),
num_points,
)
# Verify that the points match those in the label file output by the
# classification task.
label_filename = os.path.join(
settings.PROCESSING_ROOT,
'images',
'classify',
'{img_id}_{process_date}.txt'.format(
img_id=str(img_id),
process_date=Image.objects.get(pk=img_id).get_process_date_short_str(),
)
)
label_score_dict = read_label_score_file(img_id)
for point_num in range(num_points):
label_id = Annotation.objects.get(image__pk=img_id, point__point_number=point_num+1).label.id # from database
scores = np.asarray([s['score'] for s in label_score_dict[point_num]])
label_id_file = int(label_score_dict[point_num][scores.argmax()]['label']) # from file
self.assertEqual(label_id, label_id_file)
# Add another training image.
extra_training_img_id = self.upload_image(
os.path.join('1key', '001_2011-05-28.png')
)[0]
preprocess_image(extra_training_img_id)
make_features(extra_training_img_id)
self.add_human_annotations(extra_training_img_id)
add_labels_to_features(extra_training_img_id)
# Create another robot.
result = train_robot(self.source_id)
self.assertTrue(result == 1)
# Classify, 2nd time.
result = classify_image(img_id)
self.assertTrue(result == 1)
self.assertEqual(Image.objects.get(pk=img_id).status.annotatedByRobot, True)
# We'd better still have one annotation per point
# (not two annotations per point).
self.assertEqual(
Annotation.objects.filter(image__pk=img_id).count(),
Point.objects.filter(image__pk=img_id).count(),
)
# Verify, again, that the points match those in the label file output
# by the classification task.
label_score_dict = read_label_score_file(img_id)
for point_num in range(num_points):
label_id = Annotation.objects.get(image__pk=img_id, point__point_number=point_num+1).label.id # from database
scores = np.asarray([s['score'] for s in label_score_dict[point_num]])
label_id_file = int(label_score_dict[point_num][scores.argmax()]['label']) # from file
self.assertEqual(label_id, label_id_file)