def test_classify_does_not_overwrite_imported_annotations(self):
"""
Tests that when an image is partially annotated by imported data,
automatic classification doesn't overwrite any of the imported
points.
"""
self.helper_classify_does_not_overwrite_manual_annotations(
get_imported_user()
)
def import_archived_annotations(source_id, anndict, with_labels = True):
source = Source.objects.get(pk = source_id) # let's fetch the relevant source.
imported_user = get_imported_user() # the imported user.
images = source.get_all_images().filter(metadata__name__in = list(anndict.keys())) # grab all image that have names in the .csv file.
for image in images:
# Start by remove annotations and points for this image
for ann in Annotation.objects.filter(image=image):
ann.delete()
for point in Point.objects.filter(image=image):
point.delete()
# Next, set image metadata to IMPORTED.
image.point_generation_method = PointGen.args_to_db_format(
point_generation_type=PointGen.Types.IMPORTED,
imported_number_of_points=len(anndict[image.metadata.name])
)
image.save()
image.status.hasRandomPoints = True
image.status.annotatedByHuman = with_labels
image.status.save()
image.after_annotation_area_change() # set the backend status correctly.
# Iterate over this image's annotations and save them.
for (point_num, (row, col, code)) in enumerate(anndict[image.metadata.name]):
# Save the Point in the database.
point = Point(row=row, column=col, point_number=point_num + 1, image=image)
point.save()
# and save the Annotation.
if with_labels:
label = Label.objects.filter(code=code)[0]
annotation = Annotation(user=imported_user, point=point, image=image, label=label, source=source)
annotation.save()
def image_upload_process(imageFiles, imageOptionsForm, annotationOptionsForm, source, currentUser, annoFile):
"""
Helper method for the image upload view and the image+annotation
import view.
"""
uploadedImages = []
duplicates = 0
imagesUploaded = 0
annotationsImported = 0
importedUser = get_imported_user()
dupeOption = imageOptionsForm.cleaned_data['skip_or_replace_duplicates']
annotationData = None
if annoFile:
try:
annotationData = annotations_file_to_python(annoFile, source)
except FileContentError as errorDetail:
return dict(error=True,
message='Error reading labels file %s. %s' % (annoFile.name, errorDetail),
)
for imageFile in imageFiles:
filename = imageFile.name
metadataDict = None
metadata = Metadata(height_in_cm=source.image_height_in_cm)
if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames':
try:
metadataDict = filename_to_metadata(filename, source)
# Filename parse error.
# TODO: check for validity of the file type and contents, too.
except (ValueError, StopIteration):
return dict(error=True,
message='Upload failed - Error when parsing the filename %s for metadata.' % filename,
)
# Detect duplicate images and handle them
dupe = find_dupe_image(source, **metadataDict)
if dupe:
duplicates += 1
if dupeOption == 'skip':
# Skip uploading this file.
continue
elif dupeOption == 'replace':
# Proceed uploading this file, and delete the dupe.
dupe.delete()
# Set the metadata
valueDict = get_location_value_objs(source, metadataDict['values'], createNewValues=True)
photoDate = datetime.date(year = int(metadataDict['year']),
month = int(metadataDict['month']),
day = int(metadataDict['day']))
metadata.name = metadataDict['name']
metadata.photo_date = photoDate
for key, value in valueDict.iteritems():
setattr(metadata, key, value)
else:
metadata.name = filename
# Image + annotation import form
# Assumes we got the images' metadata (from filenames or otherwise)
if annotationData:
pointsOnlyOption = annotationOptionsForm.cleaned_data['points_only']
# Use the location values and the year to build a string identifier for the image, such as:
# Shore1;Reef5;...;2008
imageIdentifier = get_image_identifier(metadataDict['values'], metadataDict['year'])
# Use the identifier as the index into the annotation file's data.
if not annotationData.has_key(imageIdentifier):
return dict(error=True,
message='%s seems to have no annotations for the image file %s, which has the following keys:\n%s' % (
annoFile.name, imageFile.name, imageIdentifier.replace(';',' '))
)
imageAnnotations = annotationData[imageIdentifier]
status = ImageStatus()
status.save()
metadata.annotation_area = AnnotationAreaUtils.IMPORTED_STR
metadata.save()
img = Image(original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=PointGen.args_to_db_format(
point_generation_type=PointGen.Types.IMPORTED,
imported_number_of_points=len(imageAnnotations)
),
metadata=metadata,
source=source,
status=status,
)
img.save()
# Iterate over this image's annotations and save them.
pointNum = 1
for anno in imageAnnotations:
# Save the Point in the database.
point = Point(row=anno['row'], column=anno['col'], point_number=pointNum, image=img)
point.save()
if not pointsOnlyOption:
label = Label.objects.filter(code=anno['label'])[0]
# Save the Annotation in the database, marking the annotations as imported.
annotation = Annotation(user=importedUser,
point=point, image=img, label=label, source=source)
annotation.save()
annotationsImported += 1
pointNum += 1
img.status.hasRandomPoints = True
if not pointsOnlyOption:
img.status.annotatedByHuman = True
img.status.save()
# Image upload form, no annotations
else:
status = ImageStatus()
status.save()
metadata.annotation_area = source.image_annotation_area
metadata.save()
# Save the image into the DB
img = Image(original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=source.default_point_generation_method,
metadata=metadata,
source=source,
status=status,
)
img.save()
# Generate and save points
generate_points(img)
# Up to 5 uploaded images will be shown
# upon successful upload.
# Prepend to list, so most recent image comes first
uploadedImages.insert(0, img)
if len(uploadedImages) > 5:
uploadedImages = uploadedImages[:5]
imagesUploaded += 1
# Construct success message.
success_message = image_upload_success_message(
num_images_uploaded=imagesUploaded,
num_dupes=duplicates,
dupe_option=dupeOption,
num_annotations=annotationsImported,
)
return dict(error=False,
uploadedImages=uploadedImages,
message=success_message,
)
def image_upload_process(imageFile, imageOptionsForm,
annotation_dict_id,
csv_dict_id, metadata_import_form_class,
annotation_options_form,
source, currentUser):
is_uploading_points_or_annotations = annotation_options_form.cleaned_data['is_uploading_points_or_annotations']
filename = imageFile.name
metadata_dict = None
metadata_obj = Metadata(height_in_cm=source.image_height_in_cm)
if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames':
filename_check_result = check_image_filename(filename, source)
filename_status = filename_check_result['status']
if filename_status == 'error':
# This case should never happen if the pre-upload
# status checking is doing its job, but just in case...
return dict(
status=filename_status,
message=u"{m}".format(m=filename_check_result['message']),
link=None,
title=None,
)
# Set the metadata
metadata_dict = filename_check_result['metadata_dict']
value_dict = get_location_value_objs(source, metadata_dict['values'], createNewValues=True)
photo_date = datetime.date(
year = int(metadata_dict['year']),
month = int(metadata_dict['month']),
day = int(metadata_dict['day'])
)
metadata_obj.name = metadata_dict['name']
metadata_obj.photo_date = photo_date
for key, value in value_dict.iteritems():
setattr(metadata_obj, key, value)
elif imageOptionsForm.cleaned_data['specify_metadata'] == 'csv':
if not csv_dict_id:
return dict(
status='error',
message=u"{m}".format(m="CSV file was not uploaded."),
link=None,
title=None,
)
csv_dict_filename = os.path.join(
settings.SHELVED_ANNOTATIONS_DIR,
'csv_source{source_id}_{dict_id}.db'.format(
source_id=source.id,
dict_id=csv_dict_id,
),
)
# Corner case: the specified shelved annotation file doesn't exist.
# Perhaps the file was created a while ago and has been pruned since,
# or perhaps there is a bug.
if not os.path.isfile(csv_dict_filename):
return dict(
status='error',
message="CSV file could not be found - if you provided the .csv file a while ago, maybe it just timed out. Please retry the upload.",
link=None,
title=None,
)
csv_dict = shelve.open(csv_dict_filename)
#index into the csv_dict with the filename. the str() is to handle
#the case where the filename is a unicode object instead of a str;
#unicode objects can't index into dicts.
filename_str = str(filename)
if filename_str in csv_dict:
# There is CSV metadata for this file.
metadata_dict = csv_dict[str(filename)]
csv_dict.close()
# The reason this uses metadata_import_form_class instead of
# importing MetadataImportForm is that I'm too lazy to deal with the
# circular-import implications of the latter solution right now.
# -Stephen
metadata_import_form = metadata_import_form_class(
source.id, True, metadata_dict,
)
if not metadata_import_form.is_valid():
return dict(
status='error',
message="Unknown error with the CSV metadata.",
link=None,
title=None,
)
fields = ['photo_date', 'value1', 'value2', 'value3', 'value4',
'value5', 'height_in_cm', 'latitude', 'longitude',
'depth', 'camera', 'photographer', 'water_quality',
'strobes', 'framing', 'balance']
for field in fields:
if not field in metadata_import_form.fields:
# A location value field that's not in this form
continue
value = metadata_import_form.cleaned_data[field]
# Check for a non-empty value; don't want empty values to
# override default values that we've already set on the
# metadata_obj
if value:
setattr(metadata_obj, field, value)
else:
# No CSV metadata for this file.
csv_dict.close()
metadata_obj.name = filename
else:
# Not specifying any metadata at upload time.
metadata_obj.name = filename
image_annotations = None
has_points_or_annotations = False
if is_uploading_points_or_annotations:
# Corner case: somehow, we're uploading with points+annotations and without
# a checked annotation file specified. This probably indicates a bug.
if not annotation_dict_id:
return dict(
status='error',
message=u"{m}".format(m=str_consts.UPLOAD_ANNOTATIONS_ON_AND_NO_ANNOTATION_DICT_ERROR_STR),
link=None,
title=None,
)
annotation_dict_filename = os.path.join(
settings.SHELVED_ANNOTATIONS_DIR,
'source{source_id}_{dict_id}'.format(
source_id=source.id,
dict_id=annotation_dict_id,
),
)
# Corner case: the specified shelved annotation file doesn't exist.
# Perhaps the file was created a while ago and has been pruned since,
# or perhaps there is a bug.
if not os.path.isfile(annotation_dict_filename):
return dict(
status='error',
message="Annotations could not be found - if you provided the .txt file a while ago, maybe it just timed out. Please retry the upload.",
link=None,
title=None,
)
# Use the location values and the year to build a string identifier for the image, such as:
# Shore1;Reef5;...;2008
# Convert to a string (instead of a unicode string) for the shelve key lookup.
image_identifier = str(get_image_identifier(metadata_dict['values'], metadata_dict['year']))
annotation_dict = shelve.open(annotation_dict_filename)
if annotation_dict.has_key(image_identifier):
image_annotations = annotation_dict[image_identifier]
has_points_or_annotations = True
annotation_dict.close()
if has_points_or_annotations:
# Image upload with points/annotations
is_uploading_annotations_not_just_points = annotation_options_form.cleaned_data['is_uploading_annotations_not_just_points']
imported_user = get_imported_user()
status = ImageStatus()
status.save()
metadata_obj.annotation_area = AnnotationAreaUtils.IMPORTED_STR
metadata_obj.save()
img = Image(
original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=PointGen.args_to_db_format(
point_generation_type=PointGen.Types.IMPORTED,
imported_number_of_points=len(image_annotations)
),
metadata=metadata_obj,
source=source,
status=status,
)
img.save()
# Iterate over this image's annotations and save them.
point_num = 0
for anno in image_annotations:
# Save the Point in the database.
point_num += 1
point = Point(row=anno['row'], column=anno['col'], point_number=point_num, image=img)
point.save()
if is_uploading_annotations_not_just_points:
label = Label.objects.filter(code=anno['label'])[0]
# Save the Annotation in the database, marking the annotations as imported.
annotation = Annotation(user=imported_user,
point=point, image=img, label=label, source=source)
annotation.save()
img.status.hasRandomPoints = True
if is_uploading_annotations_not_just_points:
img.status.annotatedByHuman = True
img.status.save()
else:
# Image upload, no points/annotations
image_status = ImageStatus()
image_status.save()
metadata_obj.annotation_area = source.image_annotation_area
metadata_obj.save()
# Save the image into the DB
img = Image(original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=source.default_point_generation_method,
metadata=metadata_obj,
source=source,
status=image_status,
)
img.save()
# Generate and save points
generate_points(img)
success_message = "Uploaded"
return dict(
status='ok',
message=success_message,
link=reverse('image_detail', args=[img.id]),
title=img.get_image_element_title(),
image_id=img.id,
)