def source_new(request):
"""
Page with the form to create a new Source.
"""
# We can get here one of two ways: either we just got to the form
# page, or we just submitted the form. If POST, we submitted; if
# GET, we just got here.
if request.method == 'POST':
# A form bound to the POST data
sourceForm = ImageSourceForm(request.POST)
pointGenForm = PointGenForm(request.POST)
annotationAreaForm = AnnotationAreaPercentsForm(request.POST)
# is_valid() calls our ModelForm's clean() and checks validity
source_form_is_valid = sourceForm.is_valid()
point_gen_form_is_valid = pointGenForm.is_valid()
annotation_area_form_is_valid = annotationAreaForm.is_valid()
if source_form_is_valid and point_gen_form_is_valid and annotation_area_form_is_valid:
# After calling a ModelForm's is_valid(), an instance is created.
# We can get this instance and add a bit more to it before saving to the DB.
newSource = sourceForm.instance
newSource.default_point_generation_method = PointGen.args_to_db_format(**pointGenForm.cleaned_data)
newSource.image_annotation_area = AnnotationAreaUtils.percentages_to_db_format(**annotationAreaForm.cleaned_data)
newSource.labelset = LabelSet.getEmptyLabelset()
newSource.save()
# Make the user a source admin
newSource.assign_role(request.user, Source.PermTypes.ADMIN.code)
# Add a success message
messages.success(request, 'Source successfully created.')
# Redirect to the source's main page
return HttpResponseRedirect(reverse('source_main', args=[newSource.id]))
else:
# Show the form again, with error message
messages.error(request, 'Please correct the errors below.')
else:
# An unbound form (empty form)
sourceForm = ImageSourceForm()
pointGenForm = PointGenForm()
annotationAreaForm = AnnotationAreaPercentsForm()
# RequestContext needed for CSRF verification of POST form,
# and to correctly get the path of the CSS file being used.
return render_to_response('images/source_new.html', {
'sourceForm': sourceForm,
'pointGenForm': pointGenForm,
'annotationAreaForm': annotationAreaForm,
},
context_instance=RequestContext(request)
)
def source_edit(request, source_id):
"""
Edit a source: name, visibility, location keys, etc.
"""
source = get_object_or_404(Source, id=source_id)
if request.method == 'POST':
# Cancel
cancel = request.POST.get('cancel', None)
if cancel:
messages.success(request, 'Edit cancelled.')
return HttpResponseRedirect(reverse('source_main', args=[source_id]))
# Submit
sourceForm = ImageSourceForm(request.POST, instance=source)
pointGenForm = PointGenForm(request.POST)
annotationAreaForm = AnnotationAreaPercentsForm(request.POST)
# Make sure is_valid() is called for all forms, so all forms are checked and
# all relevant error messages appear.
source_form_is_valid = sourceForm.is_valid()
point_gen_form_is_valid = pointGenForm.is_valid()
annotation_area_form_is_valid = annotationAreaForm.is_valid()
if source_form_is_valid and point_gen_form_is_valid and annotation_area_form_is_valid:
editedSource = sourceForm.instance
editedSource.default_point_generation_method = PointGen.args_to_db_format(**pointGenForm.cleaned_data)
editedSource.image_annotation_area = AnnotationAreaUtils.percentages_to_db_format(**annotationAreaForm.cleaned_data)
editedSource.save()
messages.success(request, 'Source successfully edited.')
return HttpResponseRedirect(reverse('source_main', args=[source_id]))
else:
messages.error(request, 'Please correct the errors below.')
else:
# Just reached this form page
sourceForm = ImageSourceForm(instance=source)
pointGenForm = PointGenForm(source=source)
annotationAreaForm = AnnotationAreaPercentsForm(source=source)
return render_to_response('images/source_edit.html', {
'source': source,
'editSourceForm': sourceForm,
'pointGenForm': pointGenForm,
'annotationAreaForm': annotationAreaForm,
},
context_instance=RequestContext(request)
)
def test_post_success(self):
"""
Successful creation of a new source.
"""
datetime_before_creation = datetime.datetime.now().replace(microsecond=0)
response = self.client.post(reverse('source_new'), self.source_args)
source_id = Source.objects.latest('create_date').pk
self.assertRedirects(response, reverse('source_main',
kwargs={
'source_id': source_id,
}
))
new_source = Source.objects.get(pk=source_id)
self.assertEqual(new_source.name, self.source_args['name'])
self.assertEqual(new_source.visibility, self.source_args['visibility'])
self.assertEqual(new_source.labelset, LabelSet.getEmptyLabelset())
self.assertEqual(new_source.key1, self.source_args['key1'])
self.assertEqual(new_source.key2, '')
self.assertEqual(new_source.key3, '')
self.assertEqual(new_source.key4, '')
self.assertEqual(new_source.key5, '')
self.assertEqual(new_source.default_point_generation_method, PointGen.args_to_db_format(
point_generation_type=self.source_args['point_generation_type'],
simple_number_of_points=self.source_args['simple_number_of_points'],
))
self.assertEqual(new_source.image_height_in_cm, self.source_args['image_height_in_cm'])
self.assertEqual(new_source.image_annotation_area, AnnotationAreaUtils.percentages_to_db_format(
min_x=self.source_args['min_x'], max_x=self.source_args['max_x'],
min_y=self.source_args['min_y'], max_y=self.source_args['max_y'],
))
self.assertEqual(new_source.longitude, self.source_args['longitude'])
self.assertEqual(new_source.latitude, self.source_args['latitude'])
self.assertEqual(new_source.enable_robot_classifier, True)
# This check is of limited use since database datetimes (in
# MySQL 5.1 at least) get truncated to whole seconds. But it still
# doesn't hurt to check.
self.assertTrue(datetime_before_creation <= new_source.create_date)
self.assertTrue(new_source.create_date <= datetime.datetime.now().replace(microsecond=0))
def test_post_success(self):
"""
Successful edit of an existing source.
"""
original_source = Source.objects.get(pk=self.source_id)
original_create_date = original_source.create_date
original_enable_robot = original_source.enable_robot_classifier
response = self.client.post(reverse('source_edit', kwargs={'source_id': self.source_id}),
self.source_args,
)
self.assertRedirects(response, reverse('source_main',
kwargs={'source_id': self.source_id}
))
edited_source = Source.objects.get(pk=self.source_id)
self.assertEqual(edited_source.name, self.source_args['name'])
self.assertEqual(edited_source.visibility, self.source_args['visibility'])
self.assertEqual(edited_source.create_date, original_create_date)
self.assertEqual(edited_source.labelset, LabelSet.getEmptyLabelset())
self.assertEqual(edited_source.key1, self.source_args['key1'])
self.assertEqual(edited_source.key2, '')
self.assertEqual(edited_source.key3, '')
self.assertEqual(edited_source.key4, '')
self.assertEqual(edited_source.key5, '')
self.assertEqual(edited_source.default_point_generation_method, PointGen.args_to_db_format(
point_generation_type=self.source_args['point_generation_type'],
simple_number_of_points=self.source_args['simple_number_of_points'],
))
self.assertEqual(edited_source.image_height_in_cm, self.source_args['image_height_in_cm'])
self.assertEqual(edited_source.image_annotation_area, AnnotationAreaUtils.percentages_to_db_format(
min_x=self.source_args['min_x'], max_x=self.source_args['max_x'],
min_y=self.source_args['min_y'], max_y=self.source_args['max_y'],
))
self.assertEqual(edited_source.longitude, self.source_args['longitude'])
self.assertEqual(edited_source.latitude, self.source_args['latitude'])
self.assertEqual(edited_source.enable_robot_classifier, original_enable_robot)
def import_archived_annotations(source_id, anndict, with_labels = True):
source = Source.objects.get(pk = source_id) # let's fetch the relevant source.
imported_user = get_imported_user() # the imported user.
images = source.get_all_images().filter(metadata__name__in = list(anndict.keys())) # grab all image that have names in the .csv file.
for image in images:
# Start by remove annotations and points for this image
for ann in Annotation.objects.filter(image=image):
ann.delete()
for point in Point.objects.filter(image=image):
point.delete()
# Next, set image metadata to IMPORTED.
image.point_generation_method = PointGen.args_to_db_format(
point_generation_type=PointGen.Types.IMPORTED,
imported_number_of_points=len(anndict[image.metadata.name])
)
image.save()
image.status.hasRandomPoints = True
image.status.annotatedByHuman = with_labels
image.status.save()
image.after_annotation_area_change() # set the backend status correctly.
# Iterate over this image's annotations and save them.
for (point_num, (row, col, code)) in enumerate(anndict[image.metadata.name]):
# Save the Point in the database.
point = Point(row=row, column=col, point_number=point_num + 1, image=image)
point.save()
# and save the Annotation.
if with_labels:
label = Label.objects.filter(code=code)[0]
annotation = Annotation(user=imported_user, point=point, image=image, label=label, source=source)
annotation.save()
def image_upload_process(imageFiles, imageOptionsForm, annotationOptionsForm, source, currentUser, annoFile):
"""
Helper method for the image upload view and the image+annotation
import view.
"""
uploadedImages = []
duplicates = 0
imagesUploaded = 0
annotationsImported = 0
importedUser = get_imported_user()
dupeOption = imageOptionsForm.cleaned_data['skip_or_replace_duplicates']
annotationData = None
if annoFile:
try:
annotationData = annotations_file_to_python(annoFile, source)
except FileContentError as errorDetail:
return dict(error=True,
message='Error reading labels file %s. %s' % (annoFile.name, errorDetail),
)
for imageFile in imageFiles:
filename = imageFile.name
metadataDict = None
metadata = Metadata(height_in_cm=source.image_height_in_cm)
if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames':
try:
metadataDict = filename_to_metadata(filename, source)
# Filename parse error.
# TODO: check for validity of the file type and contents, too.
except (ValueError, StopIteration):
return dict(error=True,
message='Upload failed - Error when parsing the filename %s for metadata.' % filename,
)
# Detect duplicate images and handle them
dupe = find_dupe_image(source, **metadataDict)
if dupe:
duplicates += 1
if dupeOption == 'skip':
# Skip uploading this file.
continue
elif dupeOption == 'replace':
# Proceed uploading this file, and delete the dupe.
dupe.delete()
# Set the metadata
valueDict = get_location_value_objs(source, metadataDict['values'], createNewValues=True)
photoDate = datetime.date(year = int(metadataDict['year']),
month = int(metadataDict['month']),
day = int(metadataDict['day']))
metadata.name = metadataDict['name']
metadata.photo_date = photoDate
for key, value in valueDict.iteritems():
setattr(metadata, key, value)
else:
metadata.name = filename
# Image + annotation import form
# Assumes we got the images' metadata (from filenames or otherwise)
if annotationData:
pointsOnlyOption = annotationOptionsForm.cleaned_data['points_only']
# Use the location values and the year to build a string identifier for the image, such as:
# Shore1;Reef5;...;2008
imageIdentifier = get_image_identifier(metadataDict['values'], metadataDict['year'])
# Use the identifier as the index into the annotation file's data.
if not annotationData.has_key(imageIdentifier):
return dict(error=True,
message='%s seems to have no annotations for the image file %s, which has the following keys:\n%s' % (
annoFile.name, imageFile.name, imageIdentifier.replace(';',' '))
)
imageAnnotations = annotationData[imageIdentifier]
status = ImageStatus()
status.save()
metadata.annotation_area = AnnotationAreaUtils.IMPORTED_STR
metadata.save()
img = Image(original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=PointGen.args_to_db_format(
point_generation_type=PointGen.Types.IMPORTED,
imported_number_of_points=len(imageAnnotations)
),
metadata=metadata,
source=source,
status=status,
)
img.save()
# Iterate over this image's annotations and save them.
pointNum = 1
for anno in imageAnnotations:
# Save the Point in the database.
point = Point(row=anno['row'], column=anno['col'], point_number=pointNum, image=img)
point.save()
if not pointsOnlyOption:
label = Label.objects.filter(code=anno['label'])[0]
# Save the Annotation in the database, marking the annotations as imported.
annotation = Annotation(user=importedUser,
point=point, image=img, label=label, source=source)
annotation.save()
annotationsImported += 1
pointNum += 1
img.status.hasRandomPoints = True
if not pointsOnlyOption:
img.status.annotatedByHuman = True
img.status.save()
# Image upload form, no annotations
else:
status = ImageStatus()
status.save()
metadata.annotation_area = source.image_annotation_area
metadata.save()
# Save the image into the DB
img = Image(original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=source.default_point_generation_method,
metadata=metadata,
source=source,
status=status,
)
img.save()
# Generate and save points
generate_points(img)
# Up to 5 uploaded images will be shown
# upon successful upload.
# Prepend to list, so most recent image comes first
uploadedImages.insert(0, img)
if len(uploadedImages) > 5:
uploadedImages = uploadedImages[:5]
imagesUploaded += 1
# Construct success message.
success_message = image_upload_success_message(
num_images_uploaded=imagesUploaded,
num_dupes=duplicates,
dupe_option=dupeOption,
num_annotations=annotationsImported,
)
return dict(error=False,
uploadedImages=uploadedImages,
message=success_message,
)
def image_upload_process(imageFile, imageOptionsForm,
annotation_dict_id,
csv_dict_id, metadata_import_form_class,
annotation_options_form,
source, currentUser):
is_uploading_points_or_annotations = annotation_options_form.cleaned_data['is_uploading_points_or_annotations']
filename = imageFile.name
metadata_dict = None
metadata_obj = Metadata(height_in_cm=source.image_height_in_cm)
if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames':
filename_check_result = check_image_filename(filename, source)
filename_status = filename_check_result['status']
if filename_status == 'error':
# This case should never happen if the pre-upload
# status checking is doing its job, but just in case...
return dict(
status=filename_status,
message=u"{m}".format(m=filename_check_result['message']),
link=None,
title=None,
)
# Set the metadata
metadata_dict = filename_check_result['metadata_dict']
value_dict = get_location_value_objs(source, metadata_dict['values'], createNewValues=True)
photo_date = datetime.date(
year = int(metadata_dict['year']),
month = int(metadata_dict['month']),
day = int(metadata_dict['day'])
)
metadata_obj.name = metadata_dict['name']
metadata_obj.photo_date = photo_date
for key, value in value_dict.iteritems():
setattr(metadata_obj, key, value)
elif imageOptionsForm.cleaned_data['specify_metadata'] == 'csv':
if not csv_dict_id:
return dict(
status='error',
message=u"{m}".format(m="CSV file was not uploaded."),
link=None,
title=None,
)
csv_dict_filename = os.path.join(
settings.SHELVED_ANNOTATIONS_DIR,
'csv_source{source_id}_{dict_id}.db'.format(
source_id=source.id,
dict_id=csv_dict_id,
),
)
# Corner case: the specified shelved annotation file doesn't exist.
# Perhaps the file was created a while ago and has been pruned since,
# or perhaps there is a bug.
if not os.path.isfile(csv_dict_filename):
return dict(
status='error',
message="CSV file could not be found - if you provided the .csv file a while ago, maybe it just timed out. Please retry the upload.",
link=None,
title=None,
)
csv_dict = shelve.open(csv_dict_filename)
#index into the csv_dict with the filename. the str() is to handle
#the case where the filename is a unicode object instead of a str;
#unicode objects can't index into dicts.
filename_str = str(filename)
if filename_str in csv_dict:
# There is CSV metadata for this file.
metadata_dict = csv_dict[str(filename)]
csv_dict.close()
# The reason this uses metadata_import_form_class instead of
# importing MetadataImportForm is that I'm too lazy to deal with the
# circular-import implications of the latter solution right now.
# -Stephen
metadata_import_form = metadata_import_form_class(
source.id, True, metadata_dict,
)
if not metadata_import_form.is_valid():
return dict(
status='error',
message="Unknown error with the CSV metadata.",
link=None,
title=None,
)
fields = ['photo_date', 'value1', 'value2', 'value3', 'value4',
'value5', 'height_in_cm', 'latitude', 'longitude',
'depth', 'camera', 'photographer', 'water_quality',
'strobes', 'framing', 'balance']
for field in fields:
if not field in metadata_import_form.fields:
# A location value field that's not in this form
continue
value = metadata_import_form.cleaned_data[field]
# Check for a non-empty value; don't want empty values to
# override default values that we've already set on the
# metadata_obj
if value:
setattr(metadata_obj, field, value)
else:
# No CSV metadata for this file.
csv_dict.close()
metadata_obj.name = filename
else:
# Not specifying any metadata at upload time.
metadata_obj.name = filename
image_annotations = None
has_points_or_annotations = False
if is_uploading_points_or_annotations:
# Corner case: somehow, we're uploading with points+annotations and without
# a checked annotation file specified. This probably indicates a bug.
if not annotation_dict_id:
return dict(
status='error',
message=u"{m}".format(m=str_consts.UPLOAD_ANNOTATIONS_ON_AND_NO_ANNOTATION_DICT_ERROR_STR),
link=None,
title=None,
)
annotation_dict_filename = os.path.join(
settings.SHELVED_ANNOTATIONS_DIR,
'source{source_id}_{dict_id}'.format(
source_id=source.id,
dict_id=annotation_dict_id,
),
)
# Corner case: the specified shelved annotation file doesn't exist.
# Perhaps the file was created a while ago and has been pruned since,
# or perhaps there is a bug.
if not os.path.isfile(annotation_dict_filename):
return dict(
status='error',
message="Annotations could not be found - if you provided the .txt file a while ago, maybe it just timed out. Please retry the upload.",
link=None,
title=None,
)
# Use the location values and the year to build a string identifier for the image, such as:
# Shore1;Reef5;...;2008
# Convert to a string (instead of a unicode string) for the shelve key lookup.
image_identifier = str(get_image_identifier(metadata_dict['values'], metadata_dict['year']))
annotation_dict = shelve.open(annotation_dict_filename)
if annotation_dict.has_key(image_identifier):
image_annotations = annotation_dict[image_identifier]
has_points_or_annotations = True
annotation_dict.close()
if has_points_or_annotations:
# Image upload with points/annotations
is_uploading_annotations_not_just_points = annotation_options_form.cleaned_data['is_uploading_annotations_not_just_points']
imported_user = get_imported_user()
status = ImageStatus()
status.save()
metadata_obj.annotation_area = AnnotationAreaUtils.IMPORTED_STR
metadata_obj.save()
img = Image(
original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=PointGen.args_to_db_format(
point_generation_type=PointGen.Types.IMPORTED,
imported_number_of_points=len(image_annotations)
),
metadata=metadata_obj,
source=source,
status=status,
)
img.save()
# Iterate over this image's annotations and save them.
point_num = 0
for anno in image_annotations:
# Save the Point in the database.
point_num += 1
point = Point(row=anno['row'], column=anno['col'], point_number=point_num, image=img)
point.save()
if is_uploading_annotations_not_just_points:
label = Label.objects.filter(code=anno['label'])[0]
# Save the Annotation in the database, marking the annotations as imported.
annotation = Annotation(user=imported_user,
point=point, image=img, label=label, source=source)
annotation.save()
img.status.hasRandomPoints = True
if is_uploading_annotations_not_just_points:
img.status.annotatedByHuman = True
img.status.save()
else:
# Image upload, no points/annotations
image_status = ImageStatus()
image_status.save()
metadata_obj.annotation_area = source.image_annotation_area
metadata_obj.save()
# Save the image into the DB
img = Image(original_file=imageFile,
uploaded_by=currentUser,
point_generation_method=source.default_point_generation_method,
metadata=metadata_obj,
source=source,
status=image_status,
)
img.save()
# Generate and save points
generate_points(img)
success_message = "Uploaded"
return dict(
status='ok',
message=success_message,
link=reverse('image_detail', args=[img.id]),
title=img.get_image_element_title(),
image_id=img.id,
)
def source_new(request):
"""
Page with the form to create a new Source.
"""
# We can get here one of two ways: either we just got to the form
# page, or we just submitted the form. If POST, we submitted; if
# GET, we just got here.
if request.method == 'POST':
# Bind the forms to the submitted POST data.
sourceForm = ImageSourceForm(request.POST)
location_key_form = LocationKeyForm(request.POST)
pointGenForm = PointGenForm(request.POST)
annotationAreaForm = AnnotationAreaPercentsForm(request.POST)
# <form>.is_valid() calls <form>.clean() and checks field validity.
# Make sure is_valid() is called for all forms, so all forms are checked and
# all relevant error messages appear.
source_form_is_valid = sourceForm.is_valid()
location_key_form_is_valid = location_key_form.is_valid()
point_gen_form_is_valid = pointGenForm.is_valid()
annotation_area_form_is_valid = annotationAreaForm.is_valid()
if source_form_is_valid and location_key_form_is_valid \
and point_gen_form_is_valid and annotation_area_form_is_valid:
# Since sourceForm is a ModelForm, after calling sourceForm's
# is_valid(), a Source instance is created. We retrieve this
# instance and add the other values to it before saving to the DB.
newSource = sourceForm.instance
for key_field in ['key1', 'key2', 'key3', 'key4', 'key5']:
if key_field in location_key_form.cleaned_data:
setattr(newSource, key_field, location_key_form.cleaned_data[key_field])
newSource.default_point_generation_method = PointGen.args_to_db_format(**pointGenForm.cleaned_data)
newSource.image_annotation_area = AnnotationAreaUtils.percentages_to_db_format(**annotationAreaForm.cleaned_data)
newSource.labelset = LabelSet.getEmptyLabelset()
newSource.save()
# Make the current user an admin of the new source
newSource.assign_role(request.user, Source.PermTypes.ADMIN.code)
# Add a success message
messages.success(request, 'Source successfully created.')
# Redirect to the source's main page
return HttpResponseRedirect(reverse('source_main', args=[newSource.id]))
else:
# Show the form again, with error message
messages.error(request, 'Please correct the errors below.')
else:
# Unbound (empty) forms
sourceForm = ImageSourceForm()
location_key_form = LocationKeyForm()
pointGenForm = PointGenForm()
annotationAreaForm = AnnotationAreaPercentsForm()
# RequestContext is needed for CSRF verification of the POST form,
# and to correctly get the path of the CSS file being used.
return render_to_response('images/source_new.html', {
'sourceForm': sourceForm,
'location_key_form': location_key_form,
'pointGenForm': pointGenForm,
'annotationAreaForm': annotationAreaForm,
},
context_instance=RequestContext(request)
)
