def image_upload_process(imageFiles, imageOptionsForm, annotationOptionsForm, source, currentUser, annoFile): """ Helper method for the image upload view and the image+annotation import view. """ uploadedImages = [] duplicates = 0 imagesUploaded = 0 annotationsImported = 0 importedUser = get_imported_user() dupeOption = imageOptionsForm.cleaned_data['skip_or_replace_duplicates'] annotationData = None if annoFile: try: annotationData = annotations_file_to_python(annoFile, source) except FileContentError as errorDetail: return dict(error=True, message='Error reading labels file %s. %s' % (annoFile.name, errorDetail), ) for imageFile in imageFiles: filename = imageFile.name metadataDict = None metadata = Metadata(height_in_cm=source.image_height_in_cm) if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames': try: metadataDict = filename_to_metadata(filename, source) # Filename parse error. # TODO: check for validity of the file type and contents, too. except (ValueError, StopIteration): return dict(error=True, message='Upload failed - Error when parsing the filename %s for metadata.' % filename, ) # Detect duplicate images and handle them dupe = find_dupe_image(source, **metadataDict) if dupe: duplicates += 1 if dupeOption == 'skip': # Skip uploading this file. continue elif dupeOption == 'replace': # Proceed uploading this file, and delete the dupe. dupe.delete() # Set the metadata valueDict = get_location_value_objs(source, metadataDict['values'], createNewValues=True) photoDate = datetime.date(year = int(metadataDict['year']), month = int(metadataDict['month']), day = int(metadataDict['day'])) metadata.name = metadataDict['name'] metadata.photo_date = photoDate for key, value in valueDict.iteritems(): setattr(metadata, key, value) else: metadata.name = filename # Image + annotation import form # Assumes we got the images' metadata (from filenames or otherwise) if annotationData: pointsOnlyOption = annotationOptionsForm.cleaned_data['points_only'] # Use the location values and the year to build a string identifier for the image, such as: # Shore1;Reef5;...;2008 imageIdentifier = get_image_identifier(metadataDict['values'], metadataDict['year']) # Use the identifier as the index into the annotation file's data. if not annotationData.has_key(imageIdentifier): return dict(error=True, message='%s seems to have no annotations for the image file %s, which has the following keys:\n%s' % ( annoFile.name, imageFile.name, imageIdentifier.replace(';',' ')) ) imageAnnotations = annotationData[imageIdentifier] status = ImageStatus() status.save() metadata.annotation_area = AnnotationAreaUtils.IMPORTED_STR metadata.save() img = Image(original_file=imageFile, uploaded_by=currentUser, point_generation_method=PointGen.args_to_db_format( point_generation_type=PointGen.Types.IMPORTED, imported_number_of_points=len(imageAnnotations) ), metadata=metadata, source=source, status=status, ) img.save() # Iterate over this image's annotations and save them. pointNum = 1 for anno in imageAnnotations: # Save the Point in the database. point = Point(row=anno['row'], column=anno['col'], point_number=pointNum, image=img) point.save() if not pointsOnlyOption: label = Label.objects.filter(code=anno['label'])[0] # Save the Annotation in the database, marking the annotations as imported. annotation = Annotation(user=importedUser, point=point, image=img, label=label, source=source) annotation.save() annotationsImported += 1 pointNum += 1 img.status.hasRandomPoints = True if not pointsOnlyOption: img.status.annotatedByHuman = True img.status.save() # Image upload form, no annotations else: status = ImageStatus() status.save() metadata.annotation_area = source.image_annotation_area metadata.save() # Save the image into the DB img = Image(original_file=imageFile, uploaded_by=currentUser, point_generation_method=source.default_point_generation_method, metadata=metadata, source=source, status=status, ) img.save() # Generate and save points generate_points(img) # Up to 5 uploaded images will be shown # upon successful upload. # Prepend to list, so most recent image comes first uploadedImages.insert(0, img) if len(uploadedImages) > 5: uploadedImages = uploadedImages[:5] imagesUploaded += 1 # Construct success message. success_message = image_upload_success_message( num_images_uploaded=imagesUploaded, num_dupes=duplicates, dupe_option=dupeOption, num_annotations=annotationsImported, ) return dict(error=False, uploadedImages=uploadedImages, message=success_message, )
def upload_images_test(self, filenames, subdirectory=None, expect_success=True, expected_dupes=0, expected_errors=None, **options): """ Upload a number of images. subdirectory - If the files to upload are in a subdirectory of /sample_uploadables/data, as opposed to directly in /sample_uploadables/data, then this is the name of the subdirectory it's in. Should be a string with no slash. filenames - The filename as a string, if uploading one file; the filenames as a list of strings, if uploading multiple files. expect_success - True if expecting to successfully upload, False if expecting errors upon submitting the form. An AssertionError is thrown if any of the checks for success/non-success do not pass. expected_dupes - Number of the uploaded images that are expected to be duplicates of existing images. expected_errors - A dict of expected form errors. An AssertionError is thrown if expected errors != actual errors. """ if isinstance(filenames, basestring): filenames = [filenames] old_source_image_count = Image.objects.filter(source=Source.objects.get(pk=self.source_id)).count() if subdirectory: sample_uploadable_directory = os.path.join(settings.SAMPLE_UPLOADABLES_ROOT, 'data', subdirectory) else: sample_uploadable_directory = os.path.join(settings.SAMPLE_UPLOADABLES_ROOT, 'data') files_to_upload = [] for filename in filenames: sample_uploadable_path = os.path.join(sample_uploadable_directory, filename) f = open(sample_uploadable_path, 'rb') files_to_upload.append(f) post_dict = dict( files=files_to_upload, skip_or_replace_duplicates='skip', specify_metadata='filenames', ) post_dict.update(options) response = self.client.post(reverse('image_upload', kwargs={'source_id': self.source_id}), post_dict) for uploaded_file in files_to_upload: uploaded_file.close() self.assertStatusOK(response) new_source_image_count = Image.objects.filter(source=Source.objects.get(pk=self.source_id)).count() # TODO: Accommodate partial successes (some uploaded, some not)? if expect_success: dupe_option = post_dict['skip_or_replace_duplicates'] if dupe_option == 'skip': expected_num_images_uploaded = len(filenames) - expected_dupes else: # 'replace' expected_num_images_uploaded = len(filenames) # The 'uploaded images' display on the success page should include # up to 5 images. # TODO: If more than 5 were uploaded, check that the correct 5 images are shown? self.assertEqual(len(response.context['uploadedImages']), min(expected_num_images_uploaded, 5)) # The number of images in the source should be correct. self.assertEqual(new_source_image_count, len(filenames)+old_source_image_count-expected_dupes) # The correct page-top messages should be in the context. self.assertMessages(response, [image_upload_success_message( num_images_uploaded=expected_num_images_uploaded, num_dupes=expected_dupes, dupe_option=dupe_option, num_annotations=0, )]) else: # There should be no images in the 'uploaded images' display. self.assertEqual(len(response.context['uploadedImages']), 0) # The number of images in the source should have stayed the same. self.assertEqual(new_source_image_count, old_source_image_count) # The correct page-top messages should be in the context. self.assertMessages(response, [msg_consts.FORM_ERRORS]) # Check that the appropriate form errors have been raised. if expected_errors: self.assertFormErrors( response, 'imageForm', expected_errors, ) # If the rest of the test wants to do anything else with the response, # then it can. return response