Ejemplo n.º 1
0
 def test_classify_does_not_overwrite_imported_annotations(self):
     """
     Tests that when an image is partially annotated by imported data,
     automatic classification doesn't overwrite any of the imported
     points.
     """
     self.helper_classify_does_not_overwrite_manual_annotations(
         get_imported_user()
     )
Ejemplo n.º 2
0
def import_archived_annotations(source_id, anndict, with_labels = True):

    source = Source.objects.get(pk = source_id) # let's fetch the relevant source.
    imported_user = get_imported_user() # the imported user.

    images = source.get_all_images().filter(metadata__name__in = list(anndict.keys())) # grab all image that have names in the .csv file.

    for image in images:

        # Start by remove annotations and points for this image
        for ann in Annotation.objects.filter(image=image):
            ann.delete()
        for point in Point.objects.filter(image=image):
            point.delete()
    
        # Next, set image metadata to IMPORTED.
        image.point_generation_method = PointGen.args_to_db_format(
                point_generation_type=PointGen.Types.IMPORTED,
                imported_number_of_points=len(anndict[image.metadata.name])
        )
        image.save()
        image.status.hasRandomPoints = True
        image.status.annotatedByHuman = with_labels
        image.status.save()
        image.after_annotation_area_change() # set the backend status correctly.

        # Iterate over this image's annotations and save them.
        for (point_num, (row, col, code)) in enumerate(anndict[image.metadata.name]):
            
            # Save the Point in the database.
            point = Point(row=row, column=col, point_number=point_num + 1, image=image)
            point.save()

            # and save the Annotation.
            if with_labels:
                label = Label.objects.filter(code=code)[0]
                annotation = Annotation(user=imported_user, point=point, image=image, label=label, source=source)
                annotation.save()
Ejemplo n.º 3
0
def image_upload_process(imageFiles, imageOptionsForm, annotationOptionsForm, source, currentUser, annoFile):
    """
    Helper method for the image upload view and the image+annotation
    import view.
    """

    uploadedImages = []
    duplicates = 0
    imagesUploaded = 0
    annotationsImported = 0
    importedUser = get_imported_user()

    dupeOption = imageOptionsForm.cleaned_data['skip_or_replace_duplicates']

    annotationData = None
    if annoFile:
        try:
            annotationData = annotations_file_to_python(annoFile, source)
        except FileContentError as errorDetail:
            return dict(error=True,
                message='Error reading labels file %s. %s' % (annoFile.name, errorDetail),
            )

    for imageFile in imageFiles:

        filename = imageFile.name
        metadataDict = None
        metadata = Metadata(height_in_cm=source.image_height_in_cm)

        if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames':

            try:
                metadataDict = filename_to_metadata(filename, source)

            # Filename parse error.
            # TODO: check for validity of the file type and contents, too.
            except (ValueError, StopIteration):
                return dict(error=True,
                    message='Upload failed - Error when parsing the filename %s for metadata.' % filename,
                )

            # Detect duplicate images and handle them
            dupe = find_dupe_image(source, **metadataDict)
            if dupe:
                duplicates += 1
                if dupeOption == 'skip':
                    # Skip uploading this file.
                    continue
                elif dupeOption == 'replace':
                    # Proceed uploading this file, and delete the dupe.
                    dupe.delete()

            # Set the metadata
            valueDict = get_location_value_objs(source, metadataDict['values'], createNewValues=True)
            photoDate = datetime.date(year = int(metadataDict['year']),
                             month = int(metadataDict['month']),
                             day = int(metadataDict['day']))

            metadata.name = metadataDict['name']
            metadata.photo_date = photoDate
            for key, value in valueDict.iteritems():
                setattr(metadata, key, value)

        else:
            metadata.name = filename

        # Image + annotation import form
        # Assumes we got the images' metadata (from filenames or otherwise)
        if annotationData:

            pointsOnlyOption = annotationOptionsForm.cleaned_data['points_only']

            # Use the location values and the year to build a string identifier for the image, such as:
            # Shore1;Reef5;...;2008
            imageIdentifier = get_image_identifier(metadataDict['values'], metadataDict['year'])

            # Use the identifier as the index into the annotation file's data.
            if not annotationData.has_key(imageIdentifier):
                return dict(error=True,
                    message='%s seems to have no annotations for the image file %s, which has the following keys:\n%s' % (
                        annoFile.name, imageFile.name, imageIdentifier.replace(';',' '))
                )

            imageAnnotations = annotationData[imageIdentifier]

            status = ImageStatus()
            status.save()

            metadata.annotation_area = AnnotationAreaUtils.IMPORTED_STR
            metadata.save()

            img = Image(original_file=imageFile,
                    uploaded_by=currentUser,
                    point_generation_method=PointGen.args_to_db_format(
                        point_generation_type=PointGen.Types.IMPORTED,
                        imported_number_of_points=len(imageAnnotations)
                    ),
                    metadata=metadata,
                    source=source,
                    status=status,
                  )
            img.save()

            # Iterate over this image's annotations and save them.
            pointNum = 1
            for anno in imageAnnotations:

                # Save the Point in the database.
                point = Point(row=anno['row'], column=anno['col'], point_number=pointNum, image=img)
                point.save()

                if not pointsOnlyOption:
                    label = Label.objects.filter(code=anno['label'])[0]

                    # Save the Annotation in the database, marking the annotations as imported.
                    annotation = Annotation(user=importedUser,
                                            point=point, image=img, label=label, source=source)
                    annotation.save()

                    annotationsImported += 1

                pointNum += 1

            img.status.hasRandomPoints = True
            if not pointsOnlyOption:
                img.status.annotatedByHuman = True
            img.status.save()

        # Image upload form, no annotations
        else:
            status = ImageStatus()
            status.save()

            metadata.annotation_area = source.image_annotation_area
            metadata.save()

            # Save the image into the DB
            img = Image(original_file=imageFile,
                    uploaded_by=currentUser,
                    point_generation_method=source.default_point_generation_method,
                    metadata=metadata,
                    source=source,
                    status=status,
                  )
            img.save()

            # Generate and save points
            generate_points(img)

        # Up to 5 uploaded images will be shown
        # upon successful upload.
        # Prepend to list, so most recent image comes first
        uploadedImages.insert(0, img)
        if len(uploadedImages) > 5:
            uploadedImages = uploadedImages[:5]

        imagesUploaded += 1

    # Construct success message.
    success_message = image_upload_success_message(
        num_images_uploaded=imagesUploaded,
        num_dupes=duplicates,
        dupe_option=dupeOption,
        num_annotations=annotationsImported,
    )

    return dict(error=False,
        uploadedImages=uploadedImages,
        message=success_message,
    )
Ejemplo n.º 4
0
def image_upload_process(imageFile, imageOptionsForm,
                         annotation_dict_id,
                         csv_dict_id, metadata_import_form_class,
                         annotation_options_form,
                         source, currentUser):

    is_uploading_points_or_annotations = annotation_options_form.cleaned_data['is_uploading_points_or_annotations']

    filename = imageFile.name
    metadata_dict = None
    metadata_obj = Metadata(height_in_cm=source.image_height_in_cm)

    if imageOptionsForm.cleaned_data['specify_metadata'] == 'filenames':

        filename_check_result = check_image_filename(filename, source)
        filename_status = filename_check_result['status']

        if filename_status == 'error':
            # This case should never happen if the pre-upload
            # status checking is doing its job, but just in case...
            return dict(
                status=filename_status,
                message=u"{m}".format(m=filename_check_result['message']),
                link=None,
                title=None,
            )

        # Set the metadata
        metadata_dict = filename_check_result['metadata_dict']

        value_dict = get_location_value_objs(source, metadata_dict['values'], createNewValues=True)
        photo_date = datetime.date(
            year = int(metadata_dict['year']),
            month = int(metadata_dict['month']),
            day = int(metadata_dict['day'])
        )

        metadata_obj.name = metadata_dict['name']
        metadata_obj.photo_date = photo_date
        for key, value in value_dict.iteritems():
            setattr(metadata_obj, key, value)

    elif imageOptionsForm.cleaned_data['specify_metadata'] == 'csv':

        if not csv_dict_id:
            return dict(
                status='error',
                message=u"{m}".format(m="CSV file was not uploaded."),
                link=None,
                title=None,
            )

        csv_dict_filename = os.path.join(
            settings.SHELVED_ANNOTATIONS_DIR,
            'csv_source{source_id}_{dict_id}.db'.format(
                source_id=source.id,
                dict_id=csv_dict_id,
            ),
        )

        # Corner case: the specified shelved annotation file doesn't exist.
        # Perhaps the file was created a while ago and has been pruned since,
        # or perhaps there is a bug.
        if not os.path.isfile(csv_dict_filename):
            return dict(
                status='error',
                message="CSV file could not be found - if you provided the .csv file a while ago, maybe it just timed out. Please retry the upload.",
                link=None,
                title=None,
            )

        csv_dict = shelve.open(csv_dict_filename)

        #index into the csv_dict with the filename. the str() is to handle
        #the case where the filename is a unicode object instead of a str;
        #unicode objects can't index into dicts.
        filename_str = str(filename)

        if filename_str in csv_dict:

            # There is CSV metadata for this file.

            metadata_dict = csv_dict[str(filename)]
            csv_dict.close()

            # The reason this uses metadata_import_form_class instead of
            # importing MetadataImportForm is that I'm too lazy to deal with the
            # circular-import implications of the latter solution right now.
            # -Stephen
            metadata_import_form = metadata_import_form_class(
                source.id, True, metadata_dict,
            )

            if not metadata_import_form.is_valid():
                return dict(
                    status='error',
                    message="Unknown error with the CSV metadata.",
                    link=None,
                    title=None,
                )

            fields = ['photo_date', 'value1', 'value2', 'value3', 'value4',
                      'value5', 'height_in_cm', 'latitude', 'longitude',
                      'depth', 'camera', 'photographer', 'water_quality',
                      'strobes', 'framing', 'balance']

            for field in fields:

                if not field in metadata_import_form.fields:
                    # A location value field that's not in this form
                    continue

                value = metadata_import_form.cleaned_data[field]
                # Check for a non-empty value; don't want empty values to
                # override default values that we've already set on the
                # metadata_obj
                if value:
                    setattr(metadata_obj, field, value)

        else:

            # No CSV metadata for this file.

            csv_dict.close()

        metadata_obj.name = filename

    else:

        # Not specifying any metadata at upload time.
        metadata_obj.name = filename


    image_annotations = None
    has_points_or_annotations = False

    if is_uploading_points_or_annotations:

        # Corner case: somehow, we're uploading with points+annotations and without
        # a checked annotation file specified.  This probably indicates a bug.
        if not annotation_dict_id:
            return dict(
                status='error',
                message=u"{m}".format(m=str_consts.UPLOAD_ANNOTATIONS_ON_AND_NO_ANNOTATION_DICT_ERROR_STR),
                link=None,
                title=None,
            )

        annotation_dict_filename = os.path.join(
            settings.SHELVED_ANNOTATIONS_DIR,
            'source{source_id}_{dict_id}'.format(
                source_id=source.id,
                dict_id=annotation_dict_id,
            ),
        )

        # Corner case: the specified shelved annotation file doesn't exist.
        # Perhaps the file was created a while ago and has been pruned since,
        # or perhaps there is a bug.
        if not os.path.isfile(annotation_dict_filename):
            return dict(
                status='error',
                message="Annotations could not be found - if you provided the .txt file a while ago, maybe it just timed out. Please retry the upload.",
                link=None,
                title=None,
            )


        # Use the location values and the year to build a string identifier for the image, such as:
        # Shore1;Reef5;...;2008
        # Convert to a string (instead of a unicode string) for the shelve key lookup.
        image_identifier = str(get_image_identifier(metadata_dict['values'], metadata_dict['year']))

        annotation_dict = shelve.open(annotation_dict_filename)

        if annotation_dict.has_key(image_identifier):
            image_annotations = annotation_dict[image_identifier]
            has_points_or_annotations = True
        annotation_dict.close()

    if has_points_or_annotations:
        # Image upload with points/annotations

        is_uploading_annotations_not_just_points = annotation_options_form.cleaned_data['is_uploading_annotations_not_just_points']
        imported_user = get_imported_user()

        status = ImageStatus()
        status.save()

        metadata_obj.annotation_area = AnnotationAreaUtils.IMPORTED_STR
        metadata_obj.save()

        img = Image(
            original_file=imageFile,
            uploaded_by=currentUser,
            point_generation_method=PointGen.args_to_db_format(
                point_generation_type=PointGen.Types.IMPORTED,
                imported_number_of_points=len(image_annotations)
            ),
            metadata=metadata_obj,
            source=source,
            status=status,
        )
        img.save()

        # Iterate over this image's annotations and save them.
        point_num = 0
        for anno in image_annotations:

            # Save the Point in the database.
            point_num += 1
            point = Point(row=anno['row'], column=anno['col'], point_number=point_num, image=img)
            point.save()

            if is_uploading_annotations_not_just_points:
                label = Label.objects.filter(code=anno['label'])[0]

                # Save the Annotation in the database, marking the annotations as imported.
                annotation = Annotation(user=imported_user,
                    point=point, image=img, label=label, source=source)
                annotation.save()

        img.status.hasRandomPoints = True
        if is_uploading_annotations_not_just_points:
            img.status.annotatedByHuman = True
        img.status.save()
    else:
        # Image upload, no points/annotations
        image_status = ImageStatus()
        image_status.save()

        metadata_obj.annotation_area = source.image_annotation_area
        metadata_obj.save()

        # Save the image into the DB
        img = Image(original_file=imageFile,
            uploaded_by=currentUser,
            point_generation_method=source.default_point_generation_method,
            metadata=metadata_obj,
            source=source,
            status=image_status,
        )
        img.save()

        # Generate and save points
        generate_points(img)

    success_message = "Uploaded"

    return dict(
        status='ok',
        message=success_message,
        link=reverse('image_detail', args=[img.id]),
        title=img.get_image_element_title(),
        image_id=img.id,
    )