Example #1
0
    def test_for_collection(self):
        # This collection has no mirror_integration, so
        # there is no MirrorUploader for it.
        collection = self._collection()
        assert None == MirrorUploader.for_collection(
            collection, ExternalIntegrationLink.COVERS)

        # This collection has a properly configured mirror_integration,
        # so it can have an MirrorUploader.
        integration = self._external_integration(
            ExternalIntegration.S3,
            ExternalIntegration.STORAGE_GOAL,
            username="******",
            password="******",
            settings={
                S3UploaderConfiguration.BOOK_COVERS_BUCKET_KEY: "some-covers"
            },
        )
        integration_link = self._external_integration_link(
            integration=collection._external_integration,
            other_integration=integration,
            purpose=ExternalIntegrationLink.COVERS,
        )

        uploader = MirrorUploader.for_collection(
            collection, ExternalIntegrationLink.COVERS)
        assert isinstance(uploader, MirrorUploader)
Example #2
0
    def test_integration_by_name(self):
        integration = self._integration

        # No name was passed so nothing is found
        with pytest.raises(CannotLoadConfiguration) as excinfo:
            MirrorUploader.integration_by_name(self._db)
        assert "No storage integration with name 'None' is configured" in str(
            excinfo.value)

        # Correct name was passed
        integration = MirrorUploader.integration_by_name(
            self._db, integration.name)
        assert isinstance(integration, ExternalIntegration)
Example #3
0
    def annotate_work_entry(self, work, active_license_pool, edition,
                            identifier, feed, entry):

        super(AdminAnnotator,
              self).annotate_work_entry(work, active_license_pool, edition,
                                        identifier, feed, entry)
        VerboseAnnotator.add_ratings(work, entry)

        # Find staff rating and add a tag for it.
        for measurement in identifier.measurements:
            if measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent:
                entry.append(
                    self.rating_tag(measurement.quantity_measured,
                                    measurement.value))

        feed.add_link_to_entry(
            entry,
            rel="http://librarysimplified.org/terms/rel/refresh",
            href=self.url_for("refresh",
                              identifier_type=identifier.type,
                              identifier=identifier.identifier,
                              _external=True))

        if active_license_pool and active_license_pool.suppressed:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/restore",
                href=self.url_for("unsuppress",
                                  identifier_type=identifier.type,
                                  identifier=identifier.identifier,
                                  _external=True))
        else:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/hide",
                href=self.url_for("suppress",
                                  identifier_type=identifier.type,
                                  identifier=identifier.identifier,
                                  _external=True))

        feed.add_link_to_entry(entry,
                               rel="edit",
                               href=self.url_for(
                                   "edit",
                                   identifier_type=identifier.type,
                                   identifier=identifier.identifier,
                                   _external=True))

        # If there is a storage integration for the collection, changing the cover is allowed.
        mirror = MirrorUploader.for_collection(active_license_pool.collection,
                                               use_sitewide=True)
        if mirror:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/change_cover",
                href=self.url_for("work_change_book_cover",
                                  identifier_type=identifier.type,
                                  identifier=identifier.identifier,
                                  _external=True))
Example #4
0
    def annotate_work_entry(self, work, active_license_pool, edition, identifier, feed, entry):

        super(AdminAnnotator, self).annotate_work_entry(work, active_license_pool, edition, identifier, feed, entry)
        VerboseAnnotator.add_ratings(work, entry)

        # Find staff rating and add a tag for it.
        for measurement in identifier.measurements:
            if measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent:
                entry.append(self.rating_tag(measurement.quantity_measured, measurement.value))

        feed.add_link_to_entry(
            entry,
            rel="http://librarysimplified.org/terms/rel/refresh",
            href=self.url_for(
                "refresh",
                identifier_type=identifier.type,
                identifier=identifier.identifier, _external=True)
        )

        if active_license_pool and active_license_pool.suppressed:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/restore",
                href=self.url_for(
                    "unsuppress",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier, _external=True)
            )
        else:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/hide",
                href=self.url_for(
                    "suppress",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier, _external=True)
            )

        feed.add_link_to_entry(
            entry,
            rel="edit",
            href=self.url_for(
                "edit",
                identifier_type=identifier.type,
                identifier=identifier.identifier, _external=True)
        )

        # If there is a storage integration for the collection, changing the cover is allowed.
        mirror = MirrorUploader.for_collection(active_license_pool.collection, use_sitewide=True)
        if mirror:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/change_cover",
                href=self.url_for(
                    "work_change_book_cover",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier, _external=True)
            )
Example #5
0
    def __init__(self,
                 collection,
                 mirror=None,
                 http_get=None,
                 viaf=None,
                 provide_coverage_immediately=False,
                 force=False,
                 provider_kwargs=None,
                 **kwargs):
        """Constructor.

        :param collection: Handle all Identifiers from this Collection
        that were previously registered with this CoverageProvider.

        :param mirror: A MirrorUploader to use if coverage requires
        uploading any cover images to external storage.

        :param http_get: A drop-in replacement for
        Representation.simple_http_get, to be used if any information
        (such as a book cover) needs to be obtained from the public
        Internet.

        :param viaf_client: A VIAFClient to use if coverage requires
        gathering information about authors from VIAF.

        :param force: Force CoverageProviders to cover identifiers
        even if they believe they have already done the work.

        :param provide_coverage_immediately: If this is True, then
        resolving an identifier means registering it with all of its
        other CoverageProviders *and then attempting to provide
        coverage*.  Registration is considered a success even if the
        other CoverageProviders fail, but the attempt must be made
        immediately.

        If this is False (the default), then resolving an identifier
        just means registering it with all other relevant
        CoverageProviders.

        :param provider_kwargs: Pass this object in as provider_kwargs
        when calling gather_providers at the end of the
        constructor. Used only in testing.

        """
        _db = Session.object_session(collection)

        # Since we are the metadata wrangler, any resources we find,
        # we mirror using the sitewide MirrorUploader.
        if not mirror:
            try:
                mirror = MirrorUploader.sitewide(_db)
            except CannotLoadConfiguration, e:
                logging.error(
                    "No storage integration is configured. Cover images will not be stored anywhere.",
                    exc_info=e)
Example #6
0
    def test_implementation_registry(self):
        # The implementation class used for a given ExternalIntegration
        # is controlled by the integration's protocol and the contents
        # of the MirrorUploader's implementation registry.
        MirrorUploader.IMPLEMENTATION_REGISTRY[
            "my protocol"] = DummyFailureUploader

        integration = self._integration
        uploader = MirrorUploader.mirror(self._db, integration=integration)
        assert isinstance(uploader, DummyFailureUploader)
        del MirrorUploader.IMPLEMENTATION_REGISTRY["my protocol"]
Example #7
0
    def test_mirror(self, name, protocol, uploader_class, settings=None):
        storage_name = "some storage"
        # If there's no integration with goal=STORAGE or name=storage_name,
        # MirrorUploader.mirror raises an exception.
        with pytest.raises(CannotLoadConfiguration) as excinfo:
            MirrorUploader.mirror(self._db, storage_name)
        assert "No storage integration with name 'some storage' is configured" in str(
            excinfo.value)

        # If there's only one, mirror() uses it to initialize a
        # MirrorUploader.
        integration = self._integration
        integration.protocol = protocol

        if settings:
            for key, value in settings.items():
                integration.setting(key).value = value

        uploader = MirrorUploader.mirror(self._db, integration=integration)

        assert isinstance(uploader, uploader_class)
Example #8
0
    def test_instantiation(self):
        integration = self._external_integration(
            ExternalIntegration.S3, goal=ExternalIntegration.STORAGE_GOAL)
        integration.username = "******"
        integration.password = "******"
        integration.setting(
            S3UploaderConfiguration.URL_TEMPLATE_KEY).value = "a transform"
        uploader = MirrorUploader.implementation(integration)
        assert True == isinstance(uploader, S3Uploader)

        # The URL_TEMPLATE_KEY setting becomes the .url_transform
        # attribute on the S3Uploader object.
        assert "a transform" == uploader.url_transform
 def _default_replacement_policy(self, _db):
     """In general, data used by the metadata wrangler is a reliable source
     of metadata but not of licensing information. We always
     provide the MirrorUploader in case a data source has cover
     images available.
     """
     try:
         mirror = MirrorUploader.sitewide(_db)
     except CannotLoadConfiguration, e:
         # It's not a problem if there's no MirrorUploader
         # configured -- it just means we can't mirror cover images
         # when they show up.
         mirror = None
 def _default_replacement_policy(self, _db):
     """In general, data used by the metadata wrangler is a reliable source
     of metadata but not of licensing information. We always
     provide the MirrorUploader in case a data source has cover
     images available.
     """
     try:
         mirror = MirrorUploader.sitewide(_db)
     except CannotLoadConfiguration, e:
         # It's not a problem if there's no MirrorUploader
         # configured -- it just means we can't mirror cover images
         # when they show up.
         mirror = None
    def __init__(self, collection, mirror=None, http_get=None, viaf=None,
                 provide_coverage_immediately=False, force=False,
                 provider_kwargs=None, **kwargs
    ):
        """Constructor.

        :param collection: Handle all Identifiers from this Collection
        that were previously registered with this CoverageProvider.

        :param mirror: A MirrorUploader to use if coverage requires
        uploading any cover images to external storage.

        :param http_get: A drop-in replacement for
        Representation.simple_http_get, to be used if any information
        (such as a book cover) needs to be obtained from the public
        Internet.

        :param viaf_client: A VIAFClient to use if coverage requires
        gathering information about authors from VIAF.

        :param force: Force CoverageProviders to cover identifiers
        even if they believe they have already done the work.

        :param provide_coverage_immediately: If this is True, then
        resolving an identifier means registering it with all of its
        other CoverageProviders *and then attempting to provide
        coverage*.  Registration is considered a success even if the
        other CoverageProviders fail, but the attempt must be made
        immediately.

        If this is False (the default), then resolving an identifier
        just means registering it with all other relevant
        CoverageProviders.

        :param provider_kwargs: Pass this object in as provider_kwargs
        when calling gather_providers at the end of the
        constructor. Used only in testing.

        """
        _db = Session.object_session(collection)

        # Since we are the metadata wrangler, any resources we find,
        # we mirror using the sitewide MirrorUploader.
        if not mirror:
            try:
                mirror = MirrorUploader.sitewide(_db)
            except CannotLoadConfiguration, e:
                logging.error(
                    "No storage integration is configured. Cover images will not be stored anywhere.",
                    exc_info=e
                )
    def __init__(self, collection, *args, **kwargs):
        _db = Session.object_session(collection)

        replacement_policy = kwargs.pop('replacement_policy', None)
        if not replacement_policy:
            mirror = MirrorUploader.sitewide(_db)
            replacement_policy = ReplacementPolicy(
                mirror=mirror, links=True
            )

        # Only process identifiers that have been registered for coverage.
        kwargs['registered_only'] = kwargs.get('registered_only', True)
        super(IntegrationClientCoverImageCoverageProvider, self).__init__(
            collection, *args, replacement_policy=replacement_policy, **kwargs
        )
    def __init__(self, collection, *args, **kwargs):
        _db = Session.object_session(collection)

        replacement_policy = kwargs.pop('replacement_policy', None)
        if not replacement_policy:
            mirror = MirrorUploader.sitewide(_db)
            replacement_policy = ReplacementPolicy(mirror=mirror, links=True)

        # Only process identifiers that have been registered for coverage.
        kwargs['registered_only'] = kwargs.get('registered_only', True)
        super(IntegrationClientCoverImageCoverageProvider,
              self).__init__(collection,
                             *args,
                             replacement_policy=replacement_policy,
                             **kwargs)
Example #14
0
    def change_book_cover(self, identifier_type, identifier, mirrors=None):
        """Save a new book cover based on the submitted form."""
        self.require_librarian(flask.request.library)

        data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF)

        work = self.load_work(flask.request.library, identifier_type,
                              identifier)
        if isinstance(work, ProblemDetail):
            return work

        rights_uri = flask.request.form.get("rights_status")
        rights_explanation = flask.request.form.get("rights_explanation")

        if not rights_uri:
            return INVALID_IMAGE.detailed(
                _("You must specify the image's license."))

        collection = self._get_collection_from_pools(identifier_type,
                                                     identifier)
        if isinstance(collection, ProblemDetail):
            return collection

        # Look for an appropriate mirror to store this cover image. Since the
        # mirror should be used for covers, we don't need a mirror for books.
        mirrors = mirrors or dict(covers_mirror=MirrorUploader.for_collection(
            collection, ExternalIntegrationLink.COVERS),
                                  books_mirror=None)
        if not mirrors.get(ExternalIntegrationLink.COVERS):
            return INVALID_CONFIGURATION_OPTION.detailed(
                _("Could not find a storage integration for uploading the cover."
                  ))

        image = self.generate_cover_image(work, identifier_type, identifier)
        if isinstance(image, ProblemDetail):
            return image

        original, derivation_settings, cover_href, cover_rights_explanation = self._original_cover_info(
            image, work, data_source, rights_uri, rights_explanation)

        buffer = StringIO()
        image.save(buffer, format="PNG")
        content = buffer.getvalue()

        if not cover_href:
            cover_href = Hyperlink.generic_uri(
                data_source,
                work.presentation_edition.primary_identifier,
                Hyperlink.IMAGE,
                content=content)

        cover_data = LinkData(
            Hyperlink.IMAGE,
            href=cover_href,
            media_type=Representation.PNG_MEDIA_TYPE,
            content=content,
            rights_uri=rights_uri,
            rights_explanation=cover_rights_explanation,
            original=original,
            transformation_settings=derivation_settings,
        )

        presentation_policy = PresentationCalculationPolicy(
            choose_edition=False,
            set_edition_metadata=False,
            classify=False,
            choose_summary=False,
            calculate_quality=False,
            choose_cover=True,
            regenerate_opds_entries=True,
            regenerate_marc_record=True,
            update_search_index=False,
        )

        replacement_policy = ReplacementPolicy(
            links=True,
            # link_content is false because we already have the content.
            # We don't want the metadata layer to try to fetch it again.
            link_content=False,
            mirrors=mirrors,
            presentation_calculation_policy=presentation_policy,
        )

        metadata = Metadata(data_source, links=[cover_data])
        metadata.apply(work.presentation_edition,
                       collection,
                       replace=replacement_policy)

        # metadata.apply only updates the edition, so we also need
        # to update the work.
        work.calculate_presentation(policy=presentation_policy)

        return Response(_("Success"), 200)
Example #15
0
    def annotate_work_entry(self, work, active_license_pool, edition,
                            identifier, feed, entry):

        super(AdminAnnotator,
              self).annotate_work_entry(work, active_license_pool, edition,
                                        identifier, feed, entry)
        VerboseAnnotator.add_ratings(work, entry)

        # Find staff rating and add a tag for it.
        for measurement in identifier.measurements:
            if (measurement.data_source.name == DataSource.LIBRARY_STAFF
                    and measurement.is_most_recent):
                entry.append(
                    self.rating_tag(measurement.quantity_measured,
                                    measurement.value))

        try:
            MetadataWranglerCollectionRegistrar(
                work.license_pools[0].collection)
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/refresh",
                href=self.url_for(
                    "refresh",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier,
                    _external=True,
                ),
            )
        except CannotLoadConfiguration:
            # Leave out the refresh link if there's no metadata wrangler
            # configured.
            pass

        if active_license_pool and active_license_pool.suppressed:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/restore",
                href=self.url_for(
                    "unsuppress",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier,
                    _external=True,
                ),
            )
        else:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/hide",
                href=self.url_for(
                    "suppress",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier,
                    _external=True,
                ),
            )

        feed.add_link_to_entry(
            entry,
            rel="edit",
            href=self.url_for(
                "edit",
                identifier_type=identifier.type,
                identifier=identifier.identifier,
                _external=True,
            ),
        )

        # If there is a storage integration for the collection, changing the cover is allowed.
        mirror = MirrorUploader.for_collection(active_license_pool.collection,
                                               ExternalIntegrationLink.COVERS)
        if mirror:
            feed.add_link_to_entry(
                entry,
                rel="http://librarysimplified.org/terms/rel/change_cover",
                href=self.url_for(
                    "work_change_book_cover",
                    identifier_type=identifier.type,
                    identifier=identifier.identifier,
                    _external=True,
                ),
            )