def test_for_collection(self): # This collection has no mirror_integration, so # there is no MirrorUploader for it. collection = self._collection() assert None == MirrorUploader.for_collection( collection, ExternalIntegrationLink.COVERS) # This collection has a properly configured mirror_integration, # so it can have an MirrorUploader. integration = self._external_integration( ExternalIntegration.S3, ExternalIntegration.STORAGE_GOAL, username="******", password="******", settings={ S3UploaderConfiguration.BOOK_COVERS_BUCKET_KEY: "some-covers" }, ) integration_link = self._external_integration_link( integration=collection._external_integration, other_integration=integration, purpose=ExternalIntegrationLink.COVERS, ) uploader = MirrorUploader.for_collection( collection, ExternalIntegrationLink.COVERS) assert isinstance(uploader, MirrorUploader)
def test_integration_by_name(self): integration = self._integration # No name was passed so nothing is found with pytest.raises(CannotLoadConfiguration) as excinfo: MirrorUploader.integration_by_name(self._db) assert "No storage integration with name 'None' is configured" in str( excinfo.value) # Correct name was passed integration = MirrorUploader.integration_by_name( self._db, integration.name) assert isinstance(integration, ExternalIntegration)
def annotate_work_entry(self, work, active_license_pool, edition, identifier, feed, entry): super(AdminAnnotator, self).annotate_work_entry(work, active_license_pool, edition, identifier, feed, entry) VerboseAnnotator.add_ratings(work, entry) # Find staff rating and add a tag for it. for measurement in identifier.measurements: if measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent: entry.append( self.rating_tag(measurement.quantity_measured, measurement.value)) feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/refresh", href=self.url_for("refresh", identifier_type=identifier.type, identifier=identifier.identifier, _external=True)) if active_license_pool and active_license_pool.suppressed: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/restore", href=self.url_for("unsuppress", identifier_type=identifier.type, identifier=identifier.identifier, _external=True)) else: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/hide", href=self.url_for("suppress", identifier_type=identifier.type, identifier=identifier.identifier, _external=True)) feed.add_link_to_entry(entry, rel="edit", href=self.url_for( "edit", identifier_type=identifier.type, identifier=identifier.identifier, _external=True)) # If there is a storage integration for the collection, changing the cover is allowed. mirror = MirrorUploader.for_collection(active_license_pool.collection, use_sitewide=True) if mirror: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/change_cover", href=self.url_for("work_change_book_cover", identifier_type=identifier.type, identifier=identifier.identifier, _external=True))
def annotate_work_entry(self, work, active_license_pool, edition, identifier, feed, entry): super(AdminAnnotator, self).annotate_work_entry(work, active_license_pool, edition, identifier, feed, entry) VerboseAnnotator.add_ratings(work, entry) # Find staff rating and add a tag for it. for measurement in identifier.measurements: if measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent: entry.append(self.rating_tag(measurement.quantity_measured, measurement.value)) feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/refresh", href=self.url_for( "refresh", identifier_type=identifier.type, identifier=identifier.identifier, _external=True) ) if active_license_pool and active_license_pool.suppressed: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/restore", href=self.url_for( "unsuppress", identifier_type=identifier.type, identifier=identifier.identifier, _external=True) ) else: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/hide", href=self.url_for( "suppress", identifier_type=identifier.type, identifier=identifier.identifier, _external=True) ) feed.add_link_to_entry( entry, rel="edit", href=self.url_for( "edit", identifier_type=identifier.type, identifier=identifier.identifier, _external=True) ) # If there is a storage integration for the collection, changing the cover is allowed. mirror = MirrorUploader.for_collection(active_license_pool.collection, use_sitewide=True) if mirror: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/change_cover", href=self.url_for( "work_change_book_cover", identifier_type=identifier.type, identifier=identifier.identifier, _external=True) )
def __init__(self, collection, mirror=None, http_get=None, viaf=None, provide_coverage_immediately=False, force=False, provider_kwargs=None, **kwargs): """Constructor. :param collection: Handle all Identifiers from this Collection that were previously registered with this CoverageProvider. :param mirror: A MirrorUploader to use if coverage requires uploading any cover images to external storage. :param http_get: A drop-in replacement for Representation.simple_http_get, to be used if any information (such as a book cover) needs to be obtained from the public Internet. :param viaf_client: A VIAFClient to use if coverage requires gathering information about authors from VIAF. :param force: Force CoverageProviders to cover identifiers even if they believe they have already done the work. :param provide_coverage_immediately: If this is True, then resolving an identifier means registering it with all of its other CoverageProviders *and then attempting to provide coverage*. Registration is considered a success even if the other CoverageProviders fail, but the attempt must be made immediately. If this is False (the default), then resolving an identifier just means registering it with all other relevant CoverageProviders. :param provider_kwargs: Pass this object in as provider_kwargs when calling gather_providers at the end of the constructor. Used only in testing. """ _db = Session.object_session(collection) # Since we are the metadata wrangler, any resources we find, # we mirror using the sitewide MirrorUploader. if not mirror: try: mirror = MirrorUploader.sitewide(_db) except CannotLoadConfiguration, e: logging.error( "No storage integration is configured. Cover images will not be stored anywhere.", exc_info=e)
def test_implementation_registry(self): # The implementation class used for a given ExternalIntegration # is controlled by the integration's protocol and the contents # of the MirrorUploader's implementation registry. MirrorUploader.IMPLEMENTATION_REGISTRY[ "my protocol"] = DummyFailureUploader integration = self._integration uploader = MirrorUploader.mirror(self._db, integration=integration) assert isinstance(uploader, DummyFailureUploader) del MirrorUploader.IMPLEMENTATION_REGISTRY["my protocol"]
def test_mirror(self, name, protocol, uploader_class, settings=None): storage_name = "some storage" # If there's no integration with goal=STORAGE or name=storage_name, # MirrorUploader.mirror raises an exception. with pytest.raises(CannotLoadConfiguration) as excinfo: MirrorUploader.mirror(self._db, storage_name) assert "No storage integration with name 'some storage' is configured" in str( excinfo.value) # If there's only one, mirror() uses it to initialize a # MirrorUploader. integration = self._integration integration.protocol = protocol if settings: for key, value in settings.items(): integration.setting(key).value = value uploader = MirrorUploader.mirror(self._db, integration=integration) assert isinstance(uploader, uploader_class)
def test_instantiation(self): integration = self._external_integration( ExternalIntegration.S3, goal=ExternalIntegration.STORAGE_GOAL) integration.username = "******" integration.password = "******" integration.setting( S3UploaderConfiguration.URL_TEMPLATE_KEY).value = "a transform" uploader = MirrorUploader.implementation(integration) assert True == isinstance(uploader, S3Uploader) # The URL_TEMPLATE_KEY setting becomes the .url_transform # attribute on the S3Uploader object. assert "a transform" == uploader.url_transform
def _default_replacement_policy(self, _db): """In general, data used by the metadata wrangler is a reliable source of metadata but not of licensing information. We always provide the MirrorUploader in case a data source has cover images available. """ try: mirror = MirrorUploader.sitewide(_db) except CannotLoadConfiguration, e: # It's not a problem if there's no MirrorUploader # configured -- it just means we can't mirror cover images # when they show up. mirror = None
def __init__(self, collection, mirror=None, http_get=None, viaf=None, provide_coverage_immediately=False, force=False, provider_kwargs=None, **kwargs ): """Constructor. :param collection: Handle all Identifiers from this Collection that were previously registered with this CoverageProvider. :param mirror: A MirrorUploader to use if coverage requires uploading any cover images to external storage. :param http_get: A drop-in replacement for Representation.simple_http_get, to be used if any information (such as a book cover) needs to be obtained from the public Internet. :param viaf_client: A VIAFClient to use if coverage requires gathering information about authors from VIAF. :param force: Force CoverageProviders to cover identifiers even if they believe they have already done the work. :param provide_coverage_immediately: If this is True, then resolving an identifier means registering it with all of its other CoverageProviders *and then attempting to provide coverage*. Registration is considered a success even if the other CoverageProviders fail, but the attempt must be made immediately. If this is False (the default), then resolving an identifier just means registering it with all other relevant CoverageProviders. :param provider_kwargs: Pass this object in as provider_kwargs when calling gather_providers at the end of the constructor. Used only in testing. """ _db = Session.object_session(collection) # Since we are the metadata wrangler, any resources we find, # we mirror using the sitewide MirrorUploader. if not mirror: try: mirror = MirrorUploader.sitewide(_db) except CannotLoadConfiguration, e: logging.error( "No storage integration is configured. Cover images will not be stored anywhere.", exc_info=e )
def __init__(self, collection, *args, **kwargs): _db = Session.object_session(collection) replacement_policy = kwargs.pop('replacement_policy', None) if not replacement_policy: mirror = MirrorUploader.sitewide(_db) replacement_policy = ReplacementPolicy( mirror=mirror, links=True ) # Only process identifiers that have been registered for coverage. kwargs['registered_only'] = kwargs.get('registered_only', True) super(IntegrationClientCoverImageCoverageProvider, self).__init__( collection, *args, replacement_policy=replacement_policy, **kwargs )
def __init__(self, collection, *args, **kwargs): _db = Session.object_session(collection) replacement_policy = kwargs.pop('replacement_policy', None) if not replacement_policy: mirror = MirrorUploader.sitewide(_db) replacement_policy = ReplacementPolicy(mirror=mirror, links=True) # Only process identifiers that have been registered for coverage. kwargs['registered_only'] = kwargs.get('registered_only', True) super(IntegrationClientCoverImageCoverageProvider, self).__init__(collection, *args, replacement_policy=replacement_policy, **kwargs)
def change_book_cover(self, identifier_type, identifier, mirrors=None): """Save a new book cover based on the submitted form.""" self.require_librarian(flask.request.library) data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF) work = self.load_work(flask.request.library, identifier_type, identifier) if isinstance(work, ProblemDetail): return work rights_uri = flask.request.form.get("rights_status") rights_explanation = flask.request.form.get("rights_explanation") if not rights_uri: return INVALID_IMAGE.detailed( _("You must specify the image's license.")) collection = self._get_collection_from_pools(identifier_type, identifier) if isinstance(collection, ProblemDetail): return collection # Look for an appropriate mirror to store this cover image. Since the # mirror should be used for covers, we don't need a mirror for books. mirrors = mirrors or dict(covers_mirror=MirrorUploader.for_collection( collection, ExternalIntegrationLink.COVERS), books_mirror=None) if not mirrors.get(ExternalIntegrationLink.COVERS): return INVALID_CONFIGURATION_OPTION.detailed( _("Could not find a storage integration for uploading the cover." )) image = self.generate_cover_image(work, identifier_type, identifier) if isinstance(image, ProblemDetail): return image original, derivation_settings, cover_href, cover_rights_explanation = self._original_cover_info( image, work, data_source, rights_uri, rights_explanation) buffer = StringIO() image.save(buffer, format="PNG") content = buffer.getvalue() if not cover_href: cover_href = Hyperlink.generic_uri( data_source, work.presentation_edition.primary_identifier, Hyperlink.IMAGE, content=content) cover_data = LinkData( Hyperlink.IMAGE, href=cover_href, media_type=Representation.PNG_MEDIA_TYPE, content=content, rights_uri=rights_uri, rights_explanation=cover_rights_explanation, original=original, transformation_settings=derivation_settings, ) presentation_policy = PresentationCalculationPolicy( choose_edition=False, set_edition_metadata=False, classify=False, choose_summary=False, calculate_quality=False, choose_cover=True, regenerate_opds_entries=True, regenerate_marc_record=True, update_search_index=False, ) replacement_policy = ReplacementPolicy( links=True, # link_content is false because we already have the content. # We don't want the metadata layer to try to fetch it again. link_content=False, mirrors=mirrors, presentation_calculation_policy=presentation_policy, ) metadata = Metadata(data_source, links=[cover_data]) metadata.apply(work.presentation_edition, collection, replace=replacement_policy) # metadata.apply only updates the edition, so we also need # to update the work. work.calculate_presentation(policy=presentation_policy) return Response(_("Success"), 200)
def annotate_work_entry(self, work, active_license_pool, edition, identifier, feed, entry): super(AdminAnnotator, self).annotate_work_entry(work, active_license_pool, edition, identifier, feed, entry) VerboseAnnotator.add_ratings(work, entry) # Find staff rating and add a tag for it. for measurement in identifier.measurements: if (measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent): entry.append( self.rating_tag(measurement.quantity_measured, measurement.value)) try: MetadataWranglerCollectionRegistrar( work.license_pools[0].collection) feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/refresh", href=self.url_for( "refresh", identifier_type=identifier.type, identifier=identifier.identifier, _external=True, ), ) except CannotLoadConfiguration: # Leave out the refresh link if there's no metadata wrangler # configured. pass if active_license_pool and active_license_pool.suppressed: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/restore", href=self.url_for( "unsuppress", identifier_type=identifier.type, identifier=identifier.identifier, _external=True, ), ) else: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/hide", href=self.url_for( "suppress", identifier_type=identifier.type, identifier=identifier.identifier, _external=True, ), ) feed.add_link_to_entry( entry, rel="edit", href=self.url_for( "edit", identifier_type=identifier.type, identifier=identifier.identifier, _external=True, ), ) # If there is a storage integration for the collection, changing the cover is allowed. mirror = MirrorUploader.for_collection(active_license_pool.collection, ExternalIntegrationLink.COVERS) if mirror: feed.add_link_to_entry( entry, rel="http://librarysimplified.org/terms/rel/change_cover", href=self.url_for( "work_change_book_cover", identifier_type=identifier.type, identifier=identifier.identifier, _external=True, ), )