def test_for_collection(self):
# This collection has no mirror_integration, so
# there is no MirrorUploader for it.
collection = self._collection()
assert None == MirrorUploader.for_collection(
collection, ExternalIntegrationLink.COVERS)
# This collection has a properly configured mirror_integration,
# so it can have an MirrorUploader.
integration = self._external_integration(
ExternalIntegration.S3,
ExternalIntegration.STORAGE_GOAL,
username="******",
password="******",
settings={
S3UploaderConfiguration.BOOK_COVERS_BUCKET_KEY: "some-covers"
},
)
integration_link = self._external_integration_link(
integration=collection._external_integration,
other_integration=integration,
purpose=ExternalIntegrationLink.COVERS,
)
uploader = MirrorUploader.for_collection(
collection, ExternalIntegrationLink.COVERS)
assert isinstance(uploader, MirrorUploader)
def test_integration_by_name(self):
integration = self._integration
# No name was passed so nothing is found
with pytest.raises(CannotLoadConfiguration) as excinfo:
MirrorUploader.integration_by_name(self._db)
assert "No storage integration with name 'None' is configured" in str(
excinfo.value)
# Correct name was passed
integration = MirrorUploader.integration_by_name(
self._db, integration.name)
assert isinstance(integration, ExternalIntegration)
def annotate_work_entry(self, work, active_license_pool, edition,
identifier, feed, entry):
super(AdminAnnotator,
self).annotate_work_entry(work, active_license_pool, edition,
identifier, feed, entry)
VerboseAnnotator.add_ratings(work, entry)
# Find staff rating and add a tag for it.
for measurement in identifier.measurements:
if measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent:
entry.append(
self.rating_tag(measurement.quantity_measured,
measurement.value))
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/refresh",
href=self.url_for("refresh",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True))
if active_license_pool and active_license_pool.suppressed:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/restore",
href=self.url_for("unsuppress",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True))
else:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/hide",
href=self.url_for("suppress",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True))
feed.add_link_to_entry(entry,
rel="edit",
href=self.url_for(
"edit",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True))
# If there is a storage integration for the collection, changing the cover is allowed.
mirror = MirrorUploader.for_collection(active_license_pool.collection,
use_sitewide=True)
if mirror:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/change_cover",
href=self.url_for("work_change_book_cover",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True))
def annotate_work_entry(self, work, active_license_pool, edition, identifier, feed, entry):
super(AdminAnnotator, self).annotate_work_entry(work, active_license_pool, edition, identifier, feed, entry)
VerboseAnnotator.add_ratings(work, entry)
# Find staff rating and add a tag for it.
for measurement in identifier.measurements:
if measurement.data_source.name == DataSource.LIBRARY_STAFF and measurement.is_most_recent:
entry.append(self.rating_tag(measurement.quantity_measured, measurement.value))
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/refresh",
href=self.url_for(
"refresh",
identifier_type=identifier.type,
identifier=identifier.identifier, _external=True)
)
if active_license_pool and active_license_pool.suppressed:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/restore",
href=self.url_for(
"unsuppress",
identifier_type=identifier.type,
identifier=identifier.identifier, _external=True)
)
else:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/hide",
href=self.url_for(
"suppress",
identifier_type=identifier.type,
identifier=identifier.identifier, _external=True)
)
feed.add_link_to_entry(
entry,
rel="edit",
href=self.url_for(
"edit",
identifier_type=identifier.type,
identifier=identifier.identifier, _external=True)
)
# If there is a storage integration for the collection, changing the cover is allowed.
mirror = MirrorUploader.for_collection(active_license_pool.collection, use_sitewide=True)
if mirror:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/change_cover",
href=self.url_for(
"work_change_book_cover",
identifier_type=identifier.type,
identifier=identifier.identifier, _external=True)
)
def __init__(self,
collection,
mirror=None,
http_get=None,
viaf=None,
provide_coverage_immediately=False,
force=False,
provider_kwargs=None,
**kwargs):
"""Constructor.
:param collection: Handle all Identifiers from this Collection
that were previously registered with this CoverageProvider.
:param mirror: A MirrorUploader to use if coverage requires
uploading any cover images to external storage.
:param http_get: A drop-in replacement for
Representation.simple_http_get, to be used if any information
(such as a book cover) needs to be obtained from the public
Internet.
:param viaf_client: A VIAFClient to use if coverage requires
gathering information about authors from VIAF.
:param force: Force CoverageProviders to cover identifiers
even if they believe they have already done the work.
:param provide_coverage_immediately: If this is True, then
resolving an identifier means registering it with all of its
other CoverageProviders *and then attempting to provide
coverage*. Registration is considered a success even if the
other CoverageProviders fail, but the attempt must be made
immediately.
If this is False (the default), then resolving an identifier
just means registering it with all other relevant
CoverageProviders.
:param provider_kwargs: Pass this object in as provider_kwargs
when calling gather_providers at the end of the
constructor. Used only in testing.
"""
_db = Session.object_session(collection)
# Since we are the metadata wrangler, any resources we find,
# we mirror using the sitewide MirrorUploader.
if not mirror:
try:
mirror = MirrorUploader.sitewide(_db)
except CannotLoadConfiguration, e:
logging.error(
"No storage integration is configured. Cover images will not be stored anywhere.",
exc_info=e)
def test_implementation_registry(self):
# The implementation class used for a given ExternalIntegration
# is controlled by the integration's protocol and the contents
# of the MirrorUploader's implementation registry.
MirrorUploader.IMPLEMENTATION_REGISTRY[
"my protocol"] = DummyFailureUploader
integration = self._integration
uploader = MirrorUploader.mirror(self._db, integration=integration)
assert isinstance(uploader, DummyFailureUploader)
del MirrorUploader.IMPLEMENTATION_REGISTRY["my protocol"]
def test_mirror(self, name, protocol, uploader_class, settings=None):
storage_name = "some storage"
# If there's no integration with goal=STORAGE or name=storage_name,
# MirrorUploader.mirror raises an exception.
with pytest.raises(CannotLoadConfiguration) as excinfo:
MirrorUploader.mirror(self._db, storage_name)
assert "No storage integration with name 'some storage' is configured" in str(
excinfo.value)
# If there's only one, mirror() uses it to initialize a
# MirrorUploader.
integration = self._integration
integration.protocol = protocol
if settings:
for key, value in settings.items():
integration.setting(key).value = value
uploader = MirrorUploader.mirror(self._db, integration=integration)
assert isinstance(uploader, uploader_class)
def test_instantiation(self):
integration = self._external_integration(
ExternalIntegration.S3, goal=ExternalIntegration.STORAGE_GOAL)
integration.username = "******"
integration.password = "******"
integration.setting(
S3UploaderConfiguration.URL_TEMPLATE_KEY).value = "a transform"
uploader = MirrorUploader.implementation(integration)
assert True == isinstance(uploader, S3Uploader)
# The URL_TEMPLATE_KEY setting becomes the .url_transform
# attribute on the S3Uploader object.
assert "a transform" == uploader.url_transform
def _default_replacement_policy(self, _db):
"""In general, data used by the metadata wrangler is a reliable source
of metadata but not of licensing information. We always
provide the MirrorUploader in case a data source has cover
images available.
"""
try:
mirror = MirrorUploader.sitewide(_db)
except CannotLoadConfiguration, e:
# It's not a problem if there's no MirrorUploader
# configured -- it just means we can't mirror cover images
# when they show up.
mirror = None
def _default_replacement_policy(self, _db):
"""In general, data used by the metadata wrangler is a reliable source
of metadata but not of licensing information. We always
provide the MirrorUploader in case a data source has cover
images available.
"""
try:
mirror = MirrorUploader.sitewide(_db)
except CannotLoadConfiguration, e:
# It's not a problem if there's no MirrorUploader
# configured -- it just means we can't mirror cover images
# when they show up.
mirror = None
def __init__(self, collection, mirror=None, http_get=None, viaf=None,
provide_coverage_immediately=False, force=False,
provider_kwargs=None, **kwargs
):
"""Constructor.
:param collection: Handle all Identifiers from this Collection
that were previously registered with this CoverageProvider.
:param mirror: A MirrorUploader to use if coverage requires
uploading any cover images to external storage.
:param http_get: A drop-in replacement for
Representation.simple_http_get, to be used if any information
(such as a book cover) needs to be obtained from the public
Internet.
:param viaf_client: A VIAFClient to use if coverage requires
gathering information about authors from VIAF.
:param force: Force CoverageProviders to cover identifiers
even if they believe they have already done the work.
:param provide_coverage_immediately: If this is True, then
resolving an identifier means registering it with all of its
other CoverageProviders *and then attempting to provide
coverage*. Registration is considered a success even if the
other CoverageProviders fail, but the attempt must be made
immediately.
If this is False (the default), then resolving an identifier
just means registering it with all other relevant
CoverageProviders.
:param provider_kwargs: Pass this object in as provider_kwargs
when calling gather_providers at the end of the
constructor. Used only in testing.
"""
_db = Session.object_session(collection)
# Since we are the metadata wrangler, any resources we find,
# we mirror using the sitewide MirrorUploader.
if not mirror:
try:
mirror = MirrorUploader.sitewide(_db)
except CannotLoadConfiguration, e:
logging.error(
"No storage integration is configured. Cover images will not be stored anywhere.",
exc_info=e
)
def __init__(self, collection, *args, **kwargs):
_db = Session.object_session(collection)
replacement_policy = kwargs.pop('replacement_policy', None)
if not replacement_policy:
mirror = MirrorUploader.sitewide(_db)
replacement_policy = ReplacementPolicy(
mirror=mirror, links=True
)
# Only process identifiers that have been registered for coverage.
kwargs['registered_only'] = kwargs.get('registered_only', True)
super(IntegrationClientCoverImageCoverageProvider, self).__init__(
collection, *args, replacement_policy=replacement_policy, **kwargs
)
def __init__(self, collection, *args, **kwargs):
_db = Session.object_session(collection)
replacement_policy = kwargs.pop('replacement_policy', None)
if not replacement_policy:
mirror = MirrorUploader.sitewide(_db)
replacement_policy = ReplacementPolicy(mirror=mirror, links=True)
# Only process identifiers that have been registered for coverage.
kwargs['registered_only'] = kwargs.get('registered_only', True)
super(IntegrationClientCoverImageCoverageProvider,
self).__init__(collection,
*args,
replacement_policy=replacement_policy,
**kwargs)
def change_book_cover(self, identifier_type, identifier, mirrors=None):
"""Save a new book cover based on the submitted form."""
self.require_librarian(flask.request.library)
data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF)
work = self.load_work(flask.request.library, identifier_type,
identifier)
if isinstance(work, ProblemDetail):
return work
rights_uri = flask.request.form.get("rights_status")
rights_explanation = flask.request.form.get("rights_explanation")
if not rights_uri:
return INVALID_IMAGE.detailed(
_("You must specify the image's license."))
collection = self._get_collection_from_pools(identifier_type,
identifier)
if isinstance(collection, ProblemDetail):
return collection
# Look for an appropriate mirror to store this cover image. Since the
# mirror should be used for covers, we don't need a mirror for books.
mirrors = mirrors or dict(covers_mirror=MirrorUploader.for_collection(
collection, ExternalIntegrationLink.COVERS),
books_mirror=None)
if not mirrors.get(ExternalIntegrationLink.COVERS):
return INVALID_CONFIGURATION_OPTION.detailed(
_("Could not find a storage integration for uploading the cover."
))
image = self.generate_cover_image(work, identifier_type, identifier)
if isinstance(image, ProblemDetail):
return image
original, derivation_settings, cover_href, cover_rights_explanation = self._original_cover_info(
image, work, data_source, rights_uri, rights_explanation)
buffer = StringIO()
image.save(buffer, format="PNG")
content = buffer.getvalue()
if not cover_href:
cover_href = Hyperlink.generic_uri(
data_source,
work.presentation_edition.primary_identifier,
Hyperlink.IMAGE,
content=content)
cover_data = LinkData(
Hyperlink.IMAGE,
href=cover_href,
media_type=Representation.PNG_MEDIA_TYPE,
content=content,
rights_uri=rights_uri,
rights_explanation=cover_rights_explanation,
original=original,
transformation_settings=derivation_settings,
)
presentation_policy = PresentationCalculationPolicy(
choose_edition=False,
set_edition_metadata=False,
classify=False,
choose_summary=False,
calculate_quality=False,
choose_cover=True,
regenerate_opds_entries=True,
regenerate_marc_record=True,
update_search_index=False,
)
replacement_policy = ReplacementPolicy(
links=True,
# link_content is false because we already have the content.
# We don't want the metadata layer to try to fetch it again.
link_content=False,
mirrors=mirrors,
presentation_calculation_policy=presentation_policy,
)
metadata = Metadata(data_source, links=[cover_data])
metadata.apply(work.presentation_edition,
collection,
replace=replacement_policy)
# metadata.apply only updates the edition, so we also need
# to update the work.
work.calculate_presentation(policy=presentation_policy)
return Response(_("Success"), 200)
def annotate_work_entry(self, work, active_license_pool, edition,
identifier, feed, entry):
super(AdminAnnotator,
self).annotate_work_entry(work, active_license_pool, edition,
identifier, feed, entry)
VerboseAnnotator.add_ratings(work, entry)
# Find staff rating and add a tag for it.
for measurement in identifier.measurements:
if (measurement.data_source.name == DataSource.LIBRARY_STAFF
and measurement.is_most_recent):
entry.append(
self.rating_tag(measurement.quantity_measured,
measurement.value))
try:
MetadataWranglerCollectionRegistrar(
work.license_pools[0].collection)
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/refresh",
href=self.url_for(
"refresh",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True,
),
)
except CannotLoadConfiguration:
# Leave out the refresh link if there's no metadata wrangler
# configured.
pass
if active_license_pool and active_license_pool.suppressed:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/restore",
href=self.url_for(
"unsuppress",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True,
),
)
else:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/hide",
href=self.url_for(
"suppress",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True,
),
)
feed.add_link_to_entry(
entry,
rel="edit",
href=self.url_for(
"edit",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True,
),
)
# If there is a storage integration for the collection, changing the cover is allowed.
mirror = MirrorUploader.for_collection(active_license_pool.collection,
ExternalIntegrationLink.COVERS)
if mirror:
feed.add_link_to_entry(
entry,
rel="http://librarysimplified.org/terms/rel/change_cover",
href=self.url_for(
"work_change_book_cover",
identifier_type=identifier.type,
identifier=identifier.identifier,
_external=True,
),
)