def test_apply_removes_old_formats_based_on_replacement_policy(self):
edition, pool = self._edition(with_license_pool=True)
# Start with one delivery mechanism for this pool.
for lpdm in pool.delivery_mechanisms:
self._db.delete(lpdm)
old_lpdm = pool.set_delivery_mechanism(
Representation.PDF_MEDIA_TYPE,
DeliveryMechanism.ADOBE_DRM,
RightsStatus.IN_COPYRIGHT,
None,
)
# And it has been loaned.
patron = self._patron()
loan, ignore = pool.loan_to(patron, fulfillment=old_lpdm)
assert old_lpdm == loan.fulfillment
# We have new circulation data that has a different format.
format = FormatData(
content_type=Representation.EPUB_MEDIA_TYPE,
drm_scheme=DeliveryMechanism.ADOBE_DRM,
)
circulation_data = CirculationData(
formats=[format],
data_source=edition.data_source,
primary_identifier=edition.primary_identifier,
)
# If we apply the new CirculationData with formats false in the policy,
# we'll add the new format, but keep the old one as well.
replacement_policy = ReplacementPolicy(formats=False)
circulation_data.apply(self._db, pool.collection, replacement_policy)
assert 2 == len(pool.delivery_mechanisms)
assert set(
[Representation.PDF_MEDIA_TYPE,
Representation.EPUB_MEDIA_TYPE]) == set([
lpdm.delivery_mechanism.content_type
for lpdm in pool.delivery_mechanisms
])
assert old_lpdm == loan.fulfillment
# But if we make formats true in the policy, we'll delete the old format
# and remove it from its loan.
replacement_policy = ReplacementPolicy(formats=True)
circulation_data.apply(self._db, pool.collection, replacement_policy)
assert 1 == len(pool.delivery_mechanisms)
assert (Representation.EPUB_MEDIA_TYPE ==
pool.delivery_mechanisms[0].delivery_mechanism.content_type)
assert None == loan.fulfillment
def test_rights_status_open_access_link_with_rights(self):
identifier = IdentifierData(
Identifier.OVERDRIVE_ID,
"abcd",
)
link = LinkData(
rel=Hyperlink.OPEN_ACCESS_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
rights_uri=RightsStatus.CC_BY_ND,
)
circulation_data = CirculationData(
data_source=DataSource.OVERDRIVE,
primary_identifier=identifier,
links=[link],
)
replace = ReplacementPolicy(formats=True, )
pool, ignore = circulation_data.license_pool(self._db,
self._default_collection)
circulation_data.apply(self._db, pool.collection, replace)
assert True == pool.open_access
assert 1 == len(pool.delivery_mechanisms)
assert RightsStatus.CC_BY_ND == pool.delivery_mechanisms[
0].rights_status.uri
def test_rights_status_commercial_link_with_rights(self):
identifier = IdentifierData(
Identifier.OVERDRIVE_ID,
"abcd",
)
link = LinkData(
rel=Hyperlink.DRM_ENCRYPTED_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
rights_uri=RightsStatus.IN_COPYRIGHT,
)
format = FormatData(
content_type=link.media_type,
drm_scheme=DeliveryMechanism.ADOBE_DRM,
link=link,
rights_uri=RightsStatus.IN_COPYRIGHT,
)
circulation_data = CirculationData(
data_source=DataSource.OVERDRIVE,
primary_identifier=identifier,
links=[link],
formats=[format],
)
replace = ReplacementPolicy(formats=True, )
pool, ignore = circulation_data.license_pool(self._db,
self._default_collection)
circulation_data.apply(self._db, pool.collection, replace)
assert False == pool.open_access
assert 1 == len(pool.delivery_mechanisms)
assert (RightsStatus.IN_COPYRIGHT ==
pool.delivery_mechanisms[0].rights_status.uri)
def process_book(self, bibliographic):
"""Make the local database reflect the state of the remote Enki
collection for the given book.
:param bibliographic: A Metadata object with attached CirculationData
:return: A 2-tuple (LicensePool, Edition). If possible, a
presentation-ready Work will be created for the LicensePool.
"""
availability = bibliographic.circulation
edition, new_edition = bibliographic.edition(self._db)
now = utc_now()
policy = ReplacementPolicy(
identifiers=False,
subjects=True,
contributions=True,
formats=True,
)
bibliographic.apply(edition, self.collection, replace=policy)
license_pool, ignore = availability.license_pool(self._db, self.collection)
if new_edition:
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, now
)
return edition, license_pool
def test_rights_status_default_rights_from_data_source(self):
identifier = IdentifierData(
Identifier.GUTENBERG_ID,
"abcd",
)
link = LinkData(
rel=Hyperlink.DRM_ENCRYPTED_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
)
circulation_data = CirculationData(
data_source=DataSource.OA_CONTENT_SERVER,
primary_identifier=identifier,
links=[link],
)
replace = ReplacementPolicy(formats=True, )
# This pool starts off as not being open-access.
pool, ignore = circulation_data.license_pool(self._db,
self._default_collection)
assert False == pool.open_access
circulation_data.apply(self._db, pool.collection, replace)
# The pool became open-access because it was given a
# link that came from the OS content server.
assert True == pool.open_access
assert 1 == len(pool.delivery_mechanisms)
# The rights status is the default for the OA content server.
assert (RightsStatus.GENERIC_OPEN_ACCESS ==
pool.delivery_mechanisms[0].rights_status.uri)
def process_book(self, bibliographic, availability):
analytics = Analytics(self._db)
license_pool, new_license_pool = availability.license_pool(
self._db, self.collection, analytics)
edition, new_edition = bibliographic.edition(self._db)
license_pool.edition = edition
policy = ReplacementPolicy(
identifiers=False,
subjects=True,
contributions=True,
formats=True,
analytics=analytics,
)
availability.apply(self._db, self.collection, replace=policy)
if new_edition:
bibliographic.apply(edition, self.collection, replace=policy)
if new_license_pool or new_edition:
# At this point we have done work equivalent to that done by
# the Axis360BibliographicCoverageProvider. Register that the
# work has been done so we don't have to do it again.
identifier = edition.primary_identifier
self.bibliographic_coverage_provider.handle_success(identifier)
self.bibliographic_coverage_provider.add_coverage_record_for(
identifier)
return edition, license_pool
def test_explicit_formatdata(self):
# Creating an edition with an open-access download will
# automatically create a delivery mechanism.
edition, pool = self._edition(with_open_access_download=True)
# Let's also add a DRM format.
drm_format = FormatData(
content_type=Representation.PDF_MEDIA_TYPE,
drm_scheme=DeliveryMechanism.ADOBE_DRM,
)
circulation_data = CirculationData(
formats=[drm_format],
data_source=edition.data_source,
primary_identifier=edition.primary_identifier,
)
circulation_data.apply(self._db, pool.collection)
[epub, pdf] = sorted(pool.delivery_mechanisms,
key=lambda x: x.delivery_mechanism.content_type)
assert epub.resource == pool.best_open_access_resource
assert Representation.PDF_MEDIA_TYPE == pdf.delivery_mechanism.content_type
assert DeliveryMechanism.ADOBE_DRM == pdf.delivery_mechanism.drm_scheme
# If we tell Metadata to replace the list of formats, we only
# have the one format we manually created.
replace = ReplacementPolicy(formats=True, )
circulation_data.apply(self._db, pool.collection, replace=replace)
[pdf] = pool.delivery_mechanisms
assert Representation.PDF_MEDIA_TYPE == pdf.delivery_mechanism.content_type
def __init__(self, collection, api_class=OdiloAPI, **kwargs):
"""Constructor.
:param collection: Provide bibliographic coverage to all
Odilo books in the given Collection.
:param api_class: Instantiate this class with the given Collection,
rather than instantiating OdiloAPI.
"""
super(OdiloBibliographicCoverageProvider,
self).__init__(collection, **kwargs)
if isinstance(api_class, OdiloAPI):
# Use a previously instantiated OdiloAPI instance
# rather than creating a new one.
self.api = api_class
else:
# A web application should not use this option because it
# will put a non-scoped session in the mix.
_db = Session.object_session(collection)
self.api = api_class(_db, collection)
self.replacement_policy = ReplacementPolicy(
identifiers=True,
subjects=True,
contributions=True,
links=True,
formats=True,
rights=True,
link_content=True,
# even_if_not_apparently_updated=False,
analytics=Analytics(self._db))
def test_apply_creates_work_and_presentation_edition_if_needed(self):
edition = self._edition()
# This pool doesn't have a presentation edition or a work yet.
pool = self._licensepool(edition)
# We have new circulation data for this pool.
circulation_data = CirculationData(
formats=[],
data_source=edition.data_source,
primary_identifier=edition.primary_identifier,
)
# If we apply the new CirculationData the work gets both a
# presentation and a work.
replacement_policy = ReplacementPolicy()
circulation_data.apply(self._db, pool.collection, replacement_policy)
assert edition == pool.presentation_edition
assert pool.work != None
# If we have another new pool for the same book in another
# collection, it will share the work.
collection = self._collection()
pool2 = self._licensepool(edition, collection=collection)
circulation_data.apply(self._db, pool2.collection, replacement_policy)
assert edition == pool2.presentation_edition
assert pool.work == pool2.work
def process_book(self, bibliographic, availability):
license_pool, new_license_pool = availability.license_pool(
self._db, self.collection)
now = datetime.datetime.utcnow()
edition, new_edition = bibliographic.edition(self._db)
license_pool.edition = edition
policy = ReplacementPolicy(
identifiers=False,
subjects=True,
contributions=True,
formats=True,
)
availability.apply(
self._db,
license_pool.collection,
replace=policy,
)
if new_edition:
bibliographic.apply(edition, self.collection, replace=policy)
if new_license_pool or new_edition:
# At this point we have done work equivalent to that done by
# the EnkiBibliographicCoverageProvider. Register that the
# work has been done so we don't have to do it again.
identifier = edition.primary_identifier
self.bibliographic_coverage_provider.handle_success(identifier)
self.bibliographic_coverage_provider.add_coverage_record_for(
identifier)
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool,
CirculationEvent.DISTRIBUTOR_TITLE_ADD, now)
return edition, license_pool
def test_rights_status_default_rights_passed_in(self):
identifier = IdentifierData(
Identifier.GUTENBERG_ID,
"abcd",
)
link = LinkData(
rel=Hyperlink.DRM_ENCRYPTED_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
)
circulation_data = CirculationData(
data_source=DataSource.OA_CONTENT_SERVER,
primary_identifier=identifier,
default_rights_uri=RightsStatus.CC_BY,
links=[link],
)
replace = ReplacementPolicy(formats=True, )
pool, ignore = circulation_data.license_pool(self._db,
self._default_collection)
circulation_data.apply(self._db, pool.collection, replace)
assert True == pool.open_access
assert 1 == len(pool.delivery_mechanisms)
# The rights status is the one that was passed in to CirculationData.
assert RightsStatus.CC_BY == pool.delivery_mechanisms[
0].rights_status.uri
def _reap(self, identifier):
"""Update our local circulation information to reflect the fact that
the identified book has been removed from the remote
collection.
"""
collection = self.collection
pool = identifier.licensed_through_collection(collection)
if not pool:
self.log.warn(
"Was about to reap %r but no local license pool in this collection.",
identifier)
return
if pool.licenses_owned == 0:
# Already reaped.
return
self.log.info("Reaping %r", identifier)
availability = CirculationData(
data_source=pool.data_source,
primary_identifier=identifier,
licenses_owned=0,
licenses_available=0,
licenses_reserved=0,
patrons_in_hold_queue=0,
)
availability.apply(self._db, collection,
ReplacementPolicy.from_license_source(self._db))
def test_replacement_policy_uses_provided_mirror(self):
collection = MockOverdriveAPI.mock_collection(self._db)
mirror = MockS3Uploader()
replacement_policy = ReplacementPolicy.from_metadata_source(
mirror=mirror
)
api = MockOverdriveAPI(self._db, collection)
api.queue_collection_token()
provider = OverdriveBibliographicCoverageProvider(
collection, replacement_policy=replacement_policy,
api_class=api
)
# Any resources discovered by Overdrive will be
# sent through this mirror.
eq_(mirror, provider.replacement_policy.mirror)
http = DummyHTTPClient()
provider.replacement_policy.http_get = http.do_get
# Now let's try looking up a specific identifier through 'Overdrive'.
identifier = self._identifier(
Identifier.OVERDRIVE_ID, "3896665d-9d81-4cac-bd43-ffc5066de1f5"
)
body = self.data_file("overdrive/overdrive_metadata.json")
provider.api.queue_response(200, {}, body)
test_cover = self.data_file("covers/test-book-cover.png")
test_small_cover = self.data_file("covers/tiny-image-cover.png")
# Overdrive's full-sized image -- we will be creating our own
# thumbnail from this.
http.queue_response(200, "image/jpeg", {}, test_cover)
# Overdrive's thumbnail image -- we will not be using this
http.queue_response(200, "image/jpeg", {}, test_small_cover)
record = provider.ensure_coverage(identifier)
eq_("success", record.status)
# The full image and the thumbnail have been uploaded to
# the fake S3.
full, thumbnail = mirror.uploaded
eq_(test_cover, full.content)
# The URLs for the Resource objects are our S3 URLs, not Overdrive's
# URLs.
expect = "Overdrive/Overdrive+ID/%s" % identifier.identifier
for url in [full.mirror_url, thumbnail.mirror_url]:
assert expect in url
assert "/scaled/" in thumbnail.mirror_url
assert "/scaled/" not in full.mirror_url
# The thumbnail is a newly created image that is not the
# same as the full image or the test cover.
assert thumbnail.content != test_small_cover
assert thumbnail.content != test_cover
def default_circulation_replacement_policy(self):
return ReplacementPolicy(
identifiers=False,
subjects=True,
contributions=True,
formats=True,
analytics=Analytics(self._db),
)
def test_implicit_format_for_open_access_link(self):
# A format is a delivery mechanism. We handle delivery on open access
# pools from our mirrored content in S3.
# Tests that when a link is open access, a pool can be delivered.
edition, pool = self._edition(with_license_pool=True)
# This is the delivery mechanism created by default when you
# create a book with _edition().
[epub] = pool.delivery_mechanisms
assert Representation.EPUB_MEDIA_TYPE == epub.delivery_mechanism.content_type
assert DeliveryMechanism.ADOBE_DRM == epub.delivery_mechanism.drm_scheme
link = LinkData(
rel=Hyperlink.OPEN_ACCESS_DOWNLOAD,
media_type=Representation.PDF_MEDIA_TYPE,
href=self._url,
)
circulation_data = CirculationData(
data_source=DataSource.GUTENBERG,
primary_identifier=edition.primary_identifier,
links=[link],
)
replace = ReplacementPolicy(formats=True, )
circulation_data.apply(self._db, pool.collection, replace)
# We destroyed the default delivery format and added a new,
# open access delivery format.
[pdf] = pool.delivery_mechanisms
assert Representation.PDF_MEDIA_TYPE == pdf.delivery_mechanism.content_type
assert DeliveryMechanism.NO_DRM == pdf.delivery_mechanism.drm_scheme
circulation_data = CirculationData(
data_source=DataSource.GUTENBERG,
primary_identifier=edition.primary_identifier,
links=[],
)
replace = ReplacementPolicy(
formats=True,
links=True,
)
circulation_data.apply(self._db, pool.collection, replace)
# Now we have no formats at all.
assert 0 == len(pool.delivery_mechanisms)
def setup(self):
super(TestIntegrationClientCoverImageCoverageProvider, self).setup()
mirror = MockS3Uploader()
replacement_policy = ReplacementPolicy.from_metadata_source(
mirror=mirror)
self.collection = self._collection(
protocol=ExternalIntegration.OPDS_FOR_DISTRIBUTORS)
self.provider = IntegrationClientCoverImageCoverageProvider(
replacement_policy=replacement_policy, collection=self.collection)
def test_replacement_policy_uses_provided_mirror(self):
collection = MockOverdriveAPI.mock_collection(self._db)
mirror = MockS3Uploader()
replacement_policy = ReplacementPolicy.from_metadata_source(
mirror=mirror)
api = MockOverdriveAPI(self._db, collection)
api.queue_collection_token()
provider = OverdriveBibliographicCoverageProvider(
collection, replacement_policy=replacement_policy, api_class=api)
# Any resources discovered by Overdrive will be
# sent through this mirror.
eq_(mirror, provider.replacement_policy.mirror)
http = DummyHTTPClient()
provider.replacement_policy.http_get = http.do_get
# Now let's try looking up a specific identifier through 'Overdrive'.
identifier = self._identifier(Identifier.OVERDRIVE_ID,
"3896665d-9d81-4cac-bd43-ffc5066de1f5")
body = self.data_file("overdrive/overdrive_metadata.json")
provider.api.queue_response(200, {}, body)
test_cover = self.data_file("covers/test-book-cover.png")
test_small_cover = self.data_file("covers/tiny-image-cover.png")
# Overdrive's full-sized image -- we will be creating our own
# thumbnail from this.
http.queue_response(200, "image/jpeg", {}, test_cover)
# Overdrive's thumbnail image -- we will not be using this
http.queue_response(200, "image/jpeg", {}, test_small_cover)
record = provider.ensure_coverage(identifier)
eq_("success", record.status)
# The full image and the thumbnail have been uploaded to
# the fake S3.
full, thumbnail = mirror.uploaded
eq_(test_cover, full.content)
# The URLs for the Resource objects are our S3 URLs, not Overdrive's
# URLs.
expect = "Overdrive/Overdrive+ID/%s" % identifier.identifier
for url in [full.mirror_url, thumbnail.mirror_url]:
assert expect in url
assert "/scaled/" in thumbnail.mirror_url
assert "/scaled/" not in full.mirror_url
# The thumbnail is a newly created image that is not the
# same as the full image or the test cover.
assert thumbnail.content != test_small_cover
assert thumbnail.content != test_cover
def update_licensepool_for_identifier(self, isbn, availability):
"""Update availability information for a single book.
If the book has never been seen before, a new LicensePool
will be created for the book.
The book's LicensePool will be updated with current approximate
circulation information (we can tell if it's available, but
not how many copies).
Bibliographic coverage will be ensured for the OneClick Identifier.
Work will be created for the LicensePool and set as presentation-ready.
:param isbn the identifier OneClick uses
:param availability boolean denoting if book can be lent to patrons
"""
# find a license pool to match the isbn, and see if it'll need a metadata update later
license_pool, is_new_pool = LicensePool.for_foreign_id(
self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, isbn,
collection=self.collection
)
if is_new_pool:
# This is the first time we've seen this book. Make sure its
# identifier has bibliographic coverage.
self.bibliographic_coverage_provider.ensure_coverage(
license_pool.identifier
)
# now tell the licensepool if it's lendable
policy = ReplacementPolicy(
identifiers=False,
subjects=True,
contributions=True,
formats=True,
analytics=Analytics(self._db),
)
# licenses_available can be 0 or 999, depending on whether the book is
# lendable or not.
licenses_available = 999
if not availability:
licenses_available = 0
circulation_data = CirculationData(data_source=DataSource.ONECLICK,
primary_identifier=license_pool.identifier,
licenses_available=licenses_available)
license_pool, circulation_changed = circulation_data.apply(
self._db,
self.collection,
replace=policy,
)
return license_pool, is_new_pool, circulation_changed
def update_formats(self, licensepool):
"""Update the format information for a single book.
"""
info = self.metadata_lookup(licensepool.identifier)
metadata = OverdriveRepresentationExtractor.book_info_to_metadata(
info, include_bibliographic=False, include_formats=True)
circulation_data = metadata.circulation
replace = ReplacementPolicy(formats=True, )
circulation_data.apply(licensepool, replace)
def test_mirror_open_access_link_mirror_failure(self):
mirrors = dict(books_mirror=MockS3Uploader(fail=True),
covers_mirror=None)
h = DummyHTTPClient()
edition, pool = self._edition(with_license_pool=True)
data_source = DataSource.lookup(self._db, DataSource.GUTENBERG)
policy = ReplacementPolicy(mirrors=mirrors, http_get=h.do_get)
circulation_data = CirculationData(
data_source=edition.data_source,
primary_identifier=edition.primary_identifier,
)
link = LinkData(
rel=Hyperlink.OPEN_ACCESS_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
)
link_obj, ignore = edition.primary_identifier.add_link(
rel=link.rel,
href=link.href,
data_source=data_source,
media_type=link.media_type,
content=link.content,
)
h.queue_response(200, media_type=Representation.EPUB_MEDIA_TYPE)
circulation_data.mirror_link(pool, data_source, link, link_obj, policy)
representation = link_obj.resource.representation
# The representation was fetched successfully.
assert None == representation.fetch_exception
assert representation.fetched_at != None
# But mirroing failed.
assert representation.mirror_exception != None
assert None == representation.mirrored_at
assert link.media_type == representation.media_type
assert link.href == representation.url
# The mirror url was never set.
assert None == representation.mirror_url
# Book content is still there since it wasn't mirrored.
assert representation.content != None
# The license pool is suppressed when mirroring fails.
assert True == pool.suppressed
assert representation.mirror_exception in pool.license_exception
def setup(self):
super(TestIntegrationClientCoverImageCoverageProvider, self).setup()
mirror = MockS3Uploader()
replacement_policy = ReplacementPolicy.from_metadata_source(
mirror=mirror
)
self.collection = self._collection(
protocol=ExternalIntegration.OPDS_FOR_DISTRIBUTORS
)
self.provider = IntegrationClientCoverImageCoverageProvider(
replacement_policy=replacement_policy, collection=self.collection
)
def __init__(self, collection, *args, **kwargs):
_db = Session.object_session(collection)
replacement_policy = kwargs.pop('replacement_policy', None)
if not replacement_policy:
mirror = MirrorUploader.sitewide(_db)
replacement_policy = ReplacementPolicy(mirror=mirror, links=True)
# Only process identifiers that have been registered for coverage.
kwargs['registered_only'] = kwargs.get('registered_only', True)
super(IntegrationClientCoverImageCoverageProvider,
self).__init__(collection,
*args,
replacement_policy=replacement_policy,
**kwargs)
def test_format_change_may_change_open_access_status(self):
# In this test, whenever we call CirculationData.apply(), we
# want to destroy the old list of formats and recreate it.
replace_formats = ReplacementPolicy(formats=True)
# Here's a seemingly ordinary non-open-access LicensePool.
edition, pool = self._edition(with_license_pool=True)
assert False == pool.open_access
# One day, we learn that it has an open-access delivery mechanism.
link = LinkData(
rel=Hyperlink.OPEN_ACCESS_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
rights_uri=RightsStatus.CC_BY_ND,
)
circulation_data = CirculationData(
data_source=pool.data_source,
primary_identifier=pool.identifier,
links=[link],
)
# Applying this information turns the pool into an open-access pool.
circulation_data.apply(self._db,
pool.collection,
replace=replace_formats)
assert True == pool.open_access
# Then we find out it was a mistake -- the book is in copyright.
format = FormatData(
Representation.EPUB_MEDIA_TYPE,
DeliveryMechanism.NO_DRM,
rights_uri=RightsStatus.IN_COPYRIGHT,
)
circulation_data = CirculationData(
data_source=pool.data_source,
primary_identifier=pool.identifier,
formats=[format],
)
circulation_data.apply(self._db,
pool.collection,
replace=replace_formats)
# The original LPDM has been removed and only the new one remains.
assert False == pool.open_access
assert 1 == len(pool.delivery_mechanisms)
def update_licensepools_for_identifiers(self, identifiers):
"""Update availability information for a list of books.
If the book has never been seen before, a new LicensePool
will be created for the book.
The book's LicensePool will be updated with current
circulation information.
"""
identifier_strings = self.create_identifier_strings(identifiers)
response = self.availability(title_ids=identifier_strings)
collection = self.collection
parser = BibliographicParser(collection)
remainder = set(identifiers)
for bibliographic, availability in parser.process_all(
response.content):
identifier, is_new = bibliographic.primary_identifier.load(
self._db)
if identifier in remainder:
remainder.remove(identifier)
pool, is_new = availability.license_pool(self._db, collection)
availability.apply(self._db, pool.collection)
# We asked Axis about n books. It sent us n-k responses. Those
# k books are the identifiers in `remainder`. These books have
# been removed from the collection without us being notified.
for removed_identifier in remainder:
pool = identifier.licensed_through_collection(self.collection)
if not pool:
self.log.warn(
"Was about to reap %r but no local license pool in this collection.",
removed_identifier)
continue
if pool.licenses_owned == 0:
# Already reaped.
continue
self.log.info("Reaping %r", removed_identifier)
availability = CirculationData(
data_source=pool.data_source,
primary_identifier=removed_identifier,
licenses_owned=0,
licenses_available=0,
licenses_reserved=0,
patrons_in_hold_queue=0,
)
availability.apply(pool,
ReplacementPolicy.from_license_source(self._db))
class MetadataWranglerBibliographicCoverageProvider(
BibliographicCoverageProvider):
def _default_replacement_policy(self, _db):
"""In general, data used by the metadata wrangler is a reliable source
of metadata but not of licensing information. We always
provide the MirrorUploader in case a data source has cover
images available.
"""
try:
mirror = MirrorUploader.sitewide(_db)
except CannotLoadConfiguration, e:
# It's not a problem if there's no MirrorUploader
# configured -- it just means we can't mirror cover images
# when they show up.
mirror = None
return ReplacementPolicy.from_metadata_source(mirror=mirror)
def update_formats(self, licensepool):
"""Update the format information for a single book.
"""
info = self.metadata_lookup(licensepool.identifier)
metadata = OverdriveRepresentationExtractor.book_info_to_metadata(
info, include_bibliographic=False, include_formats=True)
circulation_data = metadata.circulation
# The identifier in the CirculationData needs to match the
# identifier associated with the LicensePool -- otherwise
# a new LicensePool will be created.
circulation_data._primary_identifier.identifier = licensepool.identifier.identifier
replace = ReplacementPolicy(formats=True)
_db = Session.object_session(licensepool)
circulation_data.apply(_db, licensepool.collection, replace)
def process_item(self, identifier):
self.log.debug(
"Seeing if %s needs reaping", identifier.identifier
)
metadata = self.api.get_item(identifier.identifier)
if metadata:
# This title is still in the collection. Do nothing.
return
# Get this collection's license pool for this identifier.
# We'll reap it by setting its licenses_owned to 0.
pool = identifier.licensed_through_collection(self.collection)
if not pool or pool.licenses_owned == 0:
# It's already been reaped.
return
if pool.presentation_edition:
self.log.warn(
"Removing %r from circulation",
pool.presentation_edition
)
else:
self.log.warn(
"Removing unknown title %s from circulation.",
identifier.identifier
)
now = datetime.datetime.utcnow()
circulationdata = CirculationData(
data_source=DataSource.ENKI,
primary_identifier= IdentifierData(
identifier.type, identifier.identifier
),
licenses_owned = 0,
licenses_available = 0,
patrons_in_hold_queue = 0,
last_checked = now
)
circulationdata.apply(
self._db,
self.collection,
replace=ReplacementPolicy.from_license_source(self._db)
)
return circulationdata
def process_item(self, identifier):
self.log.debug(
"Seeing if %s needs reaping", identifier.identifier
)
metadata = self.api.get_item(identifier.identifier)
if metadata:
# This title is still in the collection. Do nothing.
return
# Get this collection's license pool for this identifier.
# We'll reap it by setting its licenses_owned to 0.
pool = identifier.licensed_through_collection(self.collection)
if not pool or pool.licenses_owned == 0:
# It's already been reaped.
return
if pool.presentation_edition:
self.log.warn(
"Removing %r from circulation",
pool.presentation_edition
)
else:
self.log.warn(
"Removing unknown title %s from circulation.",
identifier.identifier
)
now = datetime.datetime.utcnow()
circulationdata = CirculationData(
data_source=DataSource.ENKI,
primary_identifier= IdentifierData(
identifier.type, identifier.identifier
),
licenses_owned = 0,
licenses_available = 0,
patrons_in_hold_queue = 0,
last_checked = now
)
circulationdata.apply(
self._db,
self.collection,
replace=ReplacementPolicy.from_license_source(self._db)
)
return circulationdata
def update_formats(self, licensepool):
"""Update the format information for a single book.
Incidentally updates the metadata, just in case Overdrive has
changed it.
"""
info = self.metadata_lookup(licensepool.identifier)
metadata = OverdriveRepresentationExtractor.book_info_to_metadata(
info, include_bibliographic=True, include_formats=True)
if not metadata:
# No work to be done.
return
edition, ignore = self._edition(licensepool)
replace = ReplacementPolicy.from_license_source(self._db)
metadata.apply(edition, self.collection, replace=replace)
def update_formats(self, licensepool):
"""Update the format information for a single book.
Incidentally updates the metadata, just in case Overdrive has
changed it.
"""
info = self.metadata_lookup(licensepool.identifier)
metadata = OverdriveRepresentationExtractor.book_info_to_metadata(
info, include_bibliographic=True, include_formats=True)
if not metadata:
# No work to be done.
return
edition, ignore = self._edition(licensepool)
replace = ReplacementPolicy.from_license_source(self._db)
metadata.apply(edition, self.collection, replace=replace)
def test_mirror_open_access_link_fetch_failure(self):
mirrors = dict(books_mirror=MockS3Uploader())
h = DummyHTTPClient()
edition, pool = self._edition(with_license_pool=True)
data_source = DataSource.lookup(self._db, DataSource.GUTENBERG)
policy = ReplacementPolicy(mirrors=mirrors, http_get=h.do_get)
circulation_data = CirculationData(
data_source=edition.data_source,
primary_identifier=edition.primary_identifier,
)
link = LinkData(
rel=Hyperlink.OPEN_ACCESS_DOWNLOAD,
media_type=Representation.EPUB_MEDIA_TYPE,
href=self._url,
)
link_obj, ignore = edition.primary_identifier.add_link(
rel=link.rel,
href=link.href,
data_source=data_source,
media_type=link.media_type,
content=link.content,
)
h.queue_response(403)
circulation_data.mirror_link(pool, data_source, link, link_obj, policy)
representation = link_obj.resource.representation
# Fetch failed, so we should have a fetch exception but no mirror url.
assert representation.fetch_exception != None
assert None == representation.mirror_exception
assert None == representation.mirror_url
assert link.href == representation.url
assert representation.fetched_at != None
assert None == representation.mirrored_at
# The license pool is suppressed when fetch fails.
assert True == pool.suppressed
assert representation.fetch_exception in pool.license_exception
def reaper_request(self, identifier):
self.log.debug ("Checking availability for " + str(identifier.identifier))
now = datetime.datetime.utcnow()
url = str(self.base_url) + str(self.item_endpoint)
args = dict()
args['method'] = "getItem"
args['recordid'] = identifier.identifier
args['size'] = "small"
args['lib'] = self.library_id
response = self.request(url, method='get', params=args)
try:
# If a book doesn't exist in Enki, we'll just get an HTML page saying we did something wrong.
data = json.loads(response.content)
self.log.debug ("Keeping existing book: " + str(identifier))
except:
# Get this collection's license pool for this identifier.
pool = identifier.licensed_through_collection(self.collection)
if pool and (pool.licenses_owned > 0):
if pool.presentation_edition:
self.log.warn("Removing %s (%s) from circulation",
pool.presentation_edition.title, pool.presentation_edition.author)
else:
self.log.warn(
"Removing unknown work %s from circulation.",
identifier.identifier
)
circulationdata = CirculationData(
data_source=DataSource.ENKI,
primary_identifier= IdentifierData(EnkiAPI.ENKI_ID, identifier.identifier),
licenses_owned = 0,
licenses_available = 0,
patrons_in_hold_queue = 0,
last_checked = now
)
circulationdata.apply(
self._db,
self.collection,
replace=ReplacementPolicy.from_license_source(self._db)
)
return circulationdata
def update_consolidated_copy(self, _db, copy_info, analytics=None):
"""Process information about the current status of a consolidated
copy from the consolidated copies feed.
"""
identifier = copy_info.get("identifier")
licenses = copy_info.get("licenses")
available = copy_info.get("available")
identifier_data = IdentifierData(Identifier.URI, identifier)
circulation_data = CirculationData(
data_source=self.data_source_name,
primary_identifier=identifier_data,
licenses_owned=licenses,
licenses_available=available,
)
replacement_policy = ReplacementPolicy(analytics=analytics)
pool, ignore = circulation_data.apply(_db, self.collection(_db),
replacement_policy)
# Update licenses reserved if there are holds.
if len(pool.holds) > 0 and pool.licenses_available > 0:
self.update_hold_queue(pool)
def test_work_from_metadata(self):
"""Validate the ability to create a new Work from appropriate metadata.
"""
class Mock(MockDirectoryImportScript):
"""In this test we need to verify that annotate_metadata
was called but did nothing.
"""
def annotate_metadata(self, metadata, *args, **kwargs):
metadata.annotated = True
return super(Mock, self).annotate_metadata(
metadata, *args, **kwargs
)
identifier = IdentifierData(Identifier.GUTENBERG_ID, "1003")
identifier_obj, ignore = identifier.load(self._db)
metadata = Metadata(
DataSource.GUTENBERG,
primary_identifier=identifier,
title=u"A book"
)
metadata.annotated = False
datasource = DataSource.lookup(self._db, DataSource.GUTENBERG)
policy = ReplacementPolicy.from_license_source(self._db)
mirror = MockS3Uploader()
policy.mirror = mirror
# Here, work_from_metadata calls annotate_metadata, but does
# not actually import anything because there are no files 'on
# disk' and thus no way to actually get the book.
collection = self._default_collection
args = (collection, metadata, policy, "cover directory",
"ebook directory", RightsStatus.CC0)
script = Mock(self._db)
eq_(None, script.work_from_metadata(*args))
eq_(True, metadata.annotated)
# Now let's try it with some files 'on disk'.
with open(self.sample_cover_path('test-book-cover.png')) as fh:
image = fh.read()
mock_filesystem = {
'cover directory' : (
'cover.jpg', Representation.JPEG_MEDIA_TYPE, image
),
'ebook directory' : (
'book.epub', Representation.EPUB_MEDIA_TYPE, "I'm an EPUB."
)
}
script = MockDirectoryImportScript(
self._db, mock_filesystem=mock_filesystem
)
work = script.work_from_metadata(*args)
# We have created a book. It has a cover image, which has a
# thumbnail.
eq_("A book", work.title)
assert work.cover_full_url.endswith(
'/test.cover.bucket/Gutenberg/Gutenberg+ID/1003/1003.jpg'
)
assert work.cover_thumbnail_url.endswith(
'/test.cover.bucket/scaled/300/Gutenberg/Gutenberg+ID/1003/1003.png'
)
[pool] = work.license_pools
assert pool.open_access_download_url.endswith(
'/test.content.bucket/Gutenberg/Gutenberg+ID/1003/A+book.epub'
)
eq_(RightsStatus.CC0,
pool.delivery_mechanisms[0].rights_status.uri)
# The mock S3Uploader has a record of 'uploading' all these files
# to S3.
epub, full, thumbnail = mirror.uploaded
eq_(epub.url, pool.open_access_download_url)
eq_(full.url, work.cover_full_url)
eq_(thumbnail.url, work.cover_thumbnail_url)
# The EPUB Representation was cleared out after the upload, to
# save database space.
eq_("I'm an EPUB.", mirror.content[0])
eq_(None, epub.content)
def __init__(self, _db, batch_size=10, cutoff_time=None,
uploader=None, providers=None, **kwargs):
output_source, made_new = get_one_or_create(
_db, DataSource,
name=DataSource.INTERNAL_PROCESSING
)
# Other components don't have INTERNAL_PROCESSING as offering
# licenses, but we do, because we're responsible for managing
# LicensePools.
output_source.offers_licenses=True
input_identifier_types = [Identifier.OVERDRIVE_ID, Identifier.ISBN]
super(IdentifierResolutionCoverageProvider, self).__init__(
service_name="Identifier Resolution Coverage Provider",
input_identifier_types=input_identifier_types,
output_source=output_source,
batch_size=batch_size,
operation=CoverageRecord.RESOLVE_IDENTIFIER_OPERATION,
)
# Since we are the metadata wrangler, any resources we find,
# we mirror to S3.
mirror = uploader or S3Uploader()
# We're going to be aggressive about recalculating the presentation
# for this work because either the work is currently not set up
# at all, or something went wrong trying to set it up.
presentation_calculation_policy = PresentationCalculationPolicy(
regenerate_opds_entries=True,
update_search_index=True
)
policy = ReplacementPolicy.from_metadata_source(
mirror=mirror, even_if_not_apparently_updated=True,
presentation_calculation_policy=presentation_calculation_policy
)
if providers:
# For testing purposes. Initializing the real coverage providers
# during tests can cause requests to third-parties.
(self.required_coverage_providers,
self.optional_coverage_providers) = providers
else:
overdrive = OverdriveBibliographicCoverageProvider(
_db, metadata_replacement_policy=policy
)
content_cafe = ContentCafeCoverageProvider(self._db)
content_server = ContentServerCoverageProvider(self._db)
oclc_classify = OCLCClassifyCoverageProvider(self._db)
self.required_coverage_providers = [
overdrive, content_cafe, content_server, oclc_classify
]
self.optional_coverage_providers = []
self.viaf = VIAFClient(self._db)
self.image_mirrors = {
DataSource.OVERDRIVE : OverdriveCoverImageMirror(
self._db, uploader=uploader
)
}
self.image_scaler = ImageScaler(
self._db, self.image_mirrors.values(), uploader=uploader
)
self.oclc_linked_data = LinkedDataCoverageProvider(self._db)
def test_open_access_content_mirrored(self):
# Make sure that open access material links are translated to our S3 buckets, and that
# commercial material links are left as is.
# Note: Mirroring tests passing does not guarantee that all code now
# correctly calls on CirculationData, as well as Metadata. This is a risk.
mirrors = dict(books_mirror=MockS3Uploader(), covers_mirror=None)
mirror_type = ExternalIntegrationLink.OPEN_ACCESS_BOOKS
# Here's a book.
edition, pool = self._edition(with_license_pool=True)
# Here's a link to the content of the book, which will be mirrored.
link_mirrored = LinkData(
rel=Hyperlink.OPEN_ACCESS_DOWNLOAD,
href="http://example.com/",
media_type=Representation.EPUB_MEDIA_TYPE,
content="i am a tiny book",
)
# This link will not be mirrored.
link_unmirrored = LinkData(
rel=Hyperlink.DRM_ENCRYPTED_DOWNLOAD,
href="http://example.com/2",
media_type=Representation.EPUB_MEDIA_TYPE,
content="i am a pricy book",
)
# Apply the metadata.
policy = ReplacementPolicy(mirrors=mirrors)
metadata = Metadata(
data_source=edition.data_source,
links=[link_mirrored, link_unmirrored],
)
metadata.apply(edition, pool.collection, replace=policy)
# make sure the refactor is done right, and metadata does not upload
assert 0 == len(mirrors[mirror_type].uploaded)
circulation_data = CirculationData(
data_source=edition.data_source,
primary_identifier=edition.primary_identifier,
links=[link_mirrored, link_unmirrored],
)
circulation_data.apply(self._db, pool.collection, replace=policy)
# make sure the refactor is done right, and circulation does upload
assert 1 == len(mirrors[mirror_type].uploaded)
# Only the open-access link has been 'mirrored'.
[book] = mirrors[mirror_type].uploaded
# It's remained an open-access link.
assert [Hyperlink.OPEN_ACCESS_DOWNLOAD
] == [x.rel for x in book.resource.links]
# It's been 'mirrored' to the appropriate S3 bucket.
assert book.mirror_url.startswith(
"https://test-content-bucket.s3.amazonaws.com/")
expect = "/%s/%s.epub" % (edition.primary_identifier.identifier,
edition.title)
assert book.mirror_url.endswith(expect)
# make sure the mirrored link is safely on edition
sorted_edition_links = sorted(pool.identifier.links,
key=lambda x: x.rel)
unmirrored_representation, mirrored_representation = [
edlink.resource.representation for edlink in sorted_edition_links
]
assert mirrored_representation.mirror_url.startswith(
"https://test-content-bucket.s3.amazonaws.com/")
# make sure the unmirrored link is safely on edition
assert "http://example.com/2" == unmirrored_representation.url
# make sure the unmirrored link has not been translated to an S3 URL
assert None == unmirrored_representation.mirror_url
