def test_gather_providers_overdrive(self):
# Set up an Overdrive integration.
overdrive_collection = MockOverdriveAPI.mock_collection(self._db)
provider_kwargs = {
OverdriveBibliographicCoverageProvider : dict(
api_class=MockOverdriveAPI
)
}
provider = IdentifierResolutionCoverageProvider(
overdrive_collection, provider_kwargs=provider_kwargs
)
# The OCLC provider was already configured; now there is also an OverdriveBibliographicCoverageProvider.
eq_(len(provider.providers), 2)
[overdrive] = [x for x in provider.providers if isinstance(x, OverdriveBibliographicCoverageProvider)]
assert overdrive
# The MockOverdriveAPI we passed in as part of provider_kwargs
# was instantiated as part of the
# OverdriveBibliographicCoverageProvider instantiation.
assert isinstance(overdrive.api, MockOverdriveAPI)
# Since the Overdrive coverage provider checks with VIAF after
# adding author information to the database, it's been given a
# reference to our VIAF client.
eq_(provider.viaf, overdrive.viaf)
# All subproviders are associated with the collection used in the
# main provider, and they all have the same replacement policy.
# (And thus the same MirrorUploader.)
for subprovider in provider.providers:
eq_(provider.collection, subprovider.collection)
eq_(provider.replacement_policy, subprovider.replacement_policy)
def test_providers_overdrive(self):
# For an Overdrive collection...
collection = MockOverdriveAPI.mock_collection(self._db)
# In lieu of a proper mock API, create one that will crash
# if it tries to make a real HTTP request.
mock_content_cafe = ContentCafeAPI(self._db, None, object(), object(),
self.uploader)
resolver = IdentifierResolutionCoverageProvider(
collection,
overdrive_api_class=MockOverdriveAPI,
content_cafe_api=mock_content_cafe,
uploader=self.uploader)
# We get three required coverage providers: Content Cafe, OCLC
# Classify, and Overdrive.
optional, [content_cafe, oclc_classify,
overdrive] = resolver.providers()
eq_([], optional)
assert isinstance(content_cafe, ContentCafeCoverageProvider)
assert isinstance(oclc_classify, OCLCClassifyCoverageProvider)
assert isinstance(overdrive, OverdriveBibliographicCoverageProvider)
def test_gather_providers_content_cafe(self):
# Set up a Content Cafe integration
content_cafe = self._external_integration(
goal=ExternalIntegration.METADATA_GOAL,
protocol=ExternalIntegration.CONTENT_CAFE,
username="******",
password="******"
)
# The OCLC provider was already configured; now there is also a ContentCafeCoverageProvider.
provider = IdentifierResolutionCoverageProvider(self._default_collection)
eq_(len(provider.providers), 2)
[content_cafe] = [x for x in provider.providers if isinstance(x, ContentCafeCoverageProvider)]
assert content_cafe
def test_providers_opds(self):
# For an OPDS collection that goes against the open-access content
# server...
self._default_collection.external_integration.set_setting(
Collection.DATA_SOURCE_NAME_SETTING, DataSource.OA_CONTENT_SERVER)
uploader = object()
# In lieu of a proper mock API, create one that will crash
# if it tries to make a real HTTP request.
mock_content_cafe = ContentCafeAPI(self._db, None, object(), object(),
self.uploader)
resolver = IdentifierResolutionCoverageProvider(
self._default_collection,
content_cafe_api=mock_content_cafe,
uploader=uploader)
# We get three required coverage providers: Content Cafe, OCLC
# Classify, and OPDS Lookup Protocol.
optional, [content_cafe, oclc_classify, opds] = resolver.providers()
eq_([], optional)
assert isinstance(content_cafe, ContentCafeCoverageProvider)
assert isinstance(oclc_classify, OCLCClassifyCoverageProvider)
assert isinstance(opds, LookupClientCoverageProvider)
eq_(mock_content_cafe, content_cafe.content_cafe)
eq_(self._default_collection, opds.collection)
def test_gather_providers_opds_for_distributors(self):
collection = self._default_collection
collection.protocol = ExternalIntegration.OPDS_FOR_DISTRIBUTORS
provider = IdentifierResolutionCoverageProvider(collection)
# The OCLC provider was already configured; now there is also an IntegrationClientCoverImageCoverageProvider.
eq_(len(provider.providers), 2)
[integration_client] = [x for x in provider.providers if isinstance(x, IntegrationClientCoverImageCoverageProvider)]
assert integration_client
# All subproviders are associated with the collection used in the
# main provider, and they all have the same replacement policy.
# (And thus the same MirrorUploader.)
for subprovider in provider.providers:
eq_(provider.collection, subprovider.collection)
eq_(provider.replacement_policy, subprovider.replacement_policy)
def test_all(self):
class Mock(IdentifierResolutionCoverageProvider):
def gather_providers(self, provider_kwargs):
return []
# We have 3 collections created here, plus the 'unaffiliated'
# collection.
unaffiliated, ignore = IdentifierResolutionCoverageProvider.unaffiliated_collection(self._db)
for i in range(3):
collection = self._collection()
# all() puts them in random order (not tested), but
# the unaffiliated collection is always last.
providers = Mock.all(self._db, mirror=object())
providers = list(providers)
eq_(4, len(providers))
eq_(unaffiliated, providers[-1].collection)
def setup(self):
super(TestIdentifierResolutionCoverageProvider, self).setup()
self.identifier = self._identifier(Identifier.OVERDRIVE_ID)
self.source = DataSource.license_source_for(self._db, self.identifier)
uploader = DummyS3Uploader()
self.coverage_provider = IdentifierResolutionCoverageProvider(
self._db, uploader=uploader, providers=([], [])
)
self.always_successful = AlwaysSuccessfulCoverageProvider(
"Always", [self.identifier.type], self.source
)
self.never_successful = NeverSuccessfulCoverageProvider(
"Never", [self.identifier.type], self.source
)
self.broken = BrokenCoverageProvider(
"Broken", [self.identifier.type], self.source
)
def default_collection(self):
if getattr(self, '_default_collection_id', None) is None:
default_collection, ignore = IdentifierResolutionCoverageProvider.unaffiliated_collection(self._db)
self._default_collection_id = default_collection.id
return get_one(self._db, Collection, id=self._default_collection_id)
def process_urns(self, urns, collection_details=None, **kwargs):
"""Processes URNs submitted via lookup request
An authenticated request can process up to 30 URNs at once,
but must specify a collection under which to catalog the
URNs. This is used when initially recording the fact that
certain URNs are in a collection, to get a baseline set of
metadata. Updates on the books should be obtained through the
CatalogController.
An unauthenticated request is used for testing. Such a request
does not have to specify a collection (the "Unaffiliated"
collection is used), but can only process one URN at a time.
:return: None or ProblemDetail
"""
client = authenticated_client_from_request(self._db, required=False)
if isinstance(client, ProblemDetail):
return client
resolve_now = request.args.get('resolve_now', None) is not None
collection = collection_from_details(self._db, client,
collection_details)
if client:
# Authenticated access.
if not collection:
return INVALID_INPUT.detailed(_("No collection provided."))
limit = 30
else:
# Anonymous access.
collection = self.default_collection
limit = 1
if resolve_now:
# You can't force-resolve more than one Identifier at a time.
limit = 1
if len(urns) > limit:
return INVALID_INPUT.detailed(
_("The maximum number of URNs you can provide at once is %d. (You sent %d)"
) % (limit, len(urns)))
identifiers_by_urn, failures = Identifier.parse_urns(
self._db, urns, allowed_types=self.VALID_TYPES)
self.add_urn_failure_messages(failures)
# Catalog all identifiers.
collection.catalog_identifiers(identifiers_by_urn.values())
# Load all coverage records in a single query to speed up the
# code that reports on the status of Identifiers that aren't
# ready.
self.bulk_load_coverage_records(identifiers_by_urn.values())
resolver = IdentifierResolutionCoverageProvider(
collection,
provide_coverage_immediately=resolve_now,
**self.coverage_provider_kwargs)
for urn, identifier in identifiers_by_urn.items():
self.process_identifier(identifier, urn, resolver=resolver)
def default_collection(self):
if getattr(self, '_default_collection_id', None) is None:
default_collection, ignore = IdentifierResolutionCoverageProvider.unaffiliated_collection(
self._db)
self._default_collection_id = default_collection.id
return get_one(self._db, Collection, id=self._default_collection_id)
class TestIdentifierResolutionCoverageProvider(DatabaseTest):
def setup(self):
super(TestIdentifierResolutionCoverageProvider, self).setup()
self.identifier = self._identifier(Identifier.OVERDRIVE_ID)
self.source = DataSource.license_source_for(self._db, self.identifier)
uploader = DummyS3Uploader()
self.coverage_provider = IdentifierResolutionCoverageProvider(
self._db, uploader=uploader, providers=([], [])
)
self.always_successful = AlwaysSuccessfulCoverageProvider(
"Always", [self.identifier.type], self.source
)
self.never_successful = NeverSuccessfulCoverageProvider(
"Never", [self.identifier.type], self.source
)
self.broken = BrokenCoverageProvider(
"Broken", [self.identifier.type], self.source
)
def test_items_that_need_coverage(self):
# Only items with an existing transient failure status require coverage.
self._coverage_record(
self.identifier, self.coverage_provider.output_source,
operation=CoverageRecord.RESOLVE_IDENTIFIER_OPERATION,
status=CoverageRecord.TRANSIENT_FAILURE
)
# Identifiers without coverage will be ignored.
no_coverage = self._identifier(identifier_type=Identifier.ISBN)
items = self.coverage_provider.items_that_need_coverage().all()
eq_([self.identifier], items)
def test_process_item_creates_license_pool(self):
self.coverage_provider.required_coverage_providers = [
self.always_successful
]
self.coverage_provider.process_item(self.identifier)
lp = self.identifier.licensed_through
eq_(True, isinstance(lp, LicensePool))
eq_(lp.data_source, self.coverage_provider.output_source)
def test_process_item_succeeds_if_all_required_coverage_providers_succeed(self):
self.coverage_provider.required_coverage_providers = [
self.always_successful, self.always_successful
]
# The coverage provider succeeded and returned an identifier.
result = self.coverage_provider.process_item(self.identifier)
eq_(result, self.identifier)
def test_process_item_fails_if_any_required_coverage_providers_fail(self):
self.coverage_provider.required_coverage_providers = [
self.always_successful, self.never_successful
]
result = self.coverage_provider.process_item(self.identifier)
eq_(True, isinstance(result, CoverageFailure))
eq_("500: What did you expect?", result.exception)
eq_(False, result.transient)
# The failure type of the IdentifierResolutionCoverageProvider
# coverage record matches the failure type of the required provider's
# coverage record.
self.never_successful.transient = True
result = self.coverage_provider.process_item(self.identifier)
eq_(True, isinstance(result, CoverageFailure))
eq_(True, result.transient)
def test_process_item_fails_when_required_provider_raises_exception(self):
self.coverage_provider.required_coverage_providers = [self.broken]
result = self.coverage_provider.process_item(self.identifier)
eq_(True, isinstance(result, CoverageFailure))
eq_(True, result.transient)
def test_process_item_fails_when_finalize_raises_exception(self):
class FinalizeAlwaysFails(IdentifierResolutionCoverageProvider):
def finalize(self, unresolved_identifier):
raise Exception("Oh no!")
provider = FinalizeAlwaysFails(
self._db, uploader=DummyS3Uploader(), providers=([], [])
)
result = provider.process_item(self.identifier)
eq_(True, isinstance(result, CoverageFailure))
assert "Oh no!" in result.exception
eq_(True, result.transient)
def test_process_item_succeeds_when_optional_provider_fails(self):
self.coverage_provider.required_coverage_providers = [
self.always_successful, self.always_successful
]
self.coverage_provider.optional_coverage_providers = [
self.always_successful, self.never_successful
]
result = self.coverage_provider.process_item(self.identifier)
# A successful result is achieved, even though the optional
# coverage provider failed.
eq_(result, self.identifier)
# An appropriate coverage record was created to mark the failure.
presentation_edition = DataSource.lookup(
self._db, DataSource.PRESENTATION_EDITION
)
r = self._db.query(CoverageRecord).filter(
CoverageRecord.identifier==self.identifier,
CoverageRecord.data_source!=presentation_edition).one()
eq_("What did you expect?", r.exception)
def test_process_one_provider(self):
"""Test what happens when IdentifierResolutionCoverageProvider
tells a subprovider to do something.
"""
collection = self._default_collection
provider = IdentifierResolutionCoverageProvider(
collection, force=object()
)
# If the subprovider can't cover the Identifier, nothing
# happens.
class CantCoverAnything(object):
def can_cover(self, identifier):
return False
def ensure_coverage(self, identifier, force):
raise Exception("I'll never be called")
provider.process_one_provider(object(), CantCoverAnything())
# Try again with a subprovider that doesn't need coverage
# for every collection.
class OnlyOnce(CollectionCoverageProvider):
SERVICE_NAME = "Do it once, it's done for every collection"
COVERAGE_COUNTS_FOR_EVERY_COLLECTION = True
DATA_SOURCE_NAME = DataSource.OVERDRIVE
def register(self, identifier, collection, force):
self.register_called_with = [identifier, collection, force]
return None, None
def ensure_coverage(self, identifier, force):
self.ensure_coverage_called_with = [identifier, force]
i1 = self._identifier()
subprovider = OnlyOnce(collection)
provider.process_one_provider(i1, subprovider)
# The subprovider's register method was called, with no
# collection being provided.
eq_([i1, None, provider.force], subprovider.register_called_with)
# If the main provider requires that coverage happen immediately,
# then ensure_coverage_called_with is called instead.
provider.provide_coverage_immediately = True
provider.process_one_provider(i1, subprovider)
eq_([i1, provider.force], subprovider.ensure_coverage_called_with)
# Try again with a subprovider that _does_ need separate coverage
# for every collection.
class EveryTime(CollectionCoverageProvider):
SERVICE_NAME = "Every collection must be covered separately"
COVERAGE_COUNTS_FOR_EVERY_COLLECTION = False
DATA_SOURCE_NAME = DataSource.OVERDRIVE
def register(self, identifier, collection, force):
self.register_called_with = [identifier, collection, force]
return None, None
def ensure_coverage(self, identifier, force):
self.ensure_coverage_called_with = [identifier, force]
subprovider = EveryTime(collection)
provider.provide_coverage_immediately = False
provider.process_one_provider(i1, subprovider)
# The subprovider's register method was called, with the
# collection we're covering being provided.
eq_([i1, provider.collection, provider.force],
subprovider.register_called_with)
# If the main provider requires that coverage happen immediately,
# then ensure_coverage_called_with is called instead.
provider.provide_coverage_immediately = True
provider.process_one_provider(i1, subprovider)
eq_([i1, provider.force], subprovider.ensure_coverage_called_with)
def test_gather_providers_no_credentials(self):
# The OCLC provider should be there from the beginning, but the other CoverageProviders
# require credentials, so IdentifierResolutionCoverageProvider can't configure them.
providers = IdentifierResolutionCoverageProvider(self._default_collection).providers
[oclc] = providers
assert isinstance(oclc, IdentifierLookupCoverageProvider)