def test_process_batch_with_identifier_mapping(self):
"""Test that internal identifiers are mapped to and from the form used
by the external service.
"""
# Unlike other tests in this class, we are using a real
# implementation of OPDSImportCoverageProvider.process_batch.
class TestProvider(OPDSImportCoverageProvider):
SERVICE_NAME = "Test provider"
DATA_SOURCE_NAME = DataSource.OA_CONTENT_SERVER
# Mock the identifier mapping
def create_identifier_mapping(self, batch):
return self.mapping
# This means we need to mock the lookup client instead.
lookup = MockSimplifiedOPDSLookup(self._url)
# And create an ExternalIntegration for the metadata_client object.
self._external_integration(
ExternalIntegration.METADATA_WRANGLER,
goal=ExternalIntegration.METADATA_GOAL,
url=self._url,
)
self._default_collection.external_integration.set_setting(
Collection.DATA_SOURCE_NAME_SETTING, DataSource.OA_CONTENT_SERVER
)
provider = TestProvider(self._default_collection, lookup)
# Create a hard-coded mapping. We use id1 internally, but the
# foreign data source knows the book as id2.
id1 = self._identifier()
id2 = self._identifier()
provider.mapping = {id2: id1}
feed = (
"<feed><entry><id>%s</id><title>Here's your title!</title></entry></feed>"
% id2.urn
)
headers = {"content-type": OPDSFeed.ACQUISITION_FEED_TYPE}
lookup.queue_response(200, headers=headers, content=feed)
[identifier] = provider.process_batch([id1])
# We wanted to process id1. We sent id2 to the server, the
# server responded with an <entry> for id2, and it was used to
# modify the Edition associated with id1.
assert id1 == identifier
[edition] = id1.primarily_identifies
assert "Here's your title!" == edition.title
def test_import_feed_response(self):
"""Verify that import_feed_response instantiates the
OPDS_IMPORTER_CLASS subclass and calls import_from_feed
on it.
"""
class MockOPDSImporter(OPDSImporter):
def import_from_feed(self, text):
"""Return information that's useful for verifying
that the OPDSImporter was instantiated with the
right values.
"""
return (text, self.collection, self.identifier_mapping,
self.data_source_name)
class MockProvider(MockOPDSImportCoverageProvider):
OPDS_IMPORTER_CLASS = MockOPDSImporter
provider = MockProvider(self._default_collection)
provider.lookup_client = MockSimplifiedOPDSLookup(self._url)
response = MockRequestsResponse(
200, {'content-type': OPDSFeed.ACQUISITION_FEED_TYPE}, "some data")
id_mapping = object()
(text, collection, mapping,
data_source_name) = provider.import_feed_response(
response, id_mapping)
eq_("some data", text)
eq_(provider.collection, collection)
eq_(id_mapping, mapping)
eq_(provider.data_source.name, data_source_name)
def test_process_batch_with_identifier_mapping(self):
"""Test that internal identifiers are mapped to and from the form used
by the external service.
"""
# Unlike other tests in this class, we are using a real
# implementation of OPDSImportCoverageProvider.process_batch.
class TestProvider(OPDSImportCoverageProvider):
SERVICE_NAME = "Test provider"
DATA_SOURCE_NAME = DataSource.OA_CONTENT_SERVER
# Mock the identifier mapping
def create_identifier_mapping(self, batch):
return self.mapping
# This means we need to mock the lookup client instead.
lookup = MockSimplifiedOPDSLookup(self._url)
# And create an ExternalIntegration for the metadata_client object.
self._external_integration(
ExternalIntegration.METADATA_WRANGLER,
goal=ExternalIntegration.METADATA_GOAL, url=self._url
)
self._default_collection.external_integration.set_setting(
Collection.DATA_SOURCE_NAME_SETTING, DataSource.OA_CONTENT_SERVER
)
provider = TestProvider(self._default_collection, lookup)
# Create a hard-coded mapping. We use id1 internally, but the
# foreign data source knows the book as id2.
id1 = self._identifier()
id2 = self._identifier()
provider.mapping = { id2 : id1 }
feed = "<feed><entry><id>%s</id><title>Here's your title!</title></entry></feed>" % id2.urn
headers = {"content-type" : OPDSFeed.ACQUISITION_FEED_TYPE}
lookup.queue_response(200, headers=headers, content=feed)
[identifier] = provider.process_batch([id1])
# We wanted to process id1. We sent id2 to the server, the
# server responded with an <entry> for id2, and it was used to
# modify the Edition associated with id1.
eq_(id1, identifier)
[edition] = id1.primarily_identifies
eq_("Here's your title!", edition.title)
def setup(self):
super(TestContentServerCoverageProvider, self).setup()
self.lookup = MockSimplifiedOPDSLookup("http://url/")
self.provider = ContentServerCoverageProvider(
self._db, content_server=self.lookup
)
base_path = os.path.split(__file__)[0]
self.resource_path = os.path.join(base_path, "files", "opds")
def create_provider(self, **kwargs):
with temp_config() as config:
config[Configuration.INTEGRATIONS][
Configuration.METADATA_WRANGLER_INTEGRATION] = {
Configuration.URL: "http://url.gov"
}
lookup = MockSimplifiedOPDSLookup.from_config()
return MetadataWranglerCoverageProvider(self._db,
lookup=lookup,
**kwargs)
def test_badresponseexception_on_non_opds_feed(self):
"""If the lookup protocol sends something that's not an OPDS
feed, refuse to go any further.
"""
provider = self._provider()
provider.lookup_client = MockSimplifiedOPDSLookup(self._url)
response = MockRequestsResponse(200, {"content-type": "text/plain"},
"Some data")
provider.lookup_client.queue_response(response)
assert_raises_regexp(BadResponseException,
"Wrong media type: text/plain",
provider.import_feed_response, response, None)
def test_process_batch_with_identifier_mapping(self):
"""Test that internal identifiers are mapped to and from the form used
by the external service.
"""
# Unlike other tests in this class, we are using a real
# implementation of OPDSImportCoverageProvider.process_batch.
class TestProvider(OPDSImportCoverageProvider):
# Mock the identifier mapping
def create_identifier_mapping(self, batch):
return self.mapping
# This means we need to mock the lookup client instead.
lookup = MockSimplifiedOPDSLookup(self._url)
source = DataSource.lookup(self._db, DataSource.OA_CONTENT_SERVER)
provider = TestProvider("test provider", [], source, lookup=lookup)
# Create a hard-coded mapping. We use id1 internally, but the
# foreign data source knows the book as id2.
id1 = self._identifier()
id2 = self._identifier()
provider.mapping = {id2: id1}
feed = "<feed><entry><id>%s</id><title>Here's your title!</title></entry></feed>" % id2.urn
headers = {"content-type": OPDSFeed.ACQUISITION_FEED_TYPE}
lookup.queue_response(200, headers=headers, content=feed)
[identifier] = provider.process_batch([id1])
# We wanted to process id1. We sent id2 to the server, the
# server responded with an <entry> for id2, and it was used to
# modify the Edition associated with id1.
eq_(id1, identifier)
[edition] = id1.primarily_identifies
eq_("Here's your title!", edition.title)
eq_(id1, edition.primary_identifier)
def test_only_open_access_books_considered(self):
lookup = MockSimplifiedOPDSLookup(self._url)
provider = ContentServerBibliographicCoverageProvider(
self._default_collection, lookup)
# Here's an open-access work.
w1 = self._work(with_license_pool=True, with_open_access_download=True)
# Here's a work that's not open-access.
w2 = self._work(with_license_pool=True,
with_open_access_download=False)
w2.license_pools[0].open_access = False
# Only the open-access work needs coverage.
eq_([w1.license_pools[0].identifier],
provider.items_that_need_coverage().all())
def test_finalize_license_pool(self):
identifier = self._identifier()
license_source = DataSource.lookup(self._db, DataSource.GUTENBERG)
data_source = DataSource.lookup(self._db, DataSource.OA_CONTENT_SERVER)
# Here's a LicensePool with no presentation edition.
pool, is_new = LicensePool.for_foreign_id(
self._db,
license_source,
identifier.type,
identifier.identifier,
collection=self._default_collection)
eq_(None, pool.presentation_edition)
# Here's an Edition for the same book as the LicensePool but
# from a different data source.
edition, is_new = Edition.for_foreign_id(self._db, data_source,
identifier.type,
identifier.identifier)
edition.title = self._str
# Although Edition and LicensePool share an identifier, they
# are not otherwise related.
eq_(None, pool.presentation_edition)
# finalize_license_pool() will create a Work and update the
# LicensePool's presentation edition, based on the brand-new
# Edition.
lookup = MockSimplifiedOPDSLookup(self._url)
provider = ContentServerBibliographicCoverageProvider(
self._default_collection, lookup)
provider.finalize_license_pool(pool)
work = pool.work
eq_(edition.title, pool.presentation_edition.title)
eq_(True, work.presentation_ready)
def _lookup_client(self, url):
return MockSimplifiedOPDSLookup(url)
class TestContentServerCoverageProvider(DatabaseTest):
def setup(self):
super(TestContentServerCoverageProvider, self).setup()
self.lookup = MockSimplifiedOPDSLookup("http://url/")
self.provider = ContentServerCoverageProvider(
self._db, content_server=self.lookup
)
base_path = os.path.split(__file__)[0]
self.resource_path = os.path.join(base_path, "files", "opds")
def sample_data(self, filename):
path = os.path.join(self.resource_path, filename)
return open(path).read()
def test_success(self):
data = self.sample_data("content_server_lookup.opds")
self.lookup.queue_response(
200, {"content-type": "application/atom+xml"},
content=data
)
identifier = self._identifier(identifier_type=Identifier.GUTENBERG_ID)
# Make the Identifier match the book the queued-up response is
# talking about
identifier.identifier = "20201"
success = self.provider.process_item(identifier)
eq_(success, identifier)
# The book was imported and turned into a Work.
work = identifier.licensed_through.work
eq_("Mary Gray", work.title)
# It's not presentation-ready yet, because we are the metadata
# wrangler and our work is not yet done.
eq_(False, work.presentation_ready)
def test_no_such_work(self):
data = self.sample_data("no_such_work.opds")
self.lookup.queue_response(
200, {"content-type": "application/atom+xml"},
content=data
)
identifier = self._identifier(identifier_type=Identifier.GUTENBERG_ID)
# Make the Identifier match the book the queued-up response is
# talking about
identifier.identifier = "2020110"
failure = self.provider.process_item(identifier)
eq_(identifier, failure.obj)
eq_("404: I've never heard of this work.", failure.exception)
eq_(DataSource.OA_CONTENT_SERVER, failure.data_source.name)
# Most of the time this is a persistent error but it's
# possible that we know about a book the content server
# doesn't know about yet.
eq_(True, failure.transient)
def test_wrong_work_in_response(self):
data = self.sample_data("content_server_lookup.opds")
self.lookup.queue_response(
200, {"content-type": "application/atom+xml"},
content=data
)
identifier = self._identifier(identifier_type=Identifier.GUTENBERG_ID)
# The content server told us about a different book than the
# one we asked about.
identifier.identifier = "999"
failure = self.provider.process_item(identifier)
eq_(identifier, failure.obj)
eq_('Identifier was not mentioned in lookup response', failure.exception)
eq_(DataSource.OA_CONTENT_SERVER, failure.data_source.name)
eq_(True, failure.transient)
def test_content_server_http_failure(self):
"""Test that HTTP-level failures of the content server
become transient CoverageFailures.
"""
identifier = self._identifier(identifier_type=Identifier.GUTENBERG_ID)
self.lookup.queue_response(
500, content="help me!"
)
failure = self.provider.process_item(identifier)
eq_(identifier, failure.obj)
eq_("Got status code 500 from external server, cannot continue.",
failure.exception)
eq_(True, failure.transient)
self.lookup.queue_response(
200, {"content-type": "text/plain"}, content="help me!"
)
failure = self.provider.process_item(identifier)
eq_(identifier, failure.obj)
eq_("Content Server served unhandleable media type: text/plain",
failure.exception)
eq_(True, failure.transient)
eq_(DataSource.OA_CONTENT_SERVER, failure.data_source.name)
