def test_import_with_unrecognized_distributor_creates_distributor(self):
"""We get a book from the open-access content server but the license
comes from an unrecognized data source. The book is imported and
we create a DataSource to record its provenance accurately.
"""
feed = open(
os.path.join(self.resource_path,
"unrecognized_distributor.opds")).read()
importer = OPDSImporter(self._db,
data_source_name=DataSource.OA_CONTENT_SERVER)
imported_editions, pools, works, failures = (
importer.import_from_feed(feed))
eq_({}, failures)
# We imported an Edition because there was metadata.
[edition] = imported_editions
new_data_source = edition.data_source
eq_(DataSource.OA_CONTENT_SERVER, new_data_source.name)
# We imported a LicensePool because there was an open-access
# link, even though the ultimate source of the link was one
# we'd never seen before.
[pool] = pools
eq_("Unknown Source", pool.data_source.name)
# From an Edition and a LicensePool we created a Work.
eq_(1, len(works))
def test_import_with_lendability(self):
# Tests that will create Edition, LicensePool, and Work objects, when appropriate.
# For example, on a Metadata_Wrangler data source, it is only appropriate to create
# editions, but not pools or works. On a lendable data source, should create
# pools and works as well as editions.
# Tests that the number and contents of error messages are appropriate to the task.
# will create editions, but not license pools or works, because the
# metadata wrangler data source is not lendable
feed = self.content_server_mini_feed
importer_mw = OPDSImporter(
self._db, data_source_name=DataSource.METADATA_WRANGLER)
imported_editions_mw, pools_mw, works_mw, failures_mw = (
importer_mw.import_from_feed(feed))
# Both books were imported, because they were new.
eq_(2, len(imported_editions_mw))
# But pools and works weren't created, because the data source isn't lendable.
# 1 error message, because correctly didn't even get to trying to create pools,
# so no messages there, but do have that entry stub at end of sample xml file,
# which should fail with a message.
eq_(1, len(failures_mw))
eq_(0, len(pools_mw))
eq_(0, len(works_mw))
# try again, with a license pool-acceptable data source
importer_g = OPDSImporter(self._db,
data_source_name=DataSource.GUTENBERG)
imported_editions_g, pools_g, works_g, failures_g = (
importer_g.import_from_feed(feed))
# we made new editions, because we're now creating edition per data source, not overwriting
eq_(2, len(imported_editions_g))
# TODO: and we also created presentation editions, with author and title set
# now pools and works are in, too
eq_(1, len(failures_g))
eq_(2, len(pools_g))
eq_(2, len(works_g))
# assert that bibframe datasource from feed was correctly overwritten
# with data source I passed into the importer.
for pool in pools_g:
eq_(pool.data_source.name, DataSource.GUTENBERG)
def test_import_book_that_offers_no_license(self):
path = os.path.join(self.resource_path, "book_without_license.opds")
feed = open(path).read()
importer = OPDSImporter(self._db, DataSource.OA_CONTENT_SERVER)
imported_editions, imported_pools, imported_works, failures = (
importer.import_from_feed(feed))
# We got an Edition for this book, but no LicensePool and no Work.
[edition] = imported_editions
eq_("Howards End", edition.title)
eq_([], imported_pools)
eq_([], imported_works)
def test_import_from_license_source(self):
# Instead of importing this data as though it came from the
# metadata wrangler, let's import it as though it came from the
# open-access content server.
feed = self.content_server_mini_feed
importer = OPDSImporter(
self._db, data_source_name=DataSource.OA_CONTENT_SERVER
)
imported_editions, imported_pools, imported_works, failures = (
importer.import_from_feed(feed)
)
# Two works have been created, because the content server
# actually tells you how to get copies of these books.
[crow, mouse] = sorted(imported_works, key=lambda x: x.title)
# Each work has one license pool.
[crow_pool] = crow.license_pools
[mouse_pool] = mouse.license_pools
# The OPDS importer sets the data source of the license pool
# to Project Gutenberg, since that's the authority that grants
# access to the book.
eq_(DataSource.GUTENBERG, mouse_pool.data_source.name)
# But the license pool's presentation edition has a data
# source associated with the Library Simplified open-access
# content server, since that's where the metadata comes from.
eq_(DataSource.OA_CONTENT_SERVER,
mouse_pool.presentation_edition.data_source.name
)
# Since the 'mouse' book came with an open-access link, the license
# pool delivery mechanism has been marked as open access.
eq_(True, mouse_pool.open_access)
eq_(RightsStatus.GENERIC_OPEN_ACCESS,
mouse_pool.delivery_mechanisms[0].rights_status.uri)
# The 'mouse' work has not been marked presentation-ready,
# because the OPDS importer was not told to make works
# presentation-ready as they're imported.
eq_(False, mouse_pool.work.presentation_ready)
# The OPDS feed didn't actually say where the 'crow' book
# comes from, but we did tell the importer to use the open access
# content server as the data source, so both a Work and a LicensePool
# were created, and their data source is the open access content server,
# not Project Gutenberg.
eq_(DataSource.OA_CONTENT_SERVER, crow_pool.data_source.name)
def test_import_and_make_presentation_ready(self):
# Now let's tell the OPDS importer to make works presentation-ready
# as soon as they're imported.
feed = self.content_server_mini_feed
importer = OPDSImporter(self._db,
data_source_name=DataSource.OA_CONTENT_SERVER)
imported_editions, imported_pools, imported_works, failures = (
importer.import_from_feed(feed,
immediately_presentation_ready=True))
[crow, mouse] = sorted(imported_works, key=lambda x: x.title)
# Both the 'crow' and the 'mouse' book had presentation-ready works created.
eq_(True, crow.presentation_ready)
eq_(True, mouse.presentation_ready)
def test_import_with_unrecognized_distributor_fails(self):
"""We get a book from the open-access content server but the license
comes from an unrecognized data source. We can't import the book
because we can't record its provenance accurately.
"""
feed = open(
os.path.join(self.resource_path, "unrecognized_distributor.opds")).read()
importer = OPDSImporter(
self._db,
data_source_name=DataSource.OA_CONTENT_SERVER
)
imported_editions, pools, works, failures = (
importer.import_from_feed(feed)
)
# No editions, licensepools, or works were imported.
eq_([], imported_editions)
eq_([], pools)
eq_([], works)
[failure] = failures.values()
eq_(True, failure.transient)
assert "Unrecognized circulation data source: Unknown Source" in failure.exception
def test_resources_are_mirrored_on_import(self):
svg = """<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="500">
<ellipse cx="50" cy="25" rx="50" ry="25" style="fill:blue;"/>
</svg>"""
http = DummyHTTPClient()
# The request to http://root/full-cover-image.png
# will result in a 404 error, and the image will not be mirrored.
http.queue_response(404, media_type="text/plain")
http.queue_response(
200,
content='I am 10557.epub.images',
media_type=Representation.EPUB_MEDIA_TYPE,
)
http.queue_response(200,
content=svg,
media_type=Representation.SVG_MEDIA_TYPE)
http.queue_response(200,
content='I am 10441.epub.images',
media_type=Representation.EPUB_MEDIA_TYPE)
s3 = DummyS3Uploader()
importer = OPDSImporter(self._db,
data_source_name=DataSource.OA_CONTENT_SERVER,
mirror=s3,
http_get=http.do_get)
imported_editions, pools, works, failures = (importer.import_from_feed(
self.content_server_mini_feed, feed_url='http://root'))
e1 = imported_editions[0]
e2 = imported_editions[1]
# The import process requested each remote resource in the
# order they appeared in the OPDS feed. The thumbnail
# image was not requested, since we were going to make our own
# thumbnail anyway.
eq_(http.requests, [
'http://www.gutenberg.org/ebooks/10441.epub.images',
'https://s3.amazonaws.com/book-covers.nypl.org/Gutenberg-Illustrated/10441/cover_10441_9.png',
'http://www.gutenberg.org/ebooks/10557.epub.images',
'http://root/full-cover-image.png',
])
[e1_oa_link, e1_image_link,
e1_description_link] = sorted(e1.primary_identifier.links,
key=lambda x: x.rel)
[e2_image_link, e2_oa_link] = e2.primary_identifier.links
# The two open-access links were mirrored to S3, as was the
# original SVG image and its PNG thumbnail. The PNG image was
# not mirrored because our attempt to download it resulted in
# a 404 error.
imported_representations = [
e1_oa_link.resource.representation,
e1_image_link.resource.representation,
e1_image_link.resource.representation.thumbnails[0],
e2_oa_link.resource.representation,
]
eq_(imported_representations, s3.uploaded)
eq_(4, len(s3.uploaded))
eq_("I am 10441.epub.images", s3.content[0])
eq_(svg, s3.content[1])
eq_("I am 10557.epub.images", s3.content[3])
# Each resource was 'mirrored' to an Amazon S3 bucket.
#
# The "mouse" book was mirrored to a bucket corresponding to
# Project Gutenberg, its data source.
#
# The images were mirrored to a bucket corresponding to the
# open-access content server, _their_ data source.
#
# The "crow" book was mirrored to a bucket corresponding to
# the open-access content source, the default data source used
# when no distributor was specified for a book.
url0 = 'http://s3.amazonaws.com/test.content.bucket/Gutenberg/Gutenberg%20ID/10441/The%20Green%20Mouse.epub.images'
url1 = u'http://s3.amazonaws.com/test.cover.bucket/Library%20Simplified%20Open%20Access%20Content%20Server/Gutenberg%20ID/10441/cover_10441_9.png'
url2 = u'http://s3.amazonaws.com/test.cover.bucket/scaled/300/Library%20Simplified%20Open%20Access%20Content%20Server/Gutenberg%20ID/10441/cover_10441_9.png'
url3 = 'http://s3.amazonaws.com/test.content.bucket/Library%20Simplified%20Open%20Access%20Content%20Server/Gutenberg%20ID/10557/Johnny%20Crow%27s%20Party.epub.images'
uploaded_urls = [x.mirror_url for x in s3.uploaded]
eq_([url0, url1, url2, url3], uploaded_urls)
# If we fetch the feed again, and the entries have been updated since the
# cutoff, but the content of the open access links hasn't changed, we won't mirror
# them again.
cutoff = datetime.datetime(2013, 1, 2, 16, 56, 40)
http.queue_response(304, media_type=Representation.EPUB_MEDIA_TYPE)
http.queue_response(304, media_type=Representation.SVG_MEDIA_TYPE)
http.queue_response(304, media_type=Representation.EPUB_MEDIA_TYPE)
imported_editions, pools, works, failures = (importer.import_from_feed(
self.content_server_mini_feed))
eq_([e1, e2], imported_editions)
# Nothing new has been uploaded
eq_(4, len(s3.uploaded))
# If the content has changed, it will be mirrored again.
http.queue_response(200,
content="I am a new version of 10557.epub.images",
media_type=Representation.EPUB_MEDIA_TYPE)
http.queue_response(200,
content=svg,
media_type=Representation.SVG_MEDIA_TYPE)
http.queue_response(200,
content="I am a new version of 10441.epub.images",
media_type=Representation.EPUB_MEDIA_TYPE)
imported_editions, pools, works, failures = (importer.import_from_feed(
self.content_server_mini_feed))
eq_([e1, e2], imported_editions)
eq_(8, len(s3.uploaded))
eq_("I am a new version of 10441.epub.images", s3.content[4])
eq_(svg, s3.content[5])
eq_("I am a new version of 10557.epub.images", s3.content[7])