def _reap(self, identifier):
"""Update our local circulation information to reflect the fact that
the identified book has been removed from the remote
collection.
"""
collection = self.collection
pool = identifier.licensed_through_collection(collection)
if not pool:
self.log.warn(
"Was about to reap %r but no local license pool in this collection.",
identifier)
return
if pool.licenses_owned == 0:
# Already reaped.
return
self.log.info("Reaping %r", identifier)
availability = CirculationData(
data_source=pool.data_source,
primary_identifier=identifier,
licenses_owned=0,
licenses_available=0,
licenses_reserved=0,
patrons_in_hold_queue=0,
)
availability.apply(self._db, collection,
ReplacementPolicy.from_license_source(self._db))
def update_licensepools_for_identifiers(self, identifiers):
"""Update availability information for a list of books.
If the book has never been seen before, a new LicensePool
will be created for the book.
The book's LicensePool will be updated with current
circulation information.
"""
identifier_strings = self.create_identifier_strings(identifiers)
response = self.availability(title_ids=identifier_strings)
collection = self.collection
parser = BibliographicParser(collection)
remainder = set(identifiers)
for bibliographic, availability in parser.process_all(
response.content):
identifier, is_new = bibliographic.primary_identifier.load(
self._db)
if identifier in remainder:
remainder.remove(identifier)
pool, is_new = availability.license_pool(self._db, collection)
availability.apply(self._db, pool.collection)
# We asked Axis about n books. It sent us n-k responses. Those
# k books are the identifiers in `remainder`. These books have
# been removed from the collection without us being notified.
for removed_identifier in remainder:
pool = identifier.licensed_through_collection(self.collection)
if not pool:
self.log.warn(
"Was about to reap %r but no local license pool in this collection.",
removed_identifier)
continue
if pool.licenses_owned == 0:
# Already reaped.
continue
self.log.info("Reaping %r", removed_identifier)
availability = CirculationData(
data_source=pool.data_source,
primary_identifier=removed_identifier,
licenses_owned=0,
licenses_available=0,
licenses_reserved=0,
patrons_in_hold_queue=0,
)
availability.apply(pool,
ReplacementPolicy.from_license_source(self._db))
def process_item(self, identifier):
self.log.debug(
"Seeing if %s needs reaping", identifier.identifier
)
metadata = self.api.get_item(identifier.identifier)
if metadata:
# This title is still in the collection. Do nothing.
return
# Get this collection's license pool for this identifier.
# We'll reap it by setting its licenses_owned to 0.
pool = identifier.licensed_through_collection(self.collection)
if not pool or pool.licenses_owned == 0:
# It's already been reaped.
return
if pool.presentation_edition:
self.log.warn(
"Removing %r from circulation",
pool.presentation_edition
)
else:
self.log.warn(
"Removing unknown title %s from circulation.",
identifier.identifier
)
now = datetime.datetime.utcnow()
circulationdata = CirculationData(
data_source=DataSource.ENKI,
primary_identifier= IdentifierData(
identifier.type, identifier.identifier
),
licenses_owned = 0,
licenses_available = 0,
patrons_in_hold_queue = 0,
last_checked = now
)
circulationdata.apply(
self._db,
self.collection,
replace=ReplacementPolicy.from_license_source(self._db)
)
return circulationdata
def process_item(self, identifier):
self.log.debug(
"Seeing if %s needs reaping", identifier.identifier
)
metadata = self.api.get_item(identifier.identifier)
if metadata:
# This title is still in the collection. Do nothing.
return
# Get this collection's license pool for this identifier.
# We'll reap it by setting its licenses_owned to 0.
pool = identifier.licensed_through_collection(self.collection)
if not pool or pool.licenses_owned == 0:
# It's already been reaped.
return
if pool.presentation_edition:
self.log.warn(
"Removing %r from circulation",
pool.presentation_edition
)
else:
self.log.warn(
"Removing unknown title %s from circulation.",
identifier.identifier
)
now = datetime.datetime.utcnow()
circulationdata = CirculationData(
data_source=DataSource.ENKI,
primary_identifier= IdentifierData(
identifier.type, identifier.identifier
),
licenses_owned = 0,
licenses_available = 0,
patrons_in_hold_queue = 0,
last_checked = now
)
circulationdata.apply(
self._db,
self.collection,
replace=ReplacementPolicy.from_license_source(self._db)
)
return circulationdata
def update_formats(self, licensepool):
"""Update the format information for a single book.
Incidentally updates the metadata, just in case Overdrive has
changed it.
"""
info = self.metadata_lookup(licensepool.identifier)
metadata = OverdriveRepresentationExtractor.book_info_to_metadata(
info, include_bibliographic=True, include_formats=True)
if not metadata:
# No work to be done.
return
edition, ignore = self._edition(licensepool)
replace = ReplacementPolicy.from_license_source(self._db)
metadata.apply(edition, self.collection, replace=replace)
def update_formats(self, licensepool):
"""Update the format information for a single book.
Incidentally updates the metadata, just in case Overdrive has
changed it.
"""
info = self.metadata_lookup(licensepool.identifier)
metadata = OverdriveRepresentationExtractor.book_info_to_metadata(
info, include_bibliographic=True, include_formats=True)
if not metadata:
# No work to be done.
return
edition, ignore = self._edition(licensepool)
replace = ReplacementPolicy.from_license_source(self._db)
metadata.apply(edition, self.collection, replace=replace)
def reaper_request(self, identifier):
self.log.debug ("Checking availability for " + str(identifier.identifier))
now = datetime.datetime.utcnow()
url = str(self.base_url) + str(self.item_endpoint)
args = dict()
args['method'] = "getItem"
args['recordid'] = identifier.identifier
args['size'] = "small"
args['lib'] = self.library_id
response = self.request(url, method='get', params=args)
try:
# If a book doesn't exist in Enki, we'll just get an HTML page saying we did something wrong.
data = json.loads(response.content)
self.log.debug ("Keeping existing book: " + str(identifier))
except:
# Get this collection's license pool for this identifier.
pool = identifier.licensed_through_collection(self.collection)
if pool and (pool.licenses_owned > 0):
if pool.presentation_edition:
self.log.warn("Removing %s (%s) from circulation",
pool.presentation_edition.title, pool.presentation_edition.author)
else:
self.log.warn(
"Removing unknown work %s from circulation.",
identifier.identifier
)
circulationdata = CirculationData(
data_source=DataSource.ENKI,
primary_identifier= IdentifierData(EnkiAPI.ENKI_ID, identifier.identifier),
licenses_owned = 0,
licenses_available = 0,
patrons_in_hold_queue = 0,
last_checked = now
)
circulationdata.apply(
self._db,
self.collection,
replace=ReplacementPolicy.from_license_source(self._db)
)
return circulationdata
def test_work_from_metadata(self):
"""Validate the ability to create a new Work from appropriate metadata.
"""
class Mock(MockDirectoryImportScript):
"""In this test we need to verify that annotate_metadata
was called but did nothing.
"""
def annotate_metadata(self, metadata, *args, **kwargs):
metadata.annotated = True
return super(Mock, self).annotate_metadata(
metadata, *args, **kwargs
)
identifier = IdentifierData(Identifier.GUTENBERG_ID, "1003")
identifier_obj, ignore = identifier.load(self._db)
metadata = Metadata(
DataSource.GUTENBERG,
primary_identifier=identifier,
title=u"A book"
)
metadata.annotated = False
datasource = DataSource.lookup(self._db, DataSource.GUTENBERG)
policy = ReplacementPolicy.from_license_source(self._db)
mirror = MockS3Uploader()
policy.mirror = mirror
# Here, work_from_metadata calls annotate_metadata, but does
# not actually import anything because there are no files 'on
# disk' and thus no way to actually get the book.
collection = self._default_collection
args = (collection, metadata, policy, "cover directory",
"ebook directory", RightsStatus.CC0)
script = Mock(self._db)
eq_(None, script.work_from_metadata(*args))
eq_(True, metadata.annotated)
# Now let's try it with some files 'on disk'.
with open(self.sample_cover_path('test-book-cover.png')) as fh:
image = fh.read()
mock_filesystem = {
'cover directory' : (
'cover.jpg', Representation.JPEG_MEDIA_TYPE, image
),
'ebook directory' : (
'book.epub', Representation.EPUB_MEDIA_TYPE, "I'm an EPUB."
)
}
script = MockDirectoryImportScript(
self._db, mock_filesystem=mock_filesystem
)
work = script.work_from_metadata(*args)
# We have created a book. It has a cover image, which has a
# thumbnail.
eq_("A book", work.title)
assert work.cover_full_url.endswith(
'/test.cover.bucket/Gutenberg/Gutenberg+ID/1003/1003.jpg'
)
assert work.cover_thumbnail_url.endswith(
'/test.cover.bucket/scaled/300/Gutenberg/Gutenberg+ID/1003/1003.png'
)
[pool] = work.license_pools
assert pool.open_access_download_url.endswith(
'/test.content.bucket/Gutenberg/Gutenberg+ID/1003/A+book.epub'
)
eq_(RightsStatus.CC0,
pool.delivery_mechanisms[0].rights_status.uri)
# The mock S3Uploader has a record of 'uploading' all these files
# to S3.
epub, full, thumbnail = mirror.uploaded
eq_(epub.url, pool.open_access_download_url)
eq_(full.url, work.cover_full_url)
eq_(thumbnail.url, work.cover_thumbnail_url)
# The EPUB Representation was cleared out after the upload, to
# save database space.
eq_("I'm an EPUB.", mirror.content[0])
eq_(None, epub.content)
def test_work_from_metadata(self):
"""Validate the ability to create a new Work from appropriate metadata.
"""
class Mock(MockDirectoryImportScript):
"""In this test we need to verify that annotate_metadata
was called but did nothing.
"""
def annotate_metadata(self, metadata, *args, **kwargs):
metadata.annotated = True
return super(Mock, self).annotate_metadata(
metadata, *args, **kwargs
)
identifier = IdentifierData(Identifier.GUTENBERG_ID, "1003")
identifier_obj, ignore = identifier.load(self._db)
metadata = Metadata(
DataSource.GUTENBERG,
primary_identifier=identifier,
title=u"A book"
)
metadata.annotated = False
datasource = DataSource.lookup(self._db, DataSource.GUTENBERG)
policy = ReplacementPolicy.from_license_source(self._db)
mirror = MockS3Uploader()
policy.mirror = mirror
# Here, work_from_metadata calls annotate_metadata, but does
# not actually import anything because there are no files 'on
# disk' and thus no way to actually get the book.
collection = self._default_collection
args = (collection, metadata, policy, "cover directory",
"ebook directory", RightsStatus.CC0)
script = Mock(self._db)
eq_(None, script.work_from_metadata(*args))
eq_(True, metadata.annotated)
# Now let's try it with some files 'on disk'.
with open(self.sample_cover_path('test-book-cover.png')) as fh:
image = fh.read()
mock_filesystem = {
'cover directory' : (
'cover.jpg', Representation.JPEG_MEDIA_TYPE, image
),
'ebook directory' : (
'book.epub', Representation.EPUB_MEDIA_TYPE, "I'm an EPUB."
)
}
script = MockDirectoryImportScript(
self._db, mock_filesystem=mock_filesystem
)
work = script.work_from_metadata(*args)
# We have created a book. It has a cover image, which has a
# thumbnail.
eq_("A book", work.title)
assert work.cover_full_url.endswith(
'/test.cover.bucket/Gutenberg/Gutenberg+ID/1003/1003.jpg'
)
assert work.cover_thumbnail_url.endswith(
'/test.cover.bucket/scaled/300/Gutenberg/Gutenberg+ID/1003/1003.png'
)
[pool] = work.license_pools
assert pool.open_access_download_url.endswith(
'/test.content.bucket/Gutenberg/Gutenberg+ID/1003/A+book.epub'
)
eq_(RightsStatus.CC0,
pool.delivery_mechanisms[0].rights_status.uri)
# The mock S3Uploader has a record of 'uploading' all these files
# to S3.
epub, full, thumbnail = mirror.uploaded
eq_(epub.url, pool.open_access_download_url)
eq_(full.url, work.cover_full_url)
eq_(thumbnail.url, work.cover_thumbnail_url)
# The EPUB Representation was cleared out after the upload, to
# save database space.
eq_("I'm an EPUB.", mirror.content[0])
eq_(None, epub.content)
