def revoke_loan(self, patron, pin, licensepool):
"""Revoke a patron's loan for a book."""
loan = get_one(self._db,
Loan,
patron=patron,
license_pool=licensepool,
on_multiple='interchangeable')
if loan:
__transaction = self._db.begin_nested()
logging.info("In revoke_loan(), deleting loan #%d" % loan.id)
self._db.delete(loan)
__transaction.commit()
# Send out an analytics event to record the fact that
# a loan was revoked through the circulation
# manager.
Analytics.collect_event(
self._db,
licensepool,
CirculationEvent.CM_CHECKIN,
)
if not licensepool.open_access:
api = self.api_for_license_pool(licensepool)
try:
api.checkin(patron, pin, licensepool)
except NotCheckedOut, e:
# The book wasn't checked out in the first
# place. Everything's fine.
pass
class OneClickCirculationMonitor(CollectionMonitor):
"""Maintain LicensePools for OneClick titles.
Bibliographic data isn't inserted into new LicensePools until
we hear from the metadata wrangler.
"""
SERVICE_NAME = "OneClick CirculationMonitor"
DEFAULT_START_TIME = datetime.datetime(1970, 1, 1)
INTERVAL_SECONDS = 1200
DEFAULT_BATCH_SIZE = 50
PROTOCOL = ExternalIntegration.RB_DIGITAL
def __init__(self, _db, collection, batch_size=None, api_class=OneClickAPI,
api_class_kwargs={}):
super(OneClickCirculationMonitor, self).__init__(_db, collection)
self.batch_size = batch_size or self.DEFAULT_BATCH_SIZE
self.api = api_class(_db, self.collection, **api_class_kwargs)
self.bibliographic_coverage_provider = (
OneClickBibliographicCoverageProvider(
collection=self.collection, api_class=self.api,
)
)
self.analytics = Analytics(self._db)
def process_availability(self, media_type='ebook'):
# get list of all titles, with availability info
availability_list = self.api.get_ebook_availability_info(media_type=media_type)
item_count = 0
for availability in availability_list:
isbn = availability['isbn']
# boolean True/False value, not number of licenses
available = availability['availability']
medium = availability.get('mediaType')
license_pool, is_new, is_changed = self.api.update_licensepool_for_identifier(isbn, available, medium)
# Log a circulation event for this work.
if is_new:
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked)
item_count += 1
if item_count % self.batch_size == 0:
self._db.commit()
return item_count
def run(self):
super(OneClickCirculationMonitor, self).run()
def run_once(self, start, cutoff):
ebook_count = self.process_availability(media_type='ebook')
eaudio_count = self.process_availability(media_type='eaudio')
self.log.info("Processed %d ebooks and %d audiobooks.", ebook_count, eaudio_count)
def _collect_checkout_event(self, licensepool):
"""Collect an analytics event indicating the given LicensePool
was checked out via the circulation manager.
"""
Analytics.collect_event(
self._db,
licensepool,
CirculationEvent.CM_CHECKOUT,
)
def process_batch(self, identifiers):
identifiers_by_threem_id = dict()
threem_ids = set()
for identifier in identifiers:
threem_ids.add(identifier.identifier)
identifiers_by_threem_id[identifier.identifier] = identifier
identifiers_not_mentioned_by_threem = set(identifiers)
now = datetime.datetime.utcnow()
for circ in self.api.get_circulation_for(threem_ids):
if not circ:
continue
threem_id = circ[Identifier][Identifier.THREEM_ID]
identifier = identifiers_by_threem_id[threem_id]
identifiers_not_mentioned_by_threem.remove(identifier)
pool = identifier.licensed_through
if not pool:
# We don't have a license pool for this work. That
# shouldn't happen--how did we know about the
# identifier?--but it shouldn't be a big deal to
# create one.
pool, ignore = LicensePool.for_foreign_id(
self._db, self.data_source, identifier.type,
identifier.identifier)
# 3M books are never open-access.
pool.open_access = False
Analytics.collect_event(self._db, pool,
CirculationEvent.DISTRIBUTOR_TITLE_ADD,
now)
self.api.apply_circulation_information_to_licensepool(circ, pool)
# At this point there may be some license pools left over
# that 3M doesn't know about. This is a pretty reliable
# indication that we no longer own any licenses to the
# book.
for identifier in identifiers_not_mentioned_by_threem:
pool = identifier.licensed_through
if not pool:
continue
if pool.licenses_owned > 0:
if pool.presentation_edition:
self.log.warn("Removing %s (%s) from circulation",
pool.presentation_edition.title,
pool.presentation_edition.author)
else:
self.log.warn("Removing unknown work %s from circulation.",
identifier.identifier)
pool.licenses_owned = 0
pool.licenses_available = 0
pool.licenses_reserved = 0
pool.patrons_in_hold_queue = 0
pool.last_checked = now
def run_once(self, start, cutoff):
_db = self._db
added_books = 0
overdrive_data_source = DataSource.lookup(_db, DataSource.OVERDRIVE)
total_books = 0
consecutive_unchanged_books = 0
for i, book in enumerate(self.recently_changed_ids(start, cutoff)):
total_books += 1
if not total_books % 100:
self.log.info("%s books processed", total_books)
if not book:
continue
license_pool, is_new, is_changed = self.api.update_licensepool(
book)
# Log a circulation event for this work.
if is_new:
Analytics.collect_event(_db, license_pool,
CirculationEvent.DISTRIBUTOR_TITLE_ADD,
license_pool.last_checked)
_db.commit()
if is_changed:
consecutive_unchanged_books = 0
else:
consecutive_unchanged_books += 1
if (self.maximum_consecutive_unchanged_books
and consecutive_unchanged_books >=
self.maximum_consecutive_unchanged_books):
# We're supposed to stop this run after finding a
# run of books that have not changed, and we have
# in fact seen that many consecutive unchanged
# books.
self.log.info("Stopping at %d unchanged books.",
consecutive_unchanged_books)
break
if total_books:
self.log.info("Processed %d books total.", total_books)
def process_availability(self, media_type='ebook'):
# get list of all titles, with availability info
availability_list = self.api.get_ebook_availability_info(
media_type=media_type)
item_count = 0
for availability in availability_list:
isbn = availability['isbn']
# boolean True/False value, not number of licenses
available = availability['availability']
license_pool, is_new, is_changed = self.api.update_licensepool_for_identifier(
isbn, available)
# Log a circulation event for this work.
if is_new:
Analytics.collect_event(
self._db, license_pool,
CirculationEvent.DISTRIBUTOR_AVAILABILITY_NOTIFY,
license_pool.last_checked)
item_count += 1
if item_count % self.batch_size == 0:
self._db.commit()
return item_count
def test_collect_event(self):
# This will be a site-wide integration because it will have no
# associated libraries when the Analytics singleton is instantiated.
# the first time.
sitewide_integration, ignore = create(
self._db,
ExternalIntegration,
goal=ExternalIntegration.ANALYTICS_GOAL,
protocol=MOCK_PROTOCOL,
)
# This will be a per-library integration because it will have at least
# one associated library when the Analytics singleton is instantiated.
library_integration, ignore = create(
self._db,
ExternalIntegration,
goal=ExternalIntegration.ANALYTICS_GOAL,
protocol=MOCK_PROTOCOL,
)
library, ignore = create(self._db, Library, short_name="library")
library_integration.libraries += [library]
work = self._work(title="title", with_license_pool=True)
[lp] = work.license_pools
analytics = Analytics(self._db)
sitewide_provider = analytics.sitewide_providers[0]
library_provider = analytics.library_providers[library.id][0]
analytics.collect_event(self._default_library, lp,
CirculationEvent.DISTRIBUTOR_CHECKIN, None)
# The sitewide provider was called.
assert 1 == sitewide_provider.count
assert CirculationEvent.DISTRIBUTOR_CHECKIN == sitewide_provider.event_type
# The library provider wasn't called, since the event was for a different library.
assert 0 == library_provider.count
analytics.collect_event(library, lp,
CirculationEvent.DISTRIBUTOR_CHECKIN, None)
# Now both providers were called, since the event was for the library provider's library.
assert 2 == sitewide_provider.count
assert 1 == library_provider.count
assert CirculationEvent.DISTRIBUTOR_CHECKIN == library_provider.event_type
# Here's an event that we couldn't associate with any
# particular library.
analytics.collect_event(None, lp,
CirculationEvent.DISTRIBUTOR_CHECKOUT, None)
# It's counted as a sitewide event, but not as a library event.
assert 3 == sitewide_provider.count
assert 1 == library_provider.count
class BibliothecaCirculationSweep(IdentifierSweepMonitor):
"""Check on the current circulation status of each Bibliotheca book in our
collection.
In some cases this will lead to duplicate events being logged,
because this monitor and the main Bibliotheca circulation monitor will
count the same event. However it will greatly improve our current
view of our Bibliotheca circulation, which is more important.
"""
SERVICE_NAME = "Bibliotheca Circulation Sweep"
DEFAULT_BATCH_SIZE = 25
PROTOCOL = ExternalIntegration.BIBLIOTHECA
def __init__(self, _db, collection, api_class=BibliothecaAPI, **kwargs):
_db = Session.object_session(collection)
super(BibliothecaCirculationSweep,
self).__init__(_db, collection, **kwargs)
if isinstance(api_class, BibliothecaAPI):
self.api = api_class
else:
self.api = api_class(_db, collection)
self.analytics = Analytics(_db)
def process_items(self, identifiers):
identifiers_by_bibliotheca_id = dict()
bibliotheca_ids = set()
for identifier in identifiers:
bibliotheca_ids.add(identifier.identifier)
identifiers_by_bibliotheca_id[identifier.identifier] = identifier
identifiers_not_mentioned_by_bibliotheca = set(identifiers)
now = datetime.datetime.utcnow()
for circ in self.api.get_circulation_for(bibliotheca_ids):
if not circ:
continue
self._process_circulation_data(
circ,
identifiers_by_bibliotheca_id,
identifiers_not_mentioned_by_bibliotheca,
)
# At this point there may be some license pools left over
# that Bibliotheca doesn't know about. This is a pretty reliable
# indication that we no longer own any licenses to the
# book.
for identifier in identifiers_not_mentioned_by_bibliotheca:
pools = [
lp for lp in identifier.licensed_through
if lp.data_source.name == DataSource.BIBLIOTHECA
and lp.collection == self.collection
]
if not pools:
continue
for pool in pools:
if pool.licenses_owned > 0:
if pool.presentation_edition:
self.log.warn("Removing %s (%s) from circulation",
pool.presentation_edition.title,
pool.presentation_edition.author)
else:
self.log.warn(
"Removing unknown work %s from circulation.",
identifier.identifier)
pool.update_availability(0, 0, 0, 0, self.analytics)
pool.last_checked = now
def _process_circulation_data(self, circ, identifiers_by_bibliotheca_id,
identifiers_not_mentioned_by_bibliotheca):
"""Process a single CirculationData object retrieved from
Bibliotheca.
"""
bibliotheca_id = circ[Identifier][Identifier.BIBLIOTHECA_ID]
identifier = identifiers_by_bibliotheca_id[bibliotheca_id]
identifiers_not_mentioned_by_bibliotheca.remove(identifier)
pools = [
lp for lp in identifier.licensed_through
if lp.data_source.name == DataSource.BIBLIOTHECA
and lp.collection == self.collection
]
if not pools:
# We don't have a license pool for this work. That
# shouldn't happen--how did we know about the
# identifier?--but it shouldn't be a big deal to
# create one.
pool, ignore = LicensePool.for_foreign_id(
self._db,
self.collection.data_source,
identifier.type,
identifier.identifier,
collection=self.collection)
# Bibliotheca books are never open-access.
pool.open_access = False
for library in self.collection.libraries:
self.analytics.collect_event(
library, pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD,
datetime.datetime.utcnow())
else:
[pool] = pools
self.api.apply_circulation_information_to_licensepool(
circ, pool, self.analytics)
class EnkiImport(CollectionMonitor):
"""Import content from Enki that we don't yet have in our collection
"""
SERVICE_NAME = "Enki Circulation Monitor"
INTERVAL_SECONDS = 500
PROTOCOL = EnkiAPI.ENKI_EXTERNAL
DEFAULT_BATCH_SIZE = 100
FIVE_MINUTES = datetime.timedelta(minutes=5)
def __init__(self, _db, collection, api_class=EnkiAPI):
"""Constructor."""
super(EnkiImport, self).__init__(_db, collection)
self._db = _db
self.api = api_class(_db, collection)
self.collection_id = collection.id
self.analytics = Analytics(_db)
self.bibliographic_coverage_provider = (
EnkiBibliographicCoverageProvider(collection, api_class=self.api))
@property
def collection(self):
return Collection.by_id(self._db, id=self.collection_id)
def recently_changed_ids(self, start, cutoff):
return self.api.recently_changed_ids(start, cutoff)
def run_once(self, start, cutoff):
# Give us five minutes of overlap because it's very important
# we don't miss anything.
since = start - self.FIVE_MINUTES
id_start = 0
while True:
availability = self.api.availability(since=since,
strt=id_start,
qty=self.DEFAULT_BATCH_SIZE)
if availability.status_code != 200:
self.log.error(
"Could not contact Enki server for content availability. Status: %d",
availability.status_code)
content = availability.content
count = 0
for bibliographic, circulation in BibliographicParser(
).process_all(content):
self.process_book(bibliographic, circulation)
count += 1
if count == 0:
break
self._db.commit()
id_start += self.DEFAULT_BATCH_SIZE
def process_book(self, bibliographic, availability):
license_pool, new_license_pool = availability.license_pool(
self._db, self.collection)
now = datetime.datetime.utcnow()
edition, new_edition = bibliographic.edition(self._db)
license_pool.edition = edition
policy = ReplacementPolicy(
identifiers=False,
subjects=True,
contributions=True,
formats=True,
)
availability.apply(
self._db,
license_pool.collection,
replace=policy,
)
if new_edition:
bibliographic.apply(edition, self.collection, replace=policy)
if new_license_pool or new_edition:
# At this point we have done work equivalent to that done by
# the EnkiBibliographicCoverageProvider. Register that the
# work has been done so we don't have to do it again.
identifier = edition.primary_identifier
self.bibliographic_coverage_provider.handle_success(identifier)
self.bibliographic_coverage_provider.add_coverage_record_for(
identifier)
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool,
CirculationEvent.DISTRIBUTOR_TITLE_ADD, now)
return edition, license_pool
class OverdriveCirculationMonitor(CollectionMonitor):
"""Maintain LicensePools for recently changed Overdrive titles. Create
basic Editions for any new LicensePools that show up.
"""
SERVICE_NAME = "Overdrive Circulation Monitor"
INTERVAL_SECONDS = 500
PROTOCOL = ExternalIntegration.OVERDRIVE
# Report successful completion upon finding this number of
# consecutive books in the Overdrive results whose LicensePools
# haven't changed since last time. Overdrive results are not in
# strict chronological order, but if you see 100 consecutive books
# that haven't changed, you're probably done.
MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS = None
def __init__(self, _db, collection, api_class=OverdriveAPI):
"""Constructor."""
super(OverdriveCirculationMonitor, self).__init__(_db, collection)
self.api = api_class(_db, collection)
self.maximum_consecutive_unchanged_books = (
self.MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS)
self.analytics = Analytics(_db)
def recently_changed_ids(self, start, cutoff):
return self.api.recently_changed_ids(start, cutoff)
def run_once(self, start, cutoff):
_db = self._db
added_books = 0
overdrive_data_source = DataSource.lookup(_db, DataSource.OVERDRIVE)
total_books = 0
consecutive_unchanged_books = 0
for i, book in enumerate(self.recently_changed_ids(start, cutoff)):
total_books += 1
if not total_books % 100:
self.log.info("%s books processed", total_books)
if not book:
continue
license_pool, is_new, is_changed = self.api.update_licensepool(
book)
# Log a circulation event for this work.
if is_new:
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool,
CirculationEvent.DISTRIBUTOR_TITLE_ADD,
license_pool.last_checked)
_db.commit()
if is_changed:
consecutive_unchanged_books = 0
else:
consecutive_unchanged_books += 1
if (self.maximum_consecutive_unchanged_books
and consecutive_unchanged_books >=
self.maximum_consecutive_unchanged_books):
# We're supposed to stop this run after finding a
# run of books that have not changed, and we have
# in fact seen that many consecutive unchanged
# books.
self.log.info("Stopping at %d unchanged books.",
consecutive_unchanged_books)
break
if total_books:
self.log.info("Processed %d books total.", total_books)
class OverdriveCirculationMonitor(CollectionMonitor, TimelineMonitor):
"""Maintain LicensePools for recently changed Overdrive titles. Create
basic Editions for any new LicensePools that show up.
"""
SERVICE_NAME = "Overdrive Circulation Monitor"
PROTOCOL = ExternalIntegration.OVERDRIVE
OVERLAP = datetime.timedelta(minutes=1)
# Report successful completion upon finding this number of
# consecutive books in the Overdrive results whose LicensePools
# haven't changed since last time. Overdrive results are not in
# strict chronological order, but if you see 100 consecutive books
# that haven't changed, you're probably done.
MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS = None
def __init__(self, _db, collection, api_class=OverdriveAPI):
"""Constructor."""
super(OverdriveCirculationMonitor, self).__init__(_db, collection)
self.api = api_class(_db, collection)
self.maximum_consecutive_unchanged_books = (
self.MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS
)
self.analytics = Analytics(_db)
def recently_changed_ids(self, start, cutoff):
return self.api.recently_changed_ids(start, cutoff)
def catch_up_from(self, start, cutoff, progress):
"""Find Overdrive books that changed recently.
:progress: A TimestampData representing the time previously
covered by this Monitor.
"""
_db = self._db
added_books = 0
overdrive_data_source = DataSource.lookup(
_db, DataSource.OVERDRIVE)
total_books = 0
consecutive_unchanged_books = 0
# Ask for changes between the last time covered by the Monitor
# and the current time.
for i, book in enumerate(self.recently_changed_ids(start, cutoff)):
total_books += 1
if not total_books % 100:
self.log.info("%s books processed", total_books)
if not book:
continue
license_pool, is_new, is_changed = self.api.update_licensepool(book)
# Log a circulation event for this work.
if is_new:
for library in self.collection.libraries:
self.analytics.collect_event(
library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked)
_db.commit()
if is_changed:
consecutive_unchanged_books = 0
else:
consecutive_unchanged_books += 1
if (self.maximum_consecutive_unchanged_books
and consecutive_unchanged_books >=
self.maximum_consecutive_unchanged_books):
# We're supposed to stop this run after finding a
# run of books that have not changed, and we have
# in fact seen that many consecutive unchanged
# books.
self.log.info("Stopping at %d unchanged books.",
consecutive_unchanged_books)
break
progress.achievements = "Books processed: %d." % total_books
def fulfill(self,
patron,
pin,
licensepool,
delivery_mechanism,
sync_on_failure=True):
"""Fulfil a book that a patron has previously checked out.
:param delivery_mechanism: A LicensePoolDeliveryMechanism
explaining how the patron wants the book to be delivered. If
the book has previously been delivered through some other
mechanism, this parameter is ignored and the previously used
mechanism takes precedence.
:return: A FulfillmentInfo object.
"""
fulfillment = None
loan = get_one(self._db,
Loan,
patron=patron,
license_pool=licensepool,
on_multiple='interchangeable')
if not loan:
if sync_on_failure:
# Sync and try again.
self.sync_bookshelf(patron, pin)
return self.fulfill(patron,
pin,
licensepool=licensepool,
delivery_mechanism=delivery_mechanism,
sync_on_failure=False)
else:
raise NoActiveLoan(
_("Cannot find your active loan for this work."))
if loan.fulfillment is not None and loan.fulfillment != delivery_mechanism and not delivery_mechanism.delivery_mechanism.is_streaming:
raise DeliveryMechanismConflict(
_("You already fulfilled this loan as %(loan_delivery_mechanism)s, you can't also do it as %(requested_delivery_mechanism)s",
loan_delivery_mechanism=loan.fulfillment.delivery_mechanism.
name,
requested_delivery_mechanism=delivery_mechanism.
delivery_mechanism.name))
if licensepool.open_access:
fulfillment = self.fulfill_open_access(
licensepool, delivery_mechanism.delivery_mechanism)
else:
api = self.api_for_license_pool(licensepool)
internal_format = api.internal_format(delivery_mechanism)
fulfillment = api.fulfill(patron, pin, licensepool,
internal_format)
if not fulfillment or not (fulfillment.content_link
or fulfillment.content):
raise NoAcceptableFormat()
# Send out an analytics event to record the fact that
# a fulfillment was initiated through the circulation
# manager.
Analytics.collect_event(
self._db,
licensepool,
CirculationEvent.CM_FULFILL,
)
# Make sure the delivery mechanism we just used is associated
# with the loan.
if loan.fulfillment is None and not delivery_mechanism.delivery_mechanism.is_streaming:
__transaction = self._db.begin_nested()
loan.fulfillment = delivery_mechanism
__transaction.commit()
return fulfillment
None, None)
# It's pretty rare that we'd go from having a loan for a book
# to needing to put it on hold, but we do check for that case.
__transaction = self._db.begin_nested()
hold, is_new = licensepool.on_hold_to(patron, hold_info.start_date
or now, hold_info.end_date,
hold_info.hold_position)
if hold and is_new:
# Send out an analytics event to record the fact that
# a hold was initiated through the circulation
# manager.
Analytics.collect_event(
self._db,
licensepool,
CirculationEvent.CM_HOLD_PLACE,
)
if existing_loan:
self._db.delete(existing_loan)
__transaction.commit()
return None, hold, is_new
def _collect_checkout_event(self, licensepool):
"""Collect an analytics event indicating the given LicensePool
was checked out via the circulation manager.
"""
Analytics.collect_event(
self._db,
licensepool,
