def revoke_loan(self, patron, pin, licensepool): """Revoke a patron's loan for a book.""" loan = get_one(self._db, Loan, patron=patron, license_pool=licensepool, on_multiple='interchangeable') if loan: __transaction = self._db.begin_nested() logging.info("In revoke_loan(), deleting loan #%d" % loan.id) self._db.delete(loan) __transaction.commit() # Send out an analytics event to record the fact that # a loan was revoked through the circulation # manager. Analytics.collect_event( self._db, licensepool, CirculationEvent.CM_CHECKIN, ) if not licensepool.open_access: api = self.api_for_license_pool(licensepool) try: api.checkin(patron, pin, licensepool) except NotCheckedOut, e: # The book wasn't checked out in the first # place. Everything's fine. pass
class OneClickCirculationMonitor(CollectionMonitor): """Maintain LicensePools for OneClick titles. Bibliographic data isn't inserted into new LicensePools until we hear from the metadata wrangler. """ SERVICE_NAME = "OneClick CirculationMonitor" DEFAULT_START_TIME = datetime.datetime(1970, 1, 1) INTERVAL_SECONDS = 1200 DEFAULT_BATCH_SIZE = 50 PROTOCOL = ExternalIntegration.RB_DIGITAL def __init__(self, _db, collection, batch_size=None, api_class=OneClickAPI, api_class_kwargs={}): super(OneClickCirculationMonitor, self).__init__(_db, collection) self.batch_size = batch_size or self.DEFAULT_BATCH_SIZE self.api = api_class(_db, self.collection, **api_class_kwargs) self.bibliographic_coverage_provider = ( OneClickBibliographicCoverageProvider( collection=self.collection, api_class=self.api, ) ) self.analytics = Analytics(self._db) def process_availability(self, media_type='ebook'): # get list of all titles, with availability info availability_list = self.api.get_ebook_availability_info(media_type=media_type) item_count = 0 for availability in availability_list: isbn = availability['isbn'] # boolean True/False value, not number of licenses available = availability['availability'] medium = availability.get('mediaType') license_pool, is_new, is_changed = self.api.update_licensepool_for_identifier(isbn, available, medium) # Log a circulation event for this work. if is_new: for library in self.collection.libraries: self.analytics.collect_event( library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked) item_count += 1 if item_count % self.batch_size == 0: self._db.commit() return item_count def run(self): super(OneClickCirculationMonitor, self).run() def run_once(self, start, cutoff): ebook_count = self.process_availability(media_type='ebook') eaudio_count = self.process_availability(media_type='eaudio') self.log.info("Processed %d ebooks and %d audiobooks.", ebook_count, eaudio_count)
def _collect_checkout_event(self, licensepool): """Collect an analytics event indicating the given LicensePool was checked out via the circulation manager. """ Analytics.collect_event( self._db, licensepool, CirculationEvent.CM_CHECKOUT, )
def process_batch(self, identifiers): identifiers_by_threem_id = dict() threem_ids = set() for identifier in identifiers: threem_ids.add(identifier.identifier) identifiers_by_threem_id[identifier.identifier] = identifier identifiers_not_mentioned_by_threem = set(identifiers) now = datetime.datetime.utcnow() for circ in self.api.get_circulation_for(threem_ids): if not circ: continue threem_id = circ[Identifier][Identifier.THREEM_ID] identifier = identifiers_by_threem_id[threem_id] identifiers_not_mentioned_by_threem.remove(identifier) pool = identifier.licensed_through if not pool: # We don't have a license pool for this work. That # shouldn't happen--how did we know about the # identifier?--but it shouldn't be a big deal to # create one. pool, ignore = LicensePool.for_foreign_id( self._db, self.data_source, identifier.type, identifier.identifier) # 3M books are never open-access. pool.open_access = False Analytics.collect_event(self._db, pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, now) self.api.apply_circulation_information_to_licensepool(circ, pool) # At this point there may be some license pools left over # that 3M doesn't know about. This is a pretty reliable # indication that we no longer own any licenses to the # book. for identifier in identifiers_not_mentioned_by_threem: pool = identifier.licensed_through if not pool: continue if pool.licenses_owned > 0: if pool.presentation_edition: self.log.warn("Removing %s (%s) from circulation", pool.presentation_edition.title, pool.presentation_edition.author) else: self.log.warn("Removing unknown work %s from circulation.", identifier.identifier) pool.licenses_owned = 0 pool.licenses_available = 0 pool.licenses_reserved = 0 pool.patrons_in_hold_queue = 0 pool.last_checked = now
def run_once(self, start, cutoff): _db = self._db added_books = 0 overdrive_data_source = DataSource.lookup(_db, DataSource.OVERDRIVE) total_books = 0 consecutive_unchanged_books = 0 for i, book in enumerate(self.recently_changed_ids(start, cutoff)): total_books += 1 if not total_books % 100: self.log.info("%s books processed", total_books) if not book: continue license_pool, is_new, is_changed = self.api.update_licensepool( book) # Log a circulation event for this work. if is_new: Analytics.collect_event(_db, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked) _db.commit() if is_changed: consecutive_unchanged_books = 0 else: consecutive_unchanged_books += 1 if (self.maximum_consecutive_unchanged_books and consecutive_unchanged_books >= self.maximum_consecutive_unchanged_books): # We're supposed to stop this run after finding a # run of books that have not changed, and we have # in fact seen that many consecutive unchanged # books. self.log.info("Stopping at %d unchanged books.", consecutive_unchanged_books) break if total_books: self.log.info("Processed %d books total.", total_books)
def process_availability(self, media_type='ebook'): # get list of all titles, with availability info availability_list = self.api.get_ebook_availability_info( media_type=media_type) item_count = 0 for availability in availability_list: isbn = availability['isbn'] # boolean True/False value, not number of licenses available = availability['availability'] license_pool, is_new, is_changed = self.api.update_licensepool_for_identifier( isbn, available) # Log a circulation event for this work. if is_new: Analytics.collect_event( self._db, license_pool, CirculationEvent.DISTRIBUTOR_AVAILABILITY_NOTIFY, license_pool.last_checked) item_count += 1 if item_count % self.batch_size == 0: self._db.commit() return item_count
def test_collect_event(self): # This will be a site-wide integration because it will have no # associated libraries when the Analytics singleton is instantiated. # the first time. sitewide_integration, ignore = create( self._db, ExternalIntegration, goal=ExternalIntegration.ANALYTICS_GOAL, protocol=MOCK_PROTOCOL, ) # This will be a per-library integration because it will have at least # one associated library when the Analytics singleton is instantiated. library_integration, ignore = create( self._db, ExternalIntegration, goal=ExternalIntegration.ANALYTICS_GOAL, protocol=MOCK_PROTOCOL, ) library, ignore = create(self._db, Library, short_name="library") library_integration.libraries += [library] work = self._work(title="title", with_license_pool=True) [lp] = work.license_pools analytics = Analytics(self._db) sitewide_provider = analytics.sitewide_providers[0] library_provider = analytics.library_providers[library.id][0] analytics.collect_event(self._default_library, lp, CirculationEvent.DISTRIBUTOR_CHECKIN, None) # The sitewide provider was called. assert 1 == sitewide_provider.count assert CirculationEvent.DISTRIBUTOR_CHECKIN == sitewide_provider.event_type # The library provider wasn't called, since the event was for a different library. assert 0 == library_provider.count analytics.collect_event(library, lp, CirculationEvent.DISTRIBUTOR_CHECKIN, None) # Now both providers were called, since the event was for the library provider's library. assert 2 == sitewide_provider.count assert 1 == library_provider.count assert CirculationEvent.DISTRIBUTOR_CHECKIN == library_provider.event_type # Here's an event that we couldn't associate with any # particular library. analytics.collect_event(None, lp, CirculationEvent.DISTRIBUTOR_CHECKOUT, None) # It's counted as a sitewide event, but not as a library event. assert 3 == sitewide_provider.count assert 1 == library_provider.count
class BibliothecaCirculationSweep(IdentifierSweepMonitor): """Check on the current circulation status of each Bibliotheca book in our collection. In some cases this will lead to duplicate events being logged, because this monitor and the main Bibliotheca circulation monitor will count the same event. However it will greatly improve our current view of our Bibliotheca circulation, which is more important. """ SERVICE_NAME = "Bibliotheca Circulation Sweep" DEFAULT_BATCH_SIZE = 25 PROTOCOL = ExternalIntegration.BIBLIOTHECA def __init__(self, _db, collection, api_class=BibliothecaAPI, **kwargs): _db = Session.object_session(collection) super(BibliothecaCirculationSweep, self).__init__(_db, collection, **kwargs) if isinstance(api_class, BibliothecaAPI): self.api = api_class else: self.api = api_class(_db, collection) self.analytics = Analytics(_db) def process_items(self, identifiers): identifiers_by_bibliotheca_id = dict() bibliotheca_ids = set() for identifier in identifiers: bibliotheca_ids.add(identifier.identifier) identifiers_by_bibliotheca_id[identifier.identifier] = identifier identifiers_not_mentioned_by_bibliotheca = set(identifiers) now = datetime.datetime.utcnow() for circ in self.api.get_circulation_for(bibliotheca_ids): if not circ: continue self._process_circulation_data( circ, identifiers_by_bibliotheca_id, identifiers_not_mentioned_by_bibliotheca, ) # At this point there may be some license pools left over # that Bibliotheca doesn't know about. This is a pretty reliable # indication that we no longer own any licenses to the # book. for identifier in identifiers_not_mentioned_by_bibliotheca: pools = [ lp for lp in identifier.licensed_through if lp.data_source.name == DataSource.BIBLIOTHECA and lp.collection == self.collection ] if not pools: continue for pool in pools: if pool.licenses_owned > 0: if pool.presentation_edition: self.log.warn("Removing %s (%s) from circulation", pool.presentation_edition.title, pool.presentation_edition.author) else: self.log.warn( "Removing unknown work %s from circulation.", identifier.identifier) pool.update_availability(0, 0, 0, 0, self.analytics) pool.last_checked = now def _process_circulation_data(self, circ, identifiers_by_bibliotheca_id, identifiers_not_mentioned_by_bibliotheca): """Process a single CirculationData object retrieved from Bibliotheca. """ bibliotheca_id = circ[Identifier][Identifier.BIBLIOTHECA_ID] identifier = identifiers_by_bibliotheca_id[bibliotheca_id] identifiers_not_mentioned_by_bibliotheca.remove(identifier) pools = [ lp for lp in identifier.licensed_through if lp.data_source.name == DataSource.BIBLIOTHECA and lp.collection == self.collection ] if not pools: # We don't have a license pool for this work. That # shouldn't happen--how did we know about the # identifier?--but it shouldn't be a big deal to # create one. pool, ignore = LicensePool.for_foreign_id( self._db, self.collection.data_source, identifier.type, identifier.identifier, collection=self.collection) # Bibliotheca books are never open-access. pool.open_access = False for library in self.collection.libraries: self.analytics.collect_event( library, pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, datetime.datetime.utcnow()) else: [pool] = pools self.api.apply_circulation_information_to_licensepool( circ, pool, self.analytics)
class EnkiImport(CollectionMonitor): """Import content from Enki that we don't yet have in our collection """ SERVICE_NAME = "Enki Circulation Monitor" INTERVAL_SECONDS = 500 PROTOCOL = EnkiAPI.ENKI_EXTERNAL DEFAULT_BATCH_SIZE = 100 FIVE_MINUTES = datetime.timedelta(minutes=5) def __init__(self, _db, collection, api_class=EnkiAPI): """Constructor.""" super(EnkiImport, self).__init__(_db, collection) self._db = _db self.api = api_class(_db, collection) self.collection_id = collection.id self.analytics = Analytics(_db) self.bibliographic_coverage_provider = ( EnkiBibliographicCoverageProvider(collection, api_class=self.api)) @property def collection(self): return Collection.by_id(self._db, id=self.collection_id) def recently_changed_ids(self, start, cutoff): return self.api.recently_changed_ids(start, cutoff) def run_once(self, start, cutoff): # Give us five minutes of overlap because it's very important # we don't miss anything. since = start - self.FIVE_MINUTES id_start = 0 while True: availability = self.api.availability(since=since, strt=id_start, qty=self.DEFAULT_BATCH_SIZE) if availability.status_code != 200: self.log.error( "Could not contact Enki server for content availability. Status: %d", availability.status_code) content = availability.content count = 0 for bibliographic, circulation in BibliographicParser( ).process_all(content): self.process_book(bibliographic, circulation) count += 1 if count == 0: break self._db.commit() id_start += self.DEFAULT_BATCH_SIZE def process_book(self, bibliographic, availability): license_pool, new_license_pool = availability.license_pool( self._db, self.collection) now = datetime.datetime.utcnow() edition, new_edition = bibliographic.edition(self._db) license_pool.edition = edition policy = ReplacementPolicy( identifiers=False, subjects=True, contributions=True, formats=True, ) availability.apply( self._db, license_pool.collection, replace=policy, ) if new_edition: bibliographic.apply(edition, self.collection, replace=policy) if new_license_pool or new_edition: # At this point we have done work equivalent to that done by # the EnkiBibliographicCoverageProvider. Register that the # work has been done so we don't have to do it again. identifier = edition.primary_identifier self.bibliographic_coverage_provider.handle_success(identifier) self.bibliographic_coverage_provider.add_coverage_record_for( identifier) for library in self.collection.libraries: self.analytics.collect_event( library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, now) return edition, license_pool
class OverdriveCirculationMonitor(CollectionMonitor): """Maintain LicensePools for recently changed Overdrive titles. Create basic Editions for any new LicensePools that show up. """ SERVICE_NAME = "Overdrive Circulation Monitor" INTERVAL_SECONDS = 500 PROTOCOL = ExternalIntegration.OVERDRIVE # Report successful completion upon finding this number of # consecutive books in the Overdrive results whose LicensePools # haven't changed since last time. Overdrive results are not in # strict chronological order, but if you see 100 consecutive books # that haven't changed, you're probably done. MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS = None def __init__(self, _db, collection, api_class=OverdriveAPI): """Constructor.""" super(OverdriveCirculationMonitor, self).__init__(_db, collection) self.api = api_class(_db, collection) self.maximum_consecutive_unchanged_books = ( self.MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS) self.analytics = Analytics(_db) def recently_changed_ids(self, start, cutoff): return self.api.recently_changed_ids(start, cutoff) def run_once(self, start, cutoff): _db = self._db added_books = 0 overdrive_data_source = DataSource.lookup(_db, DataSource.OVERDRIVE) total_books = 0 consecutive_unchanged_books = 0 for i, book in enumerate(self.recently_changed_ids(start, cutoff)): total_books += 1 if not total_books % 100: self.log.info("%s books processed", total_books) if not book: continue license_pool, is_new, is_changed = self.api.update_licensepool( book) # Log a circulation event for this work. if is_new: for library in self.collection.libraries: self.analytics.collect_event( library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked) _db.commit() if is_changed: consecutive_unchanged_books = 0 else: consecutive_unchanged_books += 1 if (self.maximum_consecutive_unchanged_books and consecutive_unchanged_books >= self.maximum_consecutive_unchanged_books): # We're supposed to stop this run after finding a # run of books that have not changed, and we have # in fact seen that many consecutive unchanged # books. self.log.info("Stopping at %d unchanged books.", consecutive_unchanged_books) break if total_books: self.log.info("Processed %d books total.", total_books)
class OverdriveCirculationMonitor(CollectionMonitor, TimelineMonitor): """Maintain LicensePools for recently changed Overdrive titles. Create basic Editions for any new LicensePools that show up. """ SERVICE_NAME = "Overdrive Circulation Monitor" PROTOCOL = ExternalIntegration.OVERDRIVE OVERLAP = datetime.timedelta(minutes=1) # Report successful completion upon finding this number of # consecutive books in the Overdrive results whose LicensePools # haven't changed since last time. Overdrive results are not in # strict chronological order, but if you see 100 consecutive books # that haven't changed, you're probably done. MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS = None def __init__(self, _db, collection, api_class=OverdriveAPI): """Constructor.""" super(OverdriveCirculationMonitor, self).__init__(_db, collection) self.api = api_class(_db, collection) self.maximum_consecutive_unchanged_books = ( self.MAXIMUM_CONSECUTIVE_UNCHANGED_BOOKS ) self.analytics = Analytics(_db) def recently_changed_ids(self, start, cutoff): return self.api.recently_changed_ids(start, cutoff) def catch_up_from(self, start, cutoff, progress): """Find Overdrive books that changed recently. :progress: A TimestampData representing the time previously covered by this Monitor. """ _db = self._db added_books = 0 overdrive_data_source = DataSource.lookup( _db, DataSource.OVERDRIVE) total_books = 0 consecutive_unchanged_books = 0 # Ask for changes between the last time covered by the Monitor # and the current time. for i, book in enumerate(self.recently_changed_ids(start, cutoff)): total_books += 1 if not total_books % 100: self.log.info("%s books processed", total_books) if not book: continue license_pool, is_new, is_changed = self.api.update_licensepool(book) # Log a circulation event for this work. if is_new: for library in self.collection.libraries: self.analytics.collect_event( library, license_pool, CirculationEvent.DISTRIBUTOR_TITLE_ADD, license_pool.last_checked) _db.commit() if is_changed: consecutive_unchanged_books = 0 else: consecutive_unchanged_books += 1 if (self.maximum_consecutive_unchanged_books and consecutive_unchanged_books >= self.maximum_consecutive_unchanged_books): # We're supposed to stop this run after finding a # run of books that have not changed, and we have # in fact seen that many consecutive unchanged # books. self.log.info("Stopping at %d unchanged books.", consecutive_unchanged_books) break progress.achievements = "Books processed: %d." % total_books
def fulfill(self, patron, pin, licensepool, delivery_mechanism, sync_on_failure=True): """Fulfil a book that a patron has previously checked out. :param delivery_mechanism: A LicensePoolDeliveryMechanism explaining how the patron wants the book to be delivered. If the book has previously been delivered through some other mechanism, this parameter is ignored and the previously used mechanism takes precedence. :return: A FulfillmentInfo object. """ fulfillment = None loan = get_one(self._db, Loan, patron=patron, license_pool=licensepool, on_multiple='interchangeable') if not loan: if sync_on_failure: # Sync and try again. self.sync_bookshelf(patron, pin) return self.fulfill(patron, pin, licensepool=licensepool, delivery_mechanism=delivery_mechanism, sync_on_failure=False) else: raise NoActiveLoan( _("Cannot find your active loan for this work.")) if loan.fulfillment is not None and loan.fulfillment != delivery_mechanism and not delivery_mechanism.delivery_mechanism.is_streaming: raise DeliveryMechanismConflict( _("You already fulfilled this loan as %(loan_delivery_mechanism)s, you can't also do it as %(requested_delivery_mechanism)s", loan_delivery_mechanism=loan.fulfillment.delivery_mechanism. name, requested_delivery_mechanism=delivery_mechanism. delivery_mechanism.name)) if licensepool.open_access: fulfillment = self.fulfill_open_access( licensepool, delivery_mechanism.delivery_mechanism) else: api = self.api_for_license_pool(licensepool) internal_format = api.internal_format(delivery_mechanism) fulfillment = api.fulfill(patron, pin, licensepool, internal_format) if not fulfillment or not (fulfillment.content_link or fulfillment.content): raise NoAcceptableFormat() # Send out an analytics event to record the fact that # a fulfillment was initiated through the circulation # manager. Analytics.collect_event( self._db, licensepool, CirculationEvent.CM_FULFILL, ) # Make sure the delivery mechanism we just used is associated # with the loan. if loan.fulfillment is None and not delivery_mechanism.delivery_mechanism.is_streaming: __transaction = self._db.begin_nested() loan.fulfillment = delivery_mechanism __transaction.commit() return fulfillment
None, None) # It's pretty rare that we'd go from having a loan for a book # to needing to put it on hold, but we do check for that case. __transaction = self._db.begin_nested() hold, is_new = licensepool.on_hold_to(patron, hold_info.start_date or now, hold_info.end_date, hold_info.hold_position) if hold and is_new: # Send out an analytics event to record the fact that # a hold was initiated through the circulation # manager. Analytics.collect_event( self._db, licensepool, CirculationEvent.CM_HOLD_PLACE, ) if existing_loan: self._db.delete(existing_loan) __transaction.commit() return None, hold, is_new def _collect_checkout_event(self, licensepool): """Collect an analytics event indicating the given LicensePool was checked out via the circulation manager. """ Analytics.collect_event( self._db, licensepool,