def __init__(self, _db, overdrive=None, threem=None, axis=None): self._db = _db self.overdrive = overdrive self.threem = threem self.axis = axis self.apis = [x for x in (overdrive, threem, axis) if x] self.log = logging.getLogger("Circulation API") # When we get our view of a patron's loans and holds, we need # to include loans from all licensed data sources. We do not # need to include loans from open-access sources because we # are the authorities on those. data_sources_for_sync = [] if self.overdrive: data_sources_for_sync.append( DataSource.lookup(_db, DataSource.OVERDRIVE) ) if self.threem: data_sources_for_sync.append( DataSource.lookup(_db, DataSource.THREEM) ) if self.axis: data_sources_for_sync.append( DataSource.lookup(_db, DataSource.AXIS_360) ) self.identifier_type_to_data_source_name = dict( (ds.primary_identifier_type, ds.name) for ds in data_sources_for_sync) self.data_source_ids_for_sync = [ x.id for x in data_sources_for_sync ]
def test_items_that_need_coverage(self): source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) other_source = DataSource.lookup(self._db, DataSource.OVERDRIVE) # An item that hasn't been covered by the provider yet cr = self._coverage_record(self._edition(), other_source) # An item that has been covered by the reaper operation already reaper_cr = self._coverage_record( self._edition(), source, operation=CoverageRecord.REAP_OPERATION ) # An item that has been covered by the reaper operation, but has # had its license repurchased. relicensed_edition, relicensed_licensepool = self._edition(with_license_pool=True) relicensed_coverage_record = self._coverage_record( relicensed_edition, source, operation=CoverageRecord.REAP_OPERATION ) relicensed_licensepool.update_availability(1, 0, 0, 0) items = self.provider.items_that_need_coverage().all() # Provider ignores anything that has been reaped and doesn't have # licenses. assert reaper_cr.identifier not in items # But it picks up anything that hasn't been covered at all and anything # that's been licensed anew even if its already been reaped. eq_(2, len(items)) assert relicensed_licensepool.identifier in items assert cr.identifier in items # The Wrangler Reaper coverage record is removed from the db # when it's committed. assert relicensed_coverage_record in relicensed_licensepool.identifier.coverage_records self._db.commit() assert relicensed_coverage_record not in relicensed_licensepool.identifier.coverage_records
def test_items_that_need_coverage(self): source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) other_source = DataSource.lookup(self._db, DataSource.OVERDRIVE) # An item that hasn't been covered by the provider yet cr = self._coverage_record(self._edition(), other_source) # An item that has been covered by the reaper operation already reaper_cr = self._coverage_record( self._edition(), source, operation=CoverageRecord.REAP_OPERATION ) # An item that has been covered by the reaper operation, but has # had its license repurchased. relicensed, relicensed_lp = self._edition(with_license_pool=True) self._coverage_record( relicensed, source, operation=CoverageRecord.REAP_OPERATION ) relicensed_lp.update_availability(1, 0, 0, 0) with temp_config() as config: config[Configuration.INTEGRATIONS][Configuration.METADATA_WRANGLER_INTEGRATION] = { Configuration.URL : "http://url.gov" } provider = MetadataWranglerCoverageProvider(self._db) items = provider.items_that_need_coverage.all() # Provider ignores anything that has been reaped and doesn't have # licenses. assert reaper_cr.identifier not in items # But it picks up anything that hasn't been covered at all and anything # that's been licensed anew even if its already been reaped. eq_(2, len(items)) assert relicensed_lp.identifier in items assert cr.identifier in items # The Wrangler Reaper coverage record is removed from the db # when it's committed. self._db.commit() eq_([], relicensed_lp.identifier.coverage_records)
def test_run_once(self): # Setup authentication and Metadata Wrangler details. lp = self._licensepool( None, data_source_name=DataSource.BIBLIOTHECA, collection=self.collection ) lp.identifier.type = Identifier.BIBLIOTHECA_ID isbn = Identifier.parse_urn(self._db, u'urn:isbn:9781594632556')[0] lp.identifier.equivalent_to( DataSource.lookup(self._db, DataSource.BIBLIOTHECA), isbn, 1 ) eq_([], lp.identifier.links) eq_([], lp.identifier.measurements) # Queue some data to be found. responses = ( 'metadata_updates_response.opds', 'metadata_updates_empty_response.opds', ) for filename in responses: data = sample_data(filename, 'opds') self.lookup.queue_response( 200, {'content-type' : OPDSFeed.ACQUISITION_FEED_TYPE}, data ) timestamp = self.ts new_timestamp = self.monitor.run_once(timestamp) # We have a new value to use for the Monitor's timestamp -- the # earliest date seen in the last OPDS feed that contained # any entries. eq_(datetime.datetime(2016, 9, 20, 19, 37, 2), new_timestamp.finish) eq_("Editions processed: 1", new_timestamp.achievements) # Normally run_once() doesn't update the monitor's timestamp, # but this implementation does, so that work isn't redone if # run_once() crashes or the monitor is killed. eq_(new_timestamp.finish, self.monitor.timestamp().finish) # The original Identifier has information from the # mock Metadata Wrangler. mw_source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) eq_(3, len(lp.identifier.links)) [quality] = lp.identifier.measurements eq_(mw_source, quality.data_source) # Check the URLs we processed. url1, url2 = [x[0] for x in self.lookup.requests] # The first URL processed was the default one for the # MetadataWranglerOPDSLookup. eq_(self.lookup.get_collection_url(self.lookup.UPDATES_ENDPOINT), url1) # The second URL processed was whatever we saw in the 'next' link. eq_("http://next-link/", url2)
def test_finalize_edition(self): provider_no_presentation_ready = self._provider(presentation_ready_on_success=False) provider_presentation_ready = self._provider(presentation_ready_on_success=True) identifier = self._identifier() source = DataSource.lookup(self._db, DataSource.GUTENBERG) # Here's an Edition with no LicensePool. edition, is_new = Edition.for_foreign_id( self._db, source, identifier.type, identifier.identifier ) edition.title = self._str # This will effectively do nothing. provider_no_presentation_ready.finalize_edition(edition) # No Works have been created. eq_(0, self._db.query(Work).count()) # But if there's also a LicensePool... pool, is_new = LicensePool.for_foreign_id( self._db, source, identifier.type, identifier.identifier ) # finalize_edition() will create a Work. provider_no_presentation_ready.finalize_edition(edition) work = pool.work eq_(work, edition.work) eq_(False, work.presentation_ready) # If the provider is configured to do so, finalize_edition() # will also set the Work as presentation-ready. provider_presentation_ready.finalize_edition(edition) eq_(True, work.presentation_ready)
def test_load_cover_link(self): # Create a directory import script with an empty mock filesystem. script = MockDirectoryImportScript(self._db, {}) identifier = self._identifier(Identifier.GUTENBERG_ID, "2345") gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) mirror = MockS3Uploader() args = (identifier, gutenberg, "covers", mirror) # There is nothing on the mock filesystem, so in this case # load_cover_link returns None. eq_(None, script.load_cover_link(*args)) # But we tried. eq_( ('2345', 'covers', Representation.COMMON_IMAGE_EXTENSIONS, 'cover image'), script._locate_file_args ) # Try another script that has a populated mock filesystem. mock_filesystem = { 'covers' : ( 'acover.jpeg', Representation.JPEG_MEDIA_TYPE, "I'm an image." ) } script = MockDirectoryImportScript(self._db, mock_filesystem) link = script.load_cover_link(*args) eq_(Hyperlink.IMAGE, link.rel) assert link.href.endswith( '/test.cover.bucket/Gutenberg/Gutenberg+ID/2345/2345.jpg' ) eq_(Representation.JPEG_MEDIA_TYPE, link.media_type) eq_("I'm an image.", link.content)
def __init__(self, _db, data_directory): self._db = _db self.collection = Collection.by_protocol(self._db, ExternalIntegration.GUTENBERG).one() self.source = DataSource.lookup(self._db, DataSource.GUTENBERG) self.data_directory = data_directory self.catalog_path = os.path.join(self.data_directory, self.FILENAME) self.log = logging.getLogger("Gutenberg API")
def lookup_equivalent_isbns(self, identifier): """Finds NoveList data for all ISBNs equivalent to an identifier. :return: Metadata object or None """ lookup_metadata = [] license_sources = DataSource.license_sources_for(self._db, identifier) # Look up strong ISBN equivalents. for license_source in license_sources: lookup_metadata += [self.lookup(eq.output) for eq in identifier.equivalencies if (eq.data_source==license_source and eq.strength==1 and eq.output.type==Identifier.ISBN)] if not lookup_metadata: self.log.error( "Identifiers without an ISBN equivalent can't \ be looked up with NoveList: %r", identifier ) return None # Remove None values. lookup_metadata = [metadata for metadata in lookup_metadata if metadata] if not lookup_metadata: return None best_metadata = self.choose_best_metadata(lookup_metadata, identifier) if not best_metadata: metadata, confidence = best_metadata if round(confidence, 2) < 0.5: self.log.warn(self.NO_ISBN_EQUIVALENCY, identifier) return None return metadata
def __init__(self, _db, lookup=None, input_identifier_types=None, operation=None, **kwargs): if not input_identifier_types: input_identifier_types = [ Identifier.OVERDRIVE_ID, Identifier.THREEM_ID, Identifier.GUTENBERG_ID, Identifier.AXIS_360_ID, ] output_source = DataSource.lookup( _db, DataSource.METADATA_WRANGLER ) super(MetadataWranglerCoverageProvider, self).__init__( lookup = lookup or SimplifiedOPDSLookup.from_config(), service_name=self.SERVICE_NAME, input_identifier_types=input_identifier_types, output_source=output_source, operation=operation or self.OPERATION, **kwargs ) if not self.lookup.authenticated: self.log.warn( "Authentication for the Library Simplified Metadata Wrangler " "is not set up. You can still use the metadata wrangler, but " "it will not know which collection you're asking about." )
def metadata_needed_for(self, collection_details): """Returns identifiers in the collection that could benefit from distributor metadata on the circulation manager. """ client = authenticated_client_from_request(self._db) if isinstance(client, ProblemDetail): return client collection = collection_from_details( self._db, client, collection_details ) resolver = IdentifierResolutionCoverageProvider unresolved_identifiers = collection.unresolved_catalog( self._db, resolver.DATA_SOURCE_NAME, resolver.OPERATION ) # Omit identifiers that currently have metadata pending for # the IntegrationClientCoverImageCoverageProvider. data_source = DataSource.lookup( self._db, collection.name, autocreate=True ) is_awaiting_metadata = self._db.query( CoverageRecord.id, CoverageRecord.identifier_id ).filter( CoverageRecord.data_source_id==data_source.id, CoverageRecord.status==CoverageRecord.REGISTERED, CoverageRecord.operation==IntegrationClientCoverImageCoverageProvider.OPERATION, ).subquery() unresolved_identifiers = unresolved_identifiers.outerjoin( is_awaiting_metadata, Identifier.id==is_awaiting_metadata.c.identifier_id ).filter(is_awaiting_metadata.c.id==None) # Add a message for each unresolved identifier pagination = load_pagination_from_request(default_size=25) feed_identifiers = pagination.apply(unresolved_identifiers).all() messages = list() for identifier in feed_identifiers: messages.append(OPDSMessage( identifier.urn, HTTP_ACCEPTED, "Metadata needed." )) title = "%s Metadata Requests for %s" % (collection.protocol, client.url) metadata_request_url = self.collection_feed_url( 'metadata_needed_for', collection ) request_feed = AcquisitionFeed( self._db, title, metadata_request_url, [], VerboseAnnotator, precomposed_entries=messages ) self.add_pagination_links_to_feed( pagination, unresolved_identifiers, request_feed, 'metadata_needed_for', collection ) return feed_response(request_feed)
def test_handle_import_messages(self): data_source = DataSource.lookup(self._db, DataSource.OVERDRIVE) provider = OPDSImportCoverageProvider("name", [], data_source) message = StatusMessage(201, "try again later") message2 = StatusMessage(404, "we're doomed") message3 = StatusMessage(200, "everything's fine") identifier = self._identifier() identifier2 = self._identifier() identifier3 = self._identifier() messages_by_id = { identifier.urn: message, identifier2.urn: message2, identifier3.urn: message3, } [f1, f2] = sorted(list(provider.handle_import_messages(messages_by_id)), key=lambda x: x.exception) eq_(identifier, f1.obj) eq_("201: try again later", f1.exception) eq_(True, f1.transient) eq_(identifier2, f2.obj) eq_("404: we're doomed", f2.exception) eq_(False, f2.transient)
def _provider(self, presentation_ready_on_success=True): """Create a generic MockOPDSImportCoverageProvider for testing purposes.""" source = DataSource.lookup(self._db, DataSource.OA_CONTENT_SERVER) return MockOPDSImportCoverageProvider( "mock provider", [], source, presentation_ready_on_success=presentation_ready_on_success )
def test_items_that_need_coverage_respects_cutoff(self): """Verify that this coverage provider respects the cutoff_time argument. """ source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) edition = self._edition() cr = self._coverage_record(edition, source, operation='sync') # We have a coverage record already, so this book doesn't show # up in items_that_need_coverage items = self.provider.items_that_need_coverage().all() eq_([], items) # But if we send a cutoff_time that's later than the time # associated with the coverage record... one_hour_from_now = ( datetime.datetime.utcnow() + datetime.timedelta(seconds=3600) ) provider_with_cutoff = self.create_provider( cutoff_time=one_hour_from_now ) # The book starts showing up in items_that_need_coverage. eq_([edition.primary_identifier], provider_with_cutoff.items_that_need_coverage().all())
def __init__(self, _db, input_identifier_types=None, metadata_lookup=None, cutoff_time=None, operation=None): self._db = _db if not input_identifier_types: input_identifier_types = [ Identifier.OVERDRIVE_ID, Identifier.THREEM_ID, Identifier.GUTENBERG_ID, Identifier.AXIS_360_ID, ] self.output_source = DataSource.lookup( self._db, DataSource.METADATA_WRANGLER ) if not metadata_lookup: metadata_lookup = SimplifiedOPDSLookup.from_config() self.lookup = metadata_lookup if not operation: operation = CoverageRecord.SYNC_OPERATION self.operation = operation super(MetadataWranglerCoverageProvider, self).__init__( self.service_name, input_identifier_types, self.output_source, workset_size=20, cutoff_time=cutoff_time, operation=self.operation, )
def generate_mock_api(self): """Prep an empty NoveList result.""" source = DataSource.lookup(self._db, DataSource.OVERDRIVE) metadata = Metadata(source) mock_api = MockNoveListAPI(self._db) mock_api.setup(metadata) return mock_api
def __init__(self, _db, testing=False, api=None): super(ThreeMCirculationSweep, self).__init__( _db, "3M Circulation Sweep", batch_size=25) self._db = _db if not api: api = ThreeMAPI(self._db, testing=testing) self.api = api self.data_source = DataSource.lookup(self._db, DataSource.THREEM)
def setup(self): super(TestMetadataWranglerCollectionReaper, self).setup() self.source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) with temp_config() as config: config[Configuration.INTEGRATIONS][Configuration.METADATA_WRANGLER_INTEGRATION] = { Configuration.URL : "http://url.gov" } self.reaper = MetadataWranglerCollectionReaper(self._db)
def __init__(self, _db, authenticator, node_value, temporary_token_duration=None): self._db = _db self.authenticator = authenticator self.data_source = DataSource.lookup(_db, DataSource.ADOBE) self.temporary_token_duration = ( temporary_token_duration or datetime.timedelta(minutes=10)) if isinstance(node_value, basestring): node_value = int(node_value, 16) self.node_value = node_value
def __init__(self, db, mirrors, uploader=None): self._db = db self.data_source_ids = [] self.uploader = uploader or S3Uploader() self.log = logging.getLogger("Cover Image Scaler") for mirror in mirrors: data_source_name = mirror.DATA_SOURCE data_source = DataSource.lookup(self._db, data_source_name) self.data_source_ids.append(data_source.id)
def setup(self): super(TestFeedbooksOPDSImporter, self).setup() self.http = DummyHTTPClient() self.metadata = DummyMetadataClient() self.mirror = MockS3Uploader() self.data_source = DataSource.lookup(self._db, DataSource.FEEDBOOKS) # Create a default importer that's good enough for most tests. self.collection, self.importer = self._importer()
def test_lookup_equivalent_isbns(self): identifier = self._identifier(identifier_type=Identifier.OVERDRIVE_ID) api = MockNoveListAPI.from_config(self._default_library) # If there are no ISBN equivalents, it returns None. eq_(None, api.lookup_equivalent_isbns(identifier)) source = DataSource.lookup(self._db, DataSource.OVERDRIVE) identifier.equivalent_to(source, self._identifier(), strength=1) self._db.commit() eq_(None, api.lookup_equivalent_isbns(identifier)) # If there's an ISBN equivalent, but it doesn't result in metadata, # it returns none. isbn = self._identifier(identifier_type=Identifier.ISBN) identifier.equivalent_to(source, isbn, strength=1) self._db.commit() api.responses.append(None) eq_(None, api.lookup_equivalent_isbns(identifier)) # Create an API class that can mockout NoveListAPI.choose_best_metadata class MockBestMetadataAPI(MockNoveListAPI): choose_best_metadata_return = None def choose_best_metadata(self, *args, **kwargs): return self.choose_best_metadata_return api = MockBestMetadataAPI.from_config(self._default_library) # Give the identifier another ISBN equivalent. isbn2 = self._identifier(identifier_type=Identifier.ISBN) identifier.equivalent_to(source, isbn2, strength=1) self._db.commit() # Queue metadata responses for each ISBN lookup. metadatas = [object(), object()] api.responses.extend(metadatas) # If choose_best_metadata returns None, the lookup returns None. api.choose_best_metadata_return = (None, None) eq_(None, api.lookup_equivalent_isbns(identifier)) # Lookup was performed for both ISBNs. eq_([], api.responses) # If choose_best_metadata returns a low confidence metadata, the # lookup returns None. api.responses.extend(metadatas) api.choose_best_metadata_return = (metadatas[0], 0.33) eq_(None, api.lookup_equivalent_isbns(identifier)) # If choose_best_metadata returns a high confidence metadata, the # lookup returns the metadata. api.responses.extend(metadatas) api.choose_best_metadata_return = (metadatas[1], 0.67) eq_(metadatas[1], api.lookup_equivalent_isbns(identifier))
def test_feed_includes_staff_rating(self): work = self._work(with_open_access_download=True) lp = work.license_pools[0] staff_data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF) lp.identifier.add_measurement(staff_data_source, Measurement.RATING, 3, weight=1000) feed = AcquisitionFeed(self._db, "test", "url", [work], AdminAnnotator(None, self._default_library, test_mode=True)) [entry] = feedparser.parse(unicode(feed))['entries'] rating = entry['schema_rating'] eq_(3, float(rating['schema:ratingvalue'])) eq_(Measurement.RATING, rating['additionaltype'])
def __init__(self, _db, api=None, **kwargs): input_identifier_types = [ Identifier.GUTENBERG_ID, Identifier.URI ] output_source = DataSource.lookup(_db, DataSource.OCLC) super(OCLCClassifyCoverageProvider, self).__init__( "OCLC Classify Coverage Provider", input_identifier_types, output_source) self._db = _db self.api = api or OCLCClassifyAPI(self._db)
def setup(self): super(TestVendorIDModel, self).setup() self.authenticator = DummyMilleniumPatronAPI() self.model = AdobeVendorIDModel(self._db, self.authenticator, self.TEST_NODE_VALUE) self.data_source = DataSource.lookup(self._db, DataSource.ADOBE) # Normally this test patron doesn't have an authorization identifier. # Let's make sure there is one so it'll show up as the label. self.bob_patron = self.authenticator.authenticated_patron( self._db, dict(username="******", password="******")) self.bob_patron.authorization_identifier = "5"
def test_facets(self): # Normally we yield one FeaturedFacets object for each of the # library's enabled entry points. library = self._default_library script = CacheOPDSGroupFeedPerLane( self._db, manager=object(), cmd_args=[] ) setting = library.setting(EntryPoint.ENABLED_SETTING) setting.value = json.dumps( [AudiobooksEntryPoint.INTERNAL_NAME, EbooksEntryPoint.INTERNAL_NAME] ) lane = self._lane() audio_facets, ebook_facets = script.facets(lane) eq_(AudiobooksEntryPoint, audio_facets.entrypoint) eq_(EbooksEntryPoint, ebook_facets.entrypoint) # The first entry point in the library's list of enabled entry # points is treated as the default. eq_(True, audio_facets.entrypoint_is_default) eq_(audio_facets.entrypoint, list(library.entrypoints)[0]) eq_(False, ebook_facets.entrypoint_is_default) for facets in (audio_facets, ebook_facets): # The FeaturedFacets objects knows to feature works at the # library's minimum quality level. eq_(library.minimum_featured_quality, facets.minimum_featured_quality) # The FeaturedFacets object knows that custom lists are # not in play. eq_(False, facets.uses_customlists) # The first entry point is treated as the default only for WorkLists # that have no parent. When the WorkList has a parent, the selected # entry point is treated as an explicit choice -- navigating downward # in the lane hierarchy ratifies the default value. sublane = self._lane(parent=lane) f1, f2 = script.facets(sublane) for f in f1, f2: eq_(False, f.entrypoint_is_default) # Make it look like the lane uses custom lists. lane.list_datasource = DataSource.lookup(self._db, DataSource.OVERDRIVE) # If the library has no enabled entry points, we yield one # FeaturedFacets object with no particular entry point. setting.value = json.dumps([]) no_entry_point, = script.facets(lane) eq_(None, no_entry_point.entrypoint) # The FeaturedFacets object knows that custom lists are in # play. eq_(True, no_entry_point.uses_customlists)
def process_item(self, identifier): data_source = DataSource.lookup( self._db, self.importer.data_source_name ) try: response = self.content_server.lookup([identifier]) except BadResponseException, e: return CoverageFailure( identifier, e.message, data_source )
def test_new_isbns(self): existing_id = self._identifier() metadata = Metadata( DataSource.lookup(self._db, DataSource.GUTENBERG), identifiers=[ IdentifierData(type=Identifier.OCLC_WORK, identifier="abra"), IdentifierData(type=existing_id.type, identifier=existing_id.identifier), IdentifierData(type=Identifier.ISBN, identifier="kadabra"), ] ) eq_(2, self.provider.new_isbns(metadata))
def setup(self): super(TestMetadataUploadCoverageProvider, self).setup() self.integration = self._external_integration( ExternalIntegration.METADATA_WRANGLER, goal=ExternalIntegration.METADATA_GOAL, url=self._url, username=u'abc', password=u'def' ) self.source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) self.collection = self._collection( protocol=ExternalIntegration.BIBLIOTHECA, external_account_id=u'lib' ) self.provider = self.create_provider()
def test_related_books(self): # A book with no related books returns a ProblemDetail. with temp_config() as config: config['integrations'][Configuration.NOVELIST_INTEGRATION] = {} with self.app.test_request_context('/'): response = self.manager.work_controller.related( self.datasource, self.identifier.type, self.identifier.identifier ) eq_(404, response.status_code) eq_("http://librarysimplified.org/terms/problem/unknown-lane", response.uri) # Prep book with a book in its series and a recommendation. self.lp.presentation_edition.series = "Around the World" self.french_1.presentation_edition.series = "Around the World" SessionManager.refresh_materialized_views(self._db) source = DataSource.lookup(self._db, self.datasource) metadata = Metadata(source) mock_api = MockNoveListAPI() metadata.recommendations = [self.english_2.license_pools[0].identifier] mock_api.setup(metadata) # A grouped feed is returned with both of these related books with self.app.test_request_context('/'): response = self.manager.work_controller.related( self.datasource, self.identifier.type, self.identifier.identifier, novelist_api=mock_api ) eq_(200, response.status_code) feed = feedparser.parse(response.data) eq_(3, len(feed['entries'])) # One book is in the recommendations feed. [e1] = [e for e in feed['entries'] if e['title'] == self.english_2.title] [collection_link] = [link for link in e1['links'] if link['rel']=='collection'] eq_("Recommended Books", collection_link['title']) work_url = "/works/%s/%s/%s/" % (self.datasource, self.identifier.type, self.identifier.identifier) expected = urllib.quote(work_url + 'recommendations') eq_(True, collection_link['href'].endswith(expected)) # Two books are in the series feed. The original work and its companion [e2] = [e for e in feed['entries'] if e['title'] == self.french_1.title] [collection_link] = [link for link in e2['links'] if link['rel']=='collection'] eq_("Around the World", collection_link['title']) expected = urllib.quote(work_url + 'series') eq_(True, collection_link['href'].endswith(expected)) [e3] = [e for e in feed['entries'] if e['title'] == self.english_1.title] [collection_link] = [link for link in e3['links'] if link['rel']=='collection'] eq_("Around the World", collection_link['title']) expected = urllib.quote(work_url + 'series') eq_(True, collection_link['href'].endswith(expected))
def test_load_circulation_data(self): # Create a directory import script with an empty mock filesystem. script = MockDirectoryImportScript(self._db, {}) identifier = self._identifier(Identifier.GUTENBERG_ID, "2345") gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) mirror = MockS3Uploader() args = (identifier, gutenberg, "ebooks", mirror, "Name of book", "rights URI") # There is nothing on the mock filesystem, so in this case # load_circulation_data returns None. eq_(None, script.load_circulation_data(*args)) # But we tried. eq_( ('2345', 'ebooks', Representation.COMMON_EBOOK_EXTENSIONS, 'ebook file'), script._locate_file_args ) # Try another script that has a populated mock filesystem. mock_filesystem = { 'ebooks' : ( 'book.epub', Representation.EPUB_MEDIA_TYPE, "I'm an EPUB." ) } script = MockDirectoryImportScript(self._db, mock_filesystem) # Now _locate_file finds something on the mock filesystem, and # load_circulation_data loads it into a fully populated # CirculationData object. circulation = script.load_circulation_data(*args) eq_(identifier, circulation.primary_identifier(self._db)) eq_(gutenberg, circulation.data_source(self._db)) eq_("rights URI", circulation.default_rights_uri) # The CirculationData has an open-access link associated with it. [link] = circulation.links eq_(Hyperlink.OPEN_ACCESS_DOWNLOAD, link.rel) assert link.href.endswith( '/test.content.bucket/Gutenberg/Gutenberg+ID/2345/Name+of+book.epub' ) eq_(Representation.EPUB_MEDIA_TYPE, link.media_type) eq_("I'm an EPUB.", link.content) # This open-access link will be made available through a # delivery mechanism described by this FormatData. [format] = circulation.formats eq_(link, format.link) eq_(link.media_type, format.content_type) eq_(DeliveryMechanism.NO_DRM, format.drm_scheme)
def source(self): return DataSource.lookup(self._db, DataSource.OCLC_LINKED_DATA)
def __init__(self, db, uploader=None): self._db = db self.data_source = DataSource.lookup(self._db, self.DATA_SOURCE) self.uploader = uploader or S3Uploader.from_config(self._db) self.log = logging.getLogger("Cover Image Mirror")
def test_import(self): feed = self.get_data("biblioboard_mini_feed.opds") data_source = DataSource.lookup(self._db, "Biblioboard", autocreate=True) collection = MockOPDSForDistributorsAPI.mock_collection(self._db) collection.external_integration.set_setting( Collection.DATA_SOURCE_NAME_SETTING, data_source.name) class MockMetadataClient(object): def canonicalize_author_name(self, identifier, working_display_name): return working_display_name metadata_client = MockMetadataClient() importer = OPDSForDistributorsImporter( self._db, collection=collection, metadata_client=metadata_client, ) imported_editions, imported_pools, imported_works, failures = ( importer.import_from_feed(feed)) # This importer works the same as the base OPDSImporter, except that # it adds delivery mechanisms for books with epub acquisition links # and sets pools' licenses_owned and licenses_available. # Both works were created, since we can use their acquisition links # to give copies to patrons. [camelot, southern] = sorted(imported_works, key=lambda x: x.title) # Each work has a license pool. [camelot_pool] = camelot.license_pools [southern_pool] = southern.license_pools now = datetime.datetime.utcnow() for pool in [camelot_pool, southern_pool]: eq_(False, pool.open_access) eq_(RightsStatus.IN_COPYRIGHT, pool.delivery_mechanisms[0].rights_status.uri) eq_(Representation.EPUB_MEDIA_TYPE, pool.delivery_mechanisms[0].delivery_mechanism.content_type) eq_(DeliveryMechanism.BEARER_TOKEN, pool.delivery_mechanisms[0].delivery_mechanism.drm_scheme) eq_(1, pool.licenses_owned) eq_(1, pool.licenses_available) assert (pool.work.last_update_time - now).total_seconds() <= 2 [camelot_acquisition_link] = [ l for l in camelot_pool.identifier.links if l.rel == Hyperlink.GENERIC_OPDS_ACQUISITION and l.resource. representation.media_type == Representation.EPUB_MEDIA_TYPE ] camelot_acquisition_url = camelot_acquisition_link.resource.representation.url eq_( "https://library.biblioboard.com/ext/api/media/04377e87-ab69-41c8-a2a4-812d55dc0952/assets/content.epub", camelot_acquisition_url) [southern_acquisition_link] = [ l for l in southern_pool.identifier.links if l.rel == Hyperlink.GENERIC_OPDS_ACQUISITION and l.resource. representation.media_type == Representation.EPUB_MEDIA_TYPE ] southern_acquisition_url = southern_acquisition_link.resource.representation.url eq_( "https://library.biblioboard.com/ext/api/media/04da95cd-6cfc-4e82-810f-121d418b6963/assets/content.epub", southern_acquisition_url)
def change_book_cover(self, identifier_type, identifier, mirrors=None): """Save a new book cover based on the submitted form.""" self.require_librarian(flask.request.library) data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF) work = self.load_work(flask.request.library, identifier_type, identifier) if isinstance(work, ProblemDetail): return work rights_uri = flask.request.form.get("rights_status") rights_explanation = flask.request.form.get("rights_explanation") if not rights_uri: return INVALID_IMAGE.detailed( _("You must specify the image's license.")) collection = self._get_collection_from_pools(identifier_type, identifier) if isinstance(collection, ProblemDetail): return collection # Look for an appropriate mirror to store this cover image. Since the # mirror should be used for covers, we don't need a mirror for books. mirrors = mirrors or dict(covers_mirror=MirrorUploader.for_collection( collection, ExternalIntegrationLink.COVERS), books_mirror=None) if not mirrors.get(ExternalIntegrationLink.COVERS): return INVALID_CONFIGURATION_OPTION.detailed( _("Could not find a storage integration for uploading the cover." )) image = self.generate_cover_image(work, identifier_type, identifier) if isinstance(image, ProblemDetail): return image original, derivation_settings, cover_href, cover_rights_explanation = self._original_cover_info( image, work, data_source, rights_uri, rights_explanation) buffer = StringIO() image.save(buffer, format="PNG") content = buffer.getvalue() if not cover_href: cover_href = Hyperlink.generic_uri( data_source, work.presentation_edition.primary_identifier, Hyperlink.IMAGE, content=content) cover_data = LinkData( Hyperlink.IMAGE, href=cover_href, media_type=Representation.PNG_MEDIA_TYPE, content=content, rights_uri=rights_uri, rights_explanation=cover_rights_explanation, original=original, transformation_settings=derivation_settings, ) presentation_policy = PresentationCalculationPolicy( choose_edition=False, set_edition_metadata=False, classify=False, choose_summary=False, calculate_quality=False, choose_cover=True, regenerate_opds_entries=True, regenerate_marc_record=True, update_search_index=False, ) replacement_policy = ReplacementPolicy( links=True, # link_content is false because we already have the content. # We don't want the metadata layer to try to fetch it again. link_content=False, mirrors=mirrors, presentation_calculation_policy=presentation_policy, ) metadata = Metadata(data_source, links=[cover_data]) metadata.apply(work.presentation_edition, collection, replace=replacement_policy) # metadata.apply only updates the edition, so we also need # to update the work. work.calculate_presentation(policy=presentation_policy) return Response(_("Success"), 200)
def setup_method(self): super(TestProQuestCredentialManager, self).setup_method() self._data_source = DataSource.lookup( self._db, DataSource.PROQUEST, autocreate=True )
def edit_classifications(self, identifier_type, identifier): """Edit a work's audience, target age, fiction status, and genres.""" self.require_librarian(flask.request.library) work = self.load_work(flask.request.library, identifier_type, identifier) if isinstance(work, ProblemDetail): return work staff_data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF) # Previous staff classifications primary_identifier = work.presentation_edition.primary_identifier old_classifications = self._db \ .query(Classification) \ .join(Subject) \ .filter( Classification.identifier == primary_identifier, Classification.data_source == staff_data_source ) old_genre_classifications = old_classifications \ .filter(Subject.genre_id != None) old_staff_genres = [ c.subject.genre.name for c in old_genre_classifications if c.subject.genre ] old_computed_genres = [ work_genre.genre.name for work_genre in work.work_genres ] # New genres should be compared to previously computed genres new_genres = flask.request.form.getlist("genres") genres_changed = sorted(new_genres) != sorted(old_computed_genres) # Update audience new_audience = flask.request.form.get("audience") if new_audience != work.audience: # Delete all previous staff audience classifications for c in old_classifications: if c.subject.type == Subject.FREEFORM_AUDIENCE: self._db.delete(c) # Create a new classification with a high weight primary_identifier.classify( data_source=staff_data_source, subject_type=Subject.FREEFORM_AUDIENCE, subject_identifier=new_audience, weight=WorkController.STAFF_WEIGHT, ) # Update target age if present new_target_age_min = flask.request.form.get("target_age_min") new_target_age_min = int( new_target_age_min) if new_target_age_min else None new_target_age_max = flask.request.form.get("target_age_max") new_target_age_max = int( new_target_age_max) if new_target_age_max else None if new_target_age_max < new_target_age_min: return INVALID_EDIT.detailed( _("Minimum target age must be less than maximum target age.")) if work.target_age: old_target_age_min = work.target_age.lower old_target_age_max = work.target_age.upper else: old_target_age_min = None old_target_age_max = None if new_target_age_min != old_target_age_min or new_target_age_max != old_target_age_max: # Delete all previous staff target age classifications for c in old_classifications: if c.subject.type == Subject.AGE_RANGE: self._db.delete(c) # Create a new classification with a high weight - higher than audience if new_target_age_min and new_target_age_max: age_range_identifier = "%s-%s" % (new_target_age_min, new_target_age_max) primary_identifier.classify( data_source=staff_data_source, subject_type=Subject.AGE_RANGE, subject_identifier=age_range_identifier, weight=WorkController.STAFF_WEIGHT * 100, ) # Update fiction status # If fiction status hasn't changed but genres have changed, # we still want to ensure that there's a staff classification new_fiction = True if flask.request.form.get( "fiction") == "fiction" else False if new_fiction != work.fiction or genres_changed: # Delete previous staff fiction classifications for c in old_classifications: if c.subject.type == Subject.SIMPLIFIED_FICTION_STATUS: self._db.delete(c) # Create a new classification with a high weight (higher than genre) fiction_term = "Fiction" if new_fiction else "Nonfiction" classification = primary_identifier.classify( data_source=staff_data_source, subject_type=Subject.SIMPLIFIED_FICTION_STATUS, subject_identifier=fiction_term, weight=WorkController.STAFF_WEIGHT, ) classification.subject.fiction = new_fiction # Update genres # make sure all new genres are legit for name in new_genres: genre, is_new = Genre.lookup(self._db, name) if not isinstance(genre, Genre): return GENRE_NOT_FOUND if genres[name].is_fiction is not None and genres[ name].is_fiction != new_fiction: return INCOMPATIBLE_GENRE if name == "Erotica" and new_audience != "Adults Only": return EROTICA_FOR_ADULTS_ONLY if genres_changed: # delete existing staff classifications for genres that aren't being kept for c in old_genre_classifications: if c.subject.genre.name not in new_genres: self._db.delete(c) # add new staff classifications for new genres for genre in new_genres: if genre not in old_staff_genres: classification = primary_identifier.classify( data_source=staff_data_source, subject_type=Subject.SIMPLIFIED_GENRE, subject_identifier=genre, weight=WorkController.STAFF_WEIGHT) # add NONE genre classification if we aren't keeping any genres if len(new_genres) == 0: primary_identifier.classify( data_source=staff_data_source, subject_type=Subject.SIMPLIFIED_GENRE, subject_identifier=SimplifiedGenreClassifier.NONE, weight=WorkController.STAFF_WEIGHT) else: # otherwise delete existing NONE genre classification none_classifications = self._db \ .query(Classification) \ .join(Subject) \ .filter( Classification.identifier == primary_identifier, Subject.identifier == SimplifiedGenreClassifier.NONE ) \ .all() for c in none_classifications: self._db.delete(c) # Update presentation policy = PresentationCalculationPolicy(classify=True, regenerate_opds_entries=True, regenerate_marc_record=True, update_search_index=True) work.calculate_presentation(policy=policy) return Response("", 200)
def edit(self, identifier_type, identifier): """Edit a work's metadata.""" self.require_librarian(flask.request.library) # TODO: It would be nice to use the metadata layer for this, but # this code handles empty values differently than other metadata # sources. When a staff member deletes a value, that indicates # they think it should be empty. This needs to be indicated in the # db so that it can overrule other data sources that set a value, # unlike other sources which set empty fields to None. work = self.load_work(flask.request.library, identifier_type, identifier) if isinstance(work, ProblemDetail): return work changed = False staff_data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF) primary_identifier = work.presentation_edition.primary_identifier staff_edition, is_new = get_one_or_create( self._db, Edition, primary_identifier_id=primary_identifier.id, data_source_id=staff_data_source.id) self._db.expire(primary_identifier) new_title = flask.request.form.get("title") if new_title and work.title != new_title: staff_edition.title = unicode(new_title) changed = True new_subtitle = flask.request.form.get("subtitle") if work.subtitle != new_subtitle: if work.subtitle and not new_subtitle: new_subtitle = NO_VALUE staff_edition.subtitle = unicode(new_subtitle) changed = True # The form data includes roles and names for contributors in the same order. new_contributor_roles = flask.request.form.getlist("contributor-role") new_contributor_names = [ unicode(n) for n in flask.request.form.getlist("contributor-name") ] # The first author in the form is considered the primary author, even # though there's no separate MARC code for that. for i, role in enumerate(new_contributor_roles): if role == Contributor.AUTHOR_ROLE: new_contributor_roles[i] = Contributor.PRIMARY_AUTHOR_ROLE break roles_and_names = zip(new_contributor_roles, new_contributor_names) # Remove any contributions that weren't in the form, and remove contributions # that already exist from the list so they won't be added again. deleted_contributions = False for contribution in staff_edition.contributions: if (contribution.role, contribution.contributor.display_name ) not in roles_and_names: self._db.delete(contribution) deleted_contributions = True changed = True else: roles_and_names.remove( (contribution.role, contribution.contributor.display_name)) if deleted_contributions: # Ensure the staff edition's contributions are up-to-date when # calculating the presentation edition later. self._db.refresh(staff_edition) # Any remaining roles and names are new contributions. for role, name in roles_and_names: # There may be one extra role at the end from the input for # adding a contributor, in which case it will have no # corresponding name and can be ignored. if name: if role not in Contributor.MARC_ROLE_CODES.keys(): self._db.rollback() return UNKNOWN_ROLE.detailed( _("Role %(role)s is not one of the known contributor roles.", role=role)) contributor = staff_edition.add_contributor(name=name, roles=[role]) contributor.display_name = name changed = True new_series = flask.request.form.get("series") if work.series != new_series: if work.series and not new_series: new_series = NO_VALUE staff_edition.series = unicode(new_series) changed = True new_series_position = flask.request.form.get("series_position") if new_series_position != None and new_series_position != '': try: new_series_position = int(new_series_position) except ValueError: self._db.rollback() return INVALID_SERIES_POSITION else: new_series_position = None if work.series_position != new_series_position: if work.series_position and new_series_position == None: new_series_position = NO_NUMBER staff_edition.series_position = new_series_position changed = True new_medium = flask.request.form.get("medium") if new_medium: if new_medium not in Edition.medium_to_additional_type.keys(): self._db.rollback() return UNKNOWN_MEDIUM.detailed( _("Medium %(medium)s is not one of the known media.", medium=new_medium)) staff_edition.medium = new_medium changed = True new_language = flask.request.form.get("language") if new_language != None and new_language != '': new_language = LanguageCodes.string_to_alpha_3(new_language) if not new_language: self._db.rollback() return UNKNOWN_LANGUAGE else: new_language = None if new_language != staff_edition.language: staff_edition.language = new_language changed = True new_publisher = flask.request.form.get("publisher") if new_publisher != staff_edition.publisher: if staff_edition.publisher and not new_publisher: new_publisher = NO_VALUE staff_edition.publisher = unicode(new_publisher) changed = True new_imprint = flask.request.form.get("imprint") if new_imprint != staff_edition.imprint: if staff_edition.imprint and not new_imprint: new_imprint = NO_VALUE staff_edition.imprint = unicode(new_imprint) changed = True new_issued = flask.request.form.get("issued") if new_issued != None and new_issued != '': try: new_issued = datetime.strptime(new_issued, '%Y-%m-%d') except ValueError: self._db.rollback() return INVALID_DATE_FORMAT else: new_issued = None if new_issued != staff_edition.issued: staff_edition.issued = new_issued changed = True # TODO: This lets library staff add a 1-5 rating, which is used in the # quality calculation. However, this doesn't work well if there are any # other measurements that contribute to the quality. The form will show # the calculated quality rather than the staff rating, which will be # confusing. It might also be useful to make it more clear how this # relates to the quality threshold in the library settings. changed_rating = False new_rating = flask.request.form.get("rating") if new_rating != None and new_rating != '': try: new_rating = float(new_rating) except ValueError: self._db.rollback() return INVALID_RATING scale = Measurement.RATING_SCALES[DataSource.LIBRARY_STAFF] if new_rating < scale[0] or new_rating > scale[1]: self._db.rollback() return INVALID_RATING.detailed( _("The rating must be a number between %(low)s and %(high)s.", low=scale[0], high=scale[1])) if (new_rating - scale[0]) / (scale[1] - scale[0]) != work.quality: primary_identifier.add_measurement( staff_data_source, Measurement.RATING, new_rating, weight=WorkController.STAFF_WEIGHT) changed = True changed_rating = True changed_summary = False new_summary = flask.request.form.get("summary") or "" if new_summary != work.summary_text: old_summary = None if work.summary and work.summary.data_source == staff_data_source: old_summary = work.summary work.presentation_edition.primary_identifier.add_link( Hyperlink.DESCRIPTION, None, staff_data_source, content=new_summary) # Delete previous staff summary if old_summary: for link in old_summary.links: self._db.delete(link) self._db.delete(old_summary) changed = True changed_summary = True if changed: # Even if the presentation doesn't visibly change, we want # to regenerate the OPDS entries and update the search # index for the work, because that might be the 'real' # problem the user is trying to fix. policy = PresentationCalculationPolicy( classify=True, regenerate_opds_entries=True, regenerate_marc_record=True, update_search_index=True, calculate_quality=changed_rating, choose_summary=changed_summary, ) work.calculate_presentation(policy=policy) return Response("", 200)
def data_source(self): return DataSource.lookup(self._db, DataSource.ADOBE)
def test_create_record(self): work = self._work( with_license_pool=True, title="old title", authors=["old author"], data_source_name=DataSource.OVERDRIVE, ) annotator = Annotator() # The record isn't cached yet, so a new record is created and cached. assert None == work.marc_record record = MARCExporter.create_record(work, annotator) [title_field] = record.get_fields("245") assert "old title" == title_field.get_subfields("a")[0] [author_field] = record.get_fields("100") assert "author, old" == author_field.get_subfields("a")[0] [distributor_field] = record.get_fields("264") assert DataSource.OVERDRIVE == distributor_field.get_subfields("b")[0] cached = work.marc_record assert "old title" in cached assert "author, old" in cached # The distributor isn't part of the cached record. assert DataSource.OVERDRIVE not in cached work.presentation_edition.title = "new title" work.presentation_edition.sort_author = "author, new" new_data_source = DataSource.lookup(self._db, DataSource.BIBLIOTHECA) work.license_pools[0].data_source = new_data_source # Now that the record is cached, creating a record will # use the cache. Distributor will be updated since it's # not part of the cached record. record = MARCExporter.create_record(work, annotator) [title_field] = record.get_fields("245") assert "old title" == title_field.get_subfields("a")[0] [author_field] = record.get_fields("100") assert "author, old" == author_field.get_subfields("a")[0] [distributor_field] = record.get_fields("264") assert DataSource.BIBLIOTHECA == distributor_field.get_subfields( "b")[0] # But we can force an update to the cached record. record = MARCExporter.create_record(work, annotator, force_create=True) [title_field] = record.get_fields("245") assert "new title" == title_field.get_subfields("a")[0] [author_field] = record.get_fields("100") assert "author, new" == author_field.get_subfields("a")[0] [distributor_field] = record.get_fields("264") assert DataSource.BIBLIOTHECA == distributor_field.get_subfields( "b")[0] cached = work.marc_record assert "old title" not in cached assert "author, old" not in cached assert "new title" in cached assert "author, new" in cached # If we pass in an integration, it's passed along to the annotator. integration = self._integration() class MockAnnotator(Annotator): integration = None def annotate_work_record(self, work, pool, edition, identifier, record, integration): self.integration = integration annotator = MockAnnotator() record = MARCExporter.create_record(work, annotator, integration=integration) assert integration == annotator.integration
def source(self): return DataSource.lookup(self._db, DataSource.ODILO)
def source(self): return DataSource.lookup(_db, DataSource.NYT)
def test_create_lanes_for_large_collection(self): languages = ['eng', 'spa'] create_lanes_for_large_collection(self._db, self._default_library, languages) lanes = self._db.query(Lane).filter(Lane.parent_id == None).order_by( Lane.priority).all() # We have five top-level lanes. eq_(5, len(lanes)) eq_([ 'Fiction', 'Nonfiction', 'Young Adult Fiction', 'Young Adult Nonfiction', 'Children and Middle Grade' ], [x.display_name for x in lanes]) for lane in lanes: eq_(self._default_library, lane.library) # They all are restricted to English and Spanish. eq_(x.languages, languages) # They have no restrictions on media type -- that's handled # with entry points. eq_(None, x.media) eq_([ 'Fiction', 'Nonfiction', 'Young Adult Fiction', 'Young Adult Nonfiction', 'Children and Middle Grade' ], [x.display_name for x in lanes]) # The Adult Fiction and Adult Nonfiction lanes reproduce the # genre structure found in the genre definitions. fiction, nonfiction = lanes[0:2] [sf] = [ x for x in fiction.sublanes if 'Science Fiction' in x.display_name ] [periodicals] = [ x for x in nonfiction.sublanes if 'Periodicals' in x.display_name ] eq_(True, sf.fiction) eq_("Science Fiction", sf.display_name) assert 'Science Fiction' in [genre.name for genre in sf.genres] [nonfiction_humor ] = [x for x in nonfiction.sublanes if 'Humor' in x.display_name] eq_(False, nonfiction_humor.fiction) [fiction_humor ] = [x for x in fiction.sublanes if 'Humor' in x.display_name] eq_(True, fiction_humor.fiction) [space_opera ] = [x for x in sf.sublanes if 'Space Opera' in x.display_name] eq_(True, sf.fiction) eq_("Space Opera", space_opera.display_name) eq_(["Space Opera"], [genre.name for genre in space_opera.genres]) [history ] = [x for x in nonfiction.sublanes if 'History' in x.display_name] eq_(False, history.fiction) eq_("History", history.display_name) assert 'History' in [genre.name for genre in history.genres] [european_history] = [ x for x in history.sublanes if 'European History' in x.display_name ] assert 'European History' in [ genre.name for genre in european_history.genres ] # Delete existing lanes. for lane in self._db.query(Lane).filter( Lane.library_id == self._default_library.id): self._db.delete(lane) # If there's an NYT Best Sellers integration and we create the lanes again... integration, ignore = create(self._db, ExternalIntegration, goal=ExternalIntegration.METADATA_GOAL, protocol=ExternalIntegration.NYT) create_lanes_for_large_collection(self._db, self._default_library, languages) lanes = self._db.query(Lane).filter(Lane.parent_id == None).order_by( Lane.priority).all() # Now we have six top-level lanes, with best sellers at the beginning. eq_([ u'Best Sellers', 'Fiction', 'Nonfiction', 'Young Adult Fiction', 'Young Adult Nonfiction', 'Children and Middle Grade' ], [x.display_name for x in lanes]) # Each sublane other than best sellers also contains a best sellers lane. for sublane in lanes[1:]: best_sellers = sublane.visible_children[0] eq_("Best Sellers", best_sellers.display_name) # The best sellers lane has a data source. nyt_data_source = DataSource.lookup(self._db, DataSource.NYT) eq_(nyt_data_source, lanes[0].list_datasource)
def data_source(self): """Use the collection's name as the data source name.""" return DataSource.lookup(self._db, self.collection.name, autocreate=True)
def test_annotate_metadata(self): """Verify that annotate_metadata calls load_circulation_data and load_cover_link appropriately. """ # First, test an unsuccessful annotation. class MockNoCirculationData(DirectoryImportScript): """Do nothing when load_circulation_data is called. Explode if load_cover_link is called. """ def load_circulation_data(self, *args): self.load_circulation_data_args = args return None def load_cover_link(self, *args): raise Exception("Explode!") gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) identifier = IdentifierData(Identifier.GUTENBERG_ID, "11111") identifier_obj, ignore = identifier.load(self._db) metadata = Metadata( title=self._str, data_source=gutenberg, primary_identifier=identifier ) mirror = object() policy = ReplacementPolicy(mirror=mirror) cover_directory = object() ebook_directory = object() rights_uri = object() script = MockNoCirculationData(self._db) args = (metadata, policy, cover_directory, ebook_directory, rights_uri) script.annotate_metadata(*args) # load_circulation_data was called. eq_( (identifier_obj, gutenberg, ebook_directory, mirror, metadata.title, rights_uri), script.load_circulation_data_args ) # But because load_circulation_data returned None, # metadata.circulation_data was not modified and # load_cover_link was not called (which would have raised an # exception). eq_(None, metadata.circulation) # Test a successful annotation with no cover image. class MockNoCoverLink(DirectoryImportScript): """Return an object when load_circulation_data is called. Do nothing when load_cover_link is called. """ def load_circulation_data(self, *args): return "Some circulation data" def load_cover_link(self, *args): self.load_cover_link_args = args return None script = MockNoCoverLink(self._db) script.annotate_metadata(*args) # The Metadata object was annotated with the return value of # load_circulation_data. eq_("Some circulation data", metadata.circulation) # load_cover_link was called. eq_( (identifier_obj, gutenberg, cover_directory, mirror), script.load_cover_link_args ) # But since it provided no cover link, metadata.links was empty. eq_([], metadata.links) # Finally, test a completely successful annotation. class MockWithCoverLink(DirectoryImportScript): """Mock success for both load_circulation_data and load_cover_link. """ def load_circulation_data(self, *args): return "Some circulation data" def load_cover_link(self, *args): return "A cover link" metadata.circulation = None script = MockWithCoverLink(self._db) script.annotate_metadata(*args) eq_("Some circulation data", metadata.circulation) eq_(['A cover link'], metadata.links)
def _quality(self, value, weight=1): # The only source we recognize for quality scores is the metadata # wrangler. source = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER) return self._measurement(Measurement.QUALITY, value, source, weight)
def edition_query(self): gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) return self._db.query(Edition).filter(Edition.data_source == gutenberg)
def sync_bookshelf(self, patron, pin): # Get the external view of the patron's current state. remote_loans, remote_holds, complete = self.patron_activity( patron, pin) # Get our internal view of the patron's current state. __transaction = self._db.begin_nested() local_loans = self.local_loans(patron) local_holds = self.local_holds(patron) now = datetime.datetime.utcnow() local_loans_by_identifier = {} local_holds_by_identifier = {} for l in local_loans: if not l.license_pool: self.log.error("Active loan with no license pool!") continue i = l.license_pool.identifier if not i: self.log.error( "Active loan on license pool %s, which has no identifier!", l.license_pool) continue key = (i.type, i.identifier) local_loans_by_identifier[key] = l for h in local_holds: if not h.license_pool: self.log.error("Active hold with no license pool!") continue i = h.license_pool.identifier if not i: self.log.error( "Active hold on license pool %r, which has no identifier!", h.license_pool) continue key = (i.type, i.identifier) local_holds_by_identifier[key] = h active_loans = [] active_holds = [] for loan in remote_loans: # This is a remote loan. Find or create the corresponding # local loan. source_name = self.identifier_type_to_data_source_name[ loan.identifier_type] source = DataSource.lookup(self._db, source_name) key = (loan.identifier_type, loan.identifier) pool, ignore = LicensePool.for_foreign_id(self._db, source, loan.identifier_type, loan.identifier) start = loan.start_date or now end = loan.end_date local_loan, new = pool.loan_to(patron, start, end) active_loans.append(local_loan) # Remove the local loan from the list so that we don't # delete it later. if key in local_loans_by_identifier: del local_loans_by_identifier[key] for hold in remote_holds: # This is a remote hold. Find or create the corresponding # local hold. key = (hold.identifier_type, hold.identifier) source_name = self.identifier_type_to_data_source_name[ hold.identifier_type] source = DataSource.lookup(self._db, source_name) pool, ignore = LicensePool.for_foreign_id(self._db, source, hold.identifier_type, hold.identifier) start = hold.start_date or now end = hold.end_date position = hold.hold_position local_hold, new = pool.on_hold_to(patron, start, end, position) active_holds.append(local_hold) # Remove the local hold from the list so that we don't # delete it later. if key in local_holds_by_identifier: del local_holds_by_identifier[key] # We only want to delete local loans and holds if we were able to # successfully sync with all the providers. If there was an error, # the provider might still know about a loan or hold that we don't # have in the remote lists. if complete: # Every loan remaining in loans_by_identifier is a hold that # the provider doesn't know about. This usually means it's expired # and we should get rid of it, but it's possible the patron is # borrowing a book and syncing their bookshelf at the same time, # and the local loan was created after we got the remote loans. # If the loan's start date is less than a minute ago, we'll keep it. for loan in local_loans_by_identifier.values(): if loan.license_pool.data_source.id in self.data_source_ids_for_sync: one_minute_ago = datetime.datetime.utcnow( ) - datetime.timedelta(minutes=1) if loan.start < one_minute_ago: logging.info( "In sync_bookshelf for patron %s, deleting loan %d (patron %s)" % (patron.authorization_identifier, loan.id, loan.patron.authorization_identifier)) self._db.delete(loan) else: logging.info( "In sync_bookshelf for patron %s, found local loan %d created in the past minute that wasn't in remote loans" % (patron.authorization_identifier, loan.id)) # Every hold remaining in holds_by_identifier is a hold that # the provider doesn't know about, which means it's expired # and we should get rid of it. for hold in local_holds_by_identifier.values(): if hold.license_pool.data_source.id in self.data_source_ids_for_sync: self._db.delete(hold) __transaction.commit() return active_loans, active_holds
def create_lanes_for_large_collection(_db, library, languages, priority=0): """Ensure that the lanes appropriate to a large collection are all present. This means: * A "%(language)s Adult Fiction" lane containing sublanes for each fiction genre. * A "%(language)s Adult Nonfiction" lane containing sublanes for each nonfiction genre. * A "%(language)s YA Fiction" lane containing sublanes for the most popular YA fiction genres. * A "%(language)s YA Nonfiction" lane containing sublanes for the most popular YA fiction genres. * A "%(language)s Children and Middle Grade" lane containing sublanes for childrens' books at different age levels. :param library: Newly created lanes will be associated with this library. :param languages: Newly created lanes will contain only books in these languages. :return: A list of top-level Lane objects. TODO: If there are multiple large collections, their top-level lanes do not have distinct display names. """ if isinstance(languages, basestring): languages = [languages] ADULT = Classifier.AUDIENCES_ADULT YA = [Classifier.AUDIENCE_YOUNG_ADULT] CHILDREN = [Classifier.AUDIENCE_CHILDREN] common_args = dict(languages=languages, media=None) adult_common_args = dict(common_args) adult_common_args['audiences'] = ADULT include_best_sellers = False nyt_data_source = DataSource.lookup(_db, DataSource.NYT) nyt_integration = get_one( _db, ExternalIntegration, goal=ExternalIntegration.METADATA_GOAL, protocol=ExternalIntegration.NYT, ) if nyt_integration: include_best_sellers = True language_identifier = LanguageCodes.name_for_languageset(languages) sublanes = [] if include_best_sellers: best_sellers, ignore = create(_db, Lane, library=library, display_name="Best Sellers", priority=priority, **common_args) priority += 1 best_sellers.list_datasource = nyt_data_source sublanes.append(best_sellers) adult_fiction_sublanes = [] adult_fiction_priority = 0 if include_best_sellers: adult_fiction_best_sellers, ignore = create( _db, Lane, library=library, display_name="Best Sellers", fiction=True, priority=adult_fiction_priority, **adult_common_args) adult_fiction_priority += 1 adult_fiction_best_sellers.list_datasource = nyt_data_source adult_fiction_sublanes.append(adult_fiction_best_sellers) for genre in fiction_genres: if isinstance(genre, basestring): genre_name = genre else: genre_name = genre.get("name") genre_lane = lane_from_genres(_db, library, [genre], priority=adult_fiction_priority, **adult_common_args) adult_fiction_priority += 1 adult_fiction_sublanes.append(genre_lane) adult_fiction, ignore = create(_db, Lane, library=library, display_name="Fiction", genres=[], sublanes=adult_fiction_sublanes, fiction=True, priority=priority, **adult_common_args) priority += 1 sublanes.append(adult_fiction) adult_nonfiction_sublanes = [] adult_nonfiction_priority = 0 if include_best_sellers: adult_nonfiction_best_sellers, ignore = create( _db, Lane, library=library, display_name="Best Sellers", fiction=False, priority=adult_nonfiction_priority, **adult_common_args) adult_nonfiction_priority += 1 adult_nonfiction_best_sellers.list_datasource = nyt_data_source adult_nonfiction_sublanes.append(adult_nonfiction_best_sellers) for genre in nonfiction_genres: # "Life Strategies" is a YA-specific genre that should not be # included in the Adult Nonfiction lane. if genre != genres.Life_Strategies: if isinstance(genre, basestring): genre_name = genre else: genre_name = genre.get("name") genre_lane = lane_from_genres(_db, library, [genre], priority=adult_nonfiction_priority, **adult_common_args) adult_nonfiction_priority += 1 adult_nonfiction_sublanes.append(genre_lane) adult_nonfiction, ignore = create(_db, Lane, library=library, display_name="Nonfiction", genres=[], sublanes=adult_nonfiction_sublanes, fiction=False, priority=priority, **adult_common_args) priority += 1 sublanes.append(adult_nonfiction) ya_common_args = dict(common_args) ya_common_args['audiences'] = YA ya_fiction, ignore = create(_db, Lane, library=library, display_name="Young Adult Fiction", genres=[], fiction=True, sublanes=[], priority=priority, **ya_common_args) priority += 1 sublanes.append(ya_fiction) ya_fiction_priority = 0 if include_best_sellers: ya_fiction_best_sellers, ignore = create(_db, Lane, library=library, display_name="Best Sellers", fiction=True, priority=ya_fiction_priority, **ya_common_args) ya_fiction_priority += 1 ya_fiction_best_sellers.list_datasource = nyt_data_source ya_fiction.sublanes.append(ya_fiction_best_sellers) ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Dystopian_SF], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Fantasy], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Comics_Graphic_Novels], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Literary_Fiction], display_name="Contemporary Fiction", priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.LGBTQ_Fiction], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Suspense_Thriller, genres.Mystery], display_name="Mystery & Thriller", priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Romance], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres( _db, library, [genres.Science_Fiction], exclude_genres=[genres.Dystopian_SF, genres.Steampunk], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_fiction.sublanes.append( lane_from_genres(_db, library, [genres.Steampunk], priority=ya_fiction_priority, **ya_common_args)) ya_fiction_priority += 1 ya_nonfiction, ignore = create(_db, Lane, library=library, display_name="Young Adult Nonfiction", genres=[], fiction=False, sublanes=[], priority=priority, **ya_common_args) priority += 1 sublanes.append(ya_nonfiction) ya_nonfiction_priority = 0 if include_best_sellers: ya_nonfiction_best_sellers, ignore = create( _db, Lane, library=library, display_name="Best Sellers", fiction=False, priority=ya_nonfiction_priority, **ya_common_args) ya_nonfiction_priority += 1 ya_nonfiction_best_sellers.list_datasource = nyt_data_source ya_nonfiction.sublanes.append(ya_nonfiction_best_sellers) ya_nonfiction.sublanes.append( lane_from_genres(_db, library, [genres.Biography_Memoir], display_name="Biography", priority=ya_nonfiction_priority, **ya_common_args)) ya_nonfiction_priority += 1 ya_nonfiction.sublanes.append( lane_from_genres(_db, library, [genres.History, genres.Social_Sciences], display_name="History & Sociology", priority=ya_nonfiction_priority, **ya_common_args)) ya_nonfiction_priority += 1 ya_nonfiction.sublanes.append( lane_from_genres(_db, library, [genres.Life_Strategies], priority=ya_nonfiction_priority, **ya_common_args)) ya_nonfiction_priority += 1 ya_nonfiction.sublanes.append( lane_from_genres(_db, library, [genres.Religion_Spirituality], priority=ya_nonfiction_priority, **ya_common_args)) ya_nonfiction_priority += 1 children_common_args = dict(common_args) children_common_args['audiences'] = CHILDREN children, ignore = create(_db, Lane, library=library, display_name="Children and Middle Grade", genres=[], fiction=None, sublanes=[], priority=priority, **children_common_args) priority += 1 sublanes.append(children) children_priority = 0 if include_best_sellers: children_best_sellers, ignore = create(_db, Lane, library=library, display_name="Best Sellers", priority=children_priority, **children_common_args) children_priority += 1 children_best_sellers.list_datasource = nyt_data_source children.sublanes.append(children_best_sellers) picture_books, ignore = create( _db, Lane, library=library, display_name="Picture Books", target_age=(0, 4), genres=[], fiction=None, priority=children_priority, languages=languages, ) children_priority += 1 children.sublanes.append(picture_books) easy_readers, ignore = create( _db, Lane, library=library, display_name="Easy Readers", target_age=(5, 8), genres=[], fiction=None, priority=children_priority, languages=languages, ) children_priority += 1 children.sublanes.append(easy_readers) chapter_books, ignore = create( _db, Lane, library=library, display_name="Chapter Books", target_age=(9, 12), genres=[], fiction=None, priority=children_priority, languages=languages, ) children_priority += 1 children.sublanes.append(chapter_books) children_poetry, ignore = create(_db, Lane, library=library, display_name="Poetry Books", priority=children_priority, **children_common_args) children_priority += 1 children_poetry.add_genre(genres.Poetry.name) children.sublanes.append(children_poetry) children_folklore, ignore = create(_db, Lane, library=library, display_name="Folklore", priority=children_priority, **children_common_args) children_priority += 1 children_folklore.add_genre(genres.Folklore.name) children.sublanes.append(children_folklore) children_fantasy, ignore = create(_db, Lane, library=library, display_name="Fantasy", fiction=True, priority=children_priority, **children_common_args) children_priority += 1 children_fantasy.add_genre(genres.Fantasy.name) children.sublanes.append(children_fantasy) children_sf, ignore = create(_db, Lane, library=library, display_name="Science Fiction", fiction=True, priority=children_priority, **children_common_args) children_priority += 1 children_sf.add_genre(genres.Science_Fiction.name) children.sublanes.append(children_sf) realistic_fiction, ignore = create(_db, Lane, library=library, display_name="Realistic Fiction", fiction=True, priority=children_priority, **children_common_args) children_priority += 1 realistic_fiction.add_genre(genres.Literary_Fiction.name) children.sublanes.append(realistic_fiction) children_graphic_novels, ignore = create( _db, Lane, library=library, display_name="Comics & Graphic Novels", priority=children_priority, **children_common_args) children_priority += 1 children_graphic_novels.add_genre(genres.Comics_Graphic_Novels.name) children.sublanes.append(children_graphic_novels) children_biography, ignore = create(_db, Lane, library=library, display_name="Biography", priority=children_priority, **children_common_args) children_priority += 1 children_biography.add_genre(genres.Biography_Memoir.name) children.sublanes.append(children_biography) children_historical_fiction, ignore = create( _db, Lane, library=library, display_name="Historical Fiction", priority=children_priority, **children_common_args) children_priority += 1 children_historical_fiction.add_genre(genres.Historical_Fiction.name) children.sublanes.append(children_historical_fiction) informational, ignore = create(_db, Lane, library=library, display_name="Informational Books", fiction=False, genres=[], priority=children_priority, **children_common_args) children_priority += 1 informational.add_genre(genres.Biography_Memoir.name, inclusive=False) children.sublanes.append(informational) return priority
def extract_edition(cls, _db, work_tag, existing_authors, **restrictions): """Create a new Edition object with information about a work (identified by OCLC Work ID). """ # TODO: 'pswid' is what it's called in older representations. # That code can be removed once we replace all representations. oclc_work_id = unicode(work_tag.get('owi') or work_tag.get('pswid')) # if oclc_work_id: # print " owi: %s" % oclc_work_id # else: # print " No owi in %s" % etree.tostring(work_tag) if not oclc_work_id: raise ValueError("Work has no owi") item_type = work_tag.get("itemtype") if (item_type.startswith('itemtype-book') or item_type.startswith('itemtype-compfile')): medium = Edition.BOOK_MEDIUM elif item_type.startswith('itemtype-audiobook' ) or item_type.startswith('itemtype-music'): # Pretty much all Gutenberg texts, even the audio texts, # are based on a book, and the ones that aren't # (recordings of individual songs) probably aren't in OCLC # anyway. So we just want to get the books. medium = Edition.AUDIO_MEDIUM medium = None elif item_type.startswith('itemtype-video'): #medium = Edition.VIDEO_MEDIUM medium = None elif item_type in cls.UNUSED_MEDIA: medium = None else: medium = None # Only create Editions for books with a recognized medium if medium is None: return None, False result = cls._extract_basic_info(_db, work_tag, existing_authors, **restrictions) if not result: # This record did not meet one of the restrictions. return None, False title, authors_and_roles, language = result # Record some extra OCLC-specific information editions = work_tag.get('editions') holdings = work_tag.get('holdings') # Get an identifier for this work. identifier, ignore = Identifier.for_foreign_id(_db, Identifier.OCLC_WORK, oclc_work_id) data_source = DataSource.lookup(_db, DataSource.OCLC) identifier.add_measurement(data_source, Measurement.HOLDINGS, holdings) identifier.add_measurement(data_source, Measurement.PUBLISHED_EDITIONS, editions) # Create a Edition for source + identifier edition, new = get_one_or_create(_db, Edition, data_source=data_source, primary_identifier=identifier, create_method_kwargs=dict( title=title, language=language, )) # Get the most popular Dewey and LCC classification for this # work. for tag_name, subject_type in (("ddc", Subject.DDC), ("lcc", Subject.LCC)): tag = cls._xpath1(work_tag, "//oclc:%s/oclc:mostPopular" % tag_name) if tag is not None: id = tag.get('nsfa') or tag.get('sfa') weight = int(tag.get('holdings')) identifier.classify(data_source, subject_type, id, weight=weight) # Find FAST subjects for the work. for heading in cls._xpath(work_tag, "//oclc:fast//oclc:heading"): id = heading.get('ident') weight = int(heading.get('heldby')) value = heading.text identifier.classify(data_source, Subject.FAST, id, value, weight) # Associate the authors with the Edition. for contributor, roles in authors_and_roles: edition.add_contributor(contributor, roles) return edition, new
def test_reaper(self): feed = self.get_data("biblioboard_mini_feed.opds") class MockOPDSForDistributorsReaperMonitor( OPDSForDistributorsReaperMonitor): """An OPDSForDistributorsReaperMonitor that overrides _get.""" def _get(self, url, headers): return (200, { "content-type": OPDSFeed.ACQUISITION_FEED_TYPE }, feed) data_source = DataSource.lookup(self._db, "Biblioboard", autocreate=True) collection = MockOPDSForDistributorsAPI.mock_collection(self._db) collection.external_integration.set_setting( Collection.DATA_SOURCE_NAME_SETTING, data_source.name) monitor = MockOPDSForDistributorsReaperMonitor( self._db, collection, OPDSForDistributorsImporter, metadata_client=object()) # There's a license pool in the database that isn't in the feed anymore. edition, now_gone = self._edition( identifier_type=Identifier.URI, data_source_name=data_source.name, with_license_pool=True, collection=collection, ) now_gone.licenses_owned = 1 now_gone.licenses_available = 1 edition, still_there = self._edition( identifier_type=Identifier.URI, identifier_id="urn:uuid:04377e87-ab69-41c8-a2a4-812d55dc0952", data_source_name=data_source.name, with_license_pool=True, collection=collection, ) still_there.licenses_owned = 1 still_there.licenses_available = 1 progress = monitor.run_once(monitor.timestamp().to_data()) # One LicensePool has been cleared out. assert 0 == now_gone.licenses_owned assert 0 == now_gone.licenses_available # The other is still around. assert 1 == still_there.licenses_owned assert 1 == still_there.licenses_available # The TimestampData returned by run_once() describes its # achievements. assert "License pools removed: 1." == progress.achievements # The TimestampData does not include any timing information -- # that will be applied by run(). assert None == progress.start assert None == progress.finish
def test_process_batch(self): provider = self._provider() # Here are an Edition and a LicensePool for the same identifier but # from different data sources. We would expect this to happen # when talking to the open-access content server. edition = self._edition(data_source_name=DataSource.OA_CONTENT_SERVER) identifier = edition.primary_identifier license_source = DataSource.lookup(self._db, DataSource.GUTENBERG) pool, is_new = LicensePool.for_foreign_id( self._db, license_source, identifier.type, identifier.identifier, collection=self._default_collection) eq_(None, pool.work) # Here's a second Edition/LicensePool that's going to cause a # problem: the LicensePool will show up in the results, but # the corresponding Edition will not. edition2, pool2 = self._edition(with_license_pool=True) # Here's an identifier that can't be looked up at all. identifier = self._identifier() messages_by_id = { identifier.urn: CoverageFailure(identifier, "201: try again later") } # When we call CoverageProvider.process_batch(), it's going to # return the information we just set up: a matched # Edition/LicensePool pair, a mismatched LicensePool, and an # error message. provider.queue_import_results([edition], [pool, pool2], [], messages_by_id) # Make the CoverageProvider do its thing. fake_batch = [object()] success, failure1, failure2 = provider.process_batch(fake_batch) # The fake batch was provided to lookup_and_import_batch. eq_([fake_batch], provider.batches) # The matched Edition/LicensePool pair was returned. eq_(success, edition.primary_identifier) # The LicensePool of that pair was passed into finalize_license_pool. # The mismatched LicensePool was not. eq_([pool], provider.finalized) # The mismatched LicensePool turned into a CoverageFailure # object. assert isinstance(failure1, CoverageFailure) eq_('OPDS import operation imported LicensePool, but no Edition.', failure1.exception) eq_(pool2.identifier, failure1.obj) eq_(True, failure1.transient) # The failure was returned as a CoverageFailure object. assert isinstance(failure2, CoverageFailure) eq_(identifier, failure2.obj) eq_(True, failure2.transient)
def test_work_from_metadata(self): """Validate the ability to create a new Work from appropriate metadata. """ class Mock(MockDirectoryImportScript): """In this test we need to verify that annotate_metadata was called but did nothing. """ def annotate_metadata(self, metadata, *args, **kwargs): metadata.annotated = True return super(Mock, self).annotate_metadata( metadata, *args, **kwargs ) identifier = IdentifierData(Identifier.GUTENBERG_ID, "1003") identifier_obj, ignore = identifier.load(self._db) metadata = Metadata( DataSource.GUTENBERG, primary_identifier=identifier, title=u"A book" ) metadata.annotated = False datasource = DataSource.lookup(self._db, DataSource.GUTENBERG) policy = ReplacementPolicy.from_license_source(self._db) mirror = MockS3Uploader() policy.mirror = mirror # Here, work_from_metadata calls annotate_metadata, but does # not actually import anything because there are no files 'on # disk' and thus no way to actually get the book. collection = self._default_collection args = (collection, metadata, policy, "cover directory", "ebook directory", RightsStatus.CC0) script = Mock(self._db) eq_(None, script.work_from_metadata(*args)) eq_(True, metadata.annotated) # Now let's try it with some files 'on disk'. with open(self.sample_cover_path('test-book-cover.png')) as fh: image = fh.read() mock_filesystem = { 'cover directory' : ( 'cover.jpg', Representation.JPEG_MEDIA_TYPE, image ), 'ebook directory' : ( 'book.epub', Representation.EPUB_MEDIA_TYPE, "I'm an EPUB." ) } script = MockDirectoryImportScript( self._db, mock_filesystem=mock_filesystem ) work = script.work_from_metadata(*args) # We have created a book. It has a cover image, which has a # thumbnail. eq_("A book", work.title) assert work.cover_full_url.endswith( '/test.cover.bucket/Gutenberg/Gutenberg+ID/1003/1003.jpg' ) assert work.cover_thumbnail_url.endswith( '/test.cover.bucket/scaled/300/Gutenberg/Gutenberg+ID/1003/1003.png' ) [pool] = work.license_pools assert pool.open_access_download_url.endswith( '/test.content.bucket/Gutenberg/Gutenberg+ID/1003/A+book.epub' ) eq_(RightsStatus.CC0, pool.delivery_mechanisms[0].rights_status.uri) # The mock S3Uploader has a record of 'uploading' all these files # to S3. epub, full, thumbnail = mirror.uploaded eq_(epub.url, pool.open_access_download_url) eq_(full.url, work.cover_full_url) eq_(thumbnail.url, work.cover_thumbnail_url) # The EPUB Representation was cleared out after the upload, to # save database space. eq_("I'm an EPUB.", mirror.content[0]) eq_(None, epub.content)
class OCLCXMLParser(XMLParser): # OCLC in-representation 'status codes' SINGLE_WORK_SUMMARY_STATUS = 0 SINGLE_WORK_DETAIL_STATUS = 2 MULTI_WORK_STATUS = 4 NO_INPUT_STATUS = 100 INVALID_INPUT_STATUS = 101 NOT_FOUND_STATUS = 102 UNEXPECTED_ERROR_STATUS = 200 INTS = set([OCLC.HOLDING_COUNT, OCLC.EDITION_COUNT]) NAMESPACES = {'oclc': 'http://classify.oclc.org'} LIST_TYPE = "works" log = logging.getLogger("OCLC XML Parser") @classmethod def parse(cls, _db, xml, **restrictions): """Turn XML data from the OCLC lookup service into a list of SWIDs (for a multi-work response) or a list of Edition objects (for a single-work response). """ tree = etree.fromstring(xml, parser=etree.XMLParser(recover=True)) response = cls._xpath1(tree, "oclc:response") representation_type = int(response.get('code')) workset_record = None editions = [] edition_records = [] if representation_type == cls.UNEXPECTED_ERROR_STATUS: raise IOError("Unexpected error from OCLC API: %s" % xml) elif representation_type in (cls.NO_INPUT_STATUS, cls.INVALID_INPUT_STATUS): return representation_type, [] elif representation_type == cls.SINGLE_WORK_SUMMARY_STATUS: raise IOError( "Got single-work summary from OCLC despite requesting detail: %s" % xml) # The real action happens here. if representation_type == cls.SINGLE_WORK_DETAIL_STATUS: authors_tag = cls._xpath1(tree, "//oclc:authors") work_tag = cls._xpath1(tree, "//oclc:work") if work_tag is not None: author_string = work_tag.get('author') primary_author = cls.primary_author_from_author_string( _db, author_string) existing_authors = cls.extract_authors( _db, authors_tag, primary_author=primary_author) # The representation lists a single work, its authors, its editions, # plus summary classification information for the work. edition, ignore = cls.extract_edition(_db, work_tag, existing_authors, **restrictions) if edition: cls.log.info("EXTRACTED %r", edition) records = [] if edition: records.append(edition) else: # The work record itself failed one of the # restrictions. None of its editions are likely to # succeed either. return representation_type, records elif representation_type == cls.MULTI_WORK_STATUS: # The representation lists a set of works that match the # search query. cls.log.debug("Extracting SWIDs from search results.") records = cls.extract_swids(_db, tree, **restrictions) elif representation_type == cls.NOT_FOUND_STATUS: # No problem; OCLC just doesn't have any data. records = [] else: raise IOError("Unrecognized status code from OCLC API: %s (%s)" % (representation_type, xml)) return representation_type, records @classmethod def extract_swids(cls, _db, tree, **restrictions): """Turn a multi-work response into a list of SWIDs.""" swids = [] for work_tag in cls._xpath(tree, "//oclc:work"): # We're not calling extract_basic_info because we care about # the info, we're calling it to make sure this work meets # the restriction. If this work meets the restriction, # we'll store its info when we look up the SWID. response = cls._extract_basic_info(_db, work_tag, **restrictions) if response: title, author_names, language = response # TODO: 'swid' is what it's called in older representations. # That code can be removed once we replace all representations. work_identifier = work_tag.get('wi') or work_tag.get('swid') cls.log.debug("WORK ID %s (%s, %r, %s)", work_identifier, title, author_names, language) swids.append(work_identifier) return swids ROLES = re.compile("\[([^]]+)\]$") LIFESPAN = re.compile("([0-9]+)-([0-9]*)[.;]?$") @classmethod def extract_authors(cls, _db, authors_tag, primary_author=None): results = [] if authors_tag is not None: for author_tag in cls._xpath(authors_tag, "//oclc:author"): lc = author_tag.get('lc', None) viaf = author_tag.get('viaf', None) contributor, roles, default_role_used = cls._parse_single_author( _db, author_tag.text, lc=lc, viaf=viaf, primary_author=primary_author) if contributor: results.append(contributor) return results @classmethod def _contributor_match(cls, contributor, name, lc, viaf): return (contributor.sort_name == name and (lc is None or contributor.lc == lc) and (viaf is None or contributor.viaf == viaf)) @classmethod def _parse_single_author(cls, _db, author, lc=None, viaf=None, existing_authors=[], default_role=Contributor.AUTHOR_ROLE, primary_author=None): default_role_used = False # First find roles if present # "Giles, Lionel, 1875-1958 [Writer of added commentary; Translator]" author = author.strip() m = cls.ROLES.search(author) if m: author = author[:m.start()].strip() role_string = m.groups()[0] roles = [x.strip() for x in role_string.split(";")] elif default_role: roles = [default_role] default_role_used = True else: roles = [] # Author string now looks like # "Giles, Lionel, 1875-1958" m = cls.LIFESPAN.search(author) kwargs = dict() if m: author = author[:m.start()].strip() birth, death = m.groups() if birth: kwargs[Contributor.BIRTH_DATE] = birth if death: kwargs[Contributor.DEATH_DATE] = death # Author string now looks like # "Giles, Lionel," if author.endswith(","): author = author[:-1] contributor = None if not author: # No name was given for the author. return None, roles, default_role_used if primary_author and author == primary_author.sort_name: if Contributor.AUTHOR_ROLE in roles: roles.remove(Contributor.AUTHOR_ROLE) if Contributor.UNKNOWN_ROLE in roles: roles.remove(Contributor.UNKNOWN_ROLE) roles.insert(0, Contributor.PRIMARY_AUTHOR_ROLE) if existing_authors: # Calling Contributor.lookup will result in a database # hit, and looking up a contributor based on name may # result in multiple results (see below). We'll have no # way of distinguishing between those results. If # possible, it's much more reliable to look through # existing_authors (the authors derived from an entry's # <authors> tag). for x in existing_authors: if cls._contributor_match(x, author, lc, viaf): contributor = x break if contributor: was_new = False if not contributor: contributor, was_new = Contributor.lookup(_db, author, viaf, lc, extra=kwargs) if isinstance(contributor, list): # We asked for an author based solely on the name, which makes # Contributor.lookup() return a list. if len(contributor) == 1: # Fortunately, either the database knows about only # one author with that name, or it didn't know about # any authors with that name and it just created one, # so we can unambiguously use it. contributor = contributor[0] else: # Uh-oh. The database knows about multiple authors # with that name. We have no basis for deciding which # author we mean. But we would prefer to identify with # an author who has a known LC or VIAF number. # # This should happen very rarely because of our check # against existing_authors above. But it will happen # for authors that have a work in Project Gutenberg. with_id = [ x for x in contributor if x.lc is not None or x.viaf is not None ] if with_id: contributor = with_id[0] else: contributor = contributor[0] return contributor, roles, default_role_used @classmethod def primary_author_from_author_string(cls, _db, author_string): # If the first author mentioned in the author string # does not have an explicit role set, treat them as the primary # author. if not author_string: return None authors = author_string.split("|") if not authors: return None author, roles, default_role_used = cls._parse_single_author( _db, authors[0], default_role=Contributor.PRIMARY_AUTHOR_ROLE) if roles == [Contributor.PRIMARY_AUTHOR_ROLE]: return author return None @classmethod def parse_author_string(cls, _db, author_string, existing_authors=[], primary_author=None): default_role = Contributor.PRIMARY_AUTHOR_ROLE authors = [] if not author_string: return authors for author in author_string.split("|"): author, roles, default_role_used = cls._parse_single_author( _db, author, existing_authors=existing_authors, default_role=default_role, primary_author=primary_author) if roles: if Contributor.PRIMARY_AUTHOR_ROLE in roles: # That was the primary author. If we see someone # with no explicit role after this point, they're # just a regular author. default_role = Contributor.AUTHOR_ROLE elif not default_role_used: # We're dealing with someone whose role was # explicitly specified. If we see someone with no # explicit role after this point, it's probably # because their role is so minor as to not be # worth mentioning, not because it's so major that # we can assume they're an author. default_role = Contributor.UNKNOWN_ROLE roles = roles or [default_role] if author: authors.append((author, roles)) return authors @classmethod def _extract_basic_info(cls, _db, tag, existing_authors=None, **restrictions): """Extract information common to work tag and edition tag.""" title = tag.get('title') author_string = tag.get('author') authors_and_roles = cls.parse_author_string(_db, author_string, existing_authors) if 'language' in tag.keys(): language = tag.get('language') else: language = None if title and 'title' in restrictions: must_resemble_title = restrictions['title'] threshold = restrictions.get('title_similarity', 0.25) similarity = MetadataSimilarity.title_similarity( must_resemble_title, title) if similarity < threshold: # The title of the book under consideration is not # similar enough to the given title. cls.log.debug("FAILURE TO RESEMBLE: %s vs %s (%.2f)", title, must_resemble_title, similarity) return None # The semicolon is frequently used to separate multiple # works in an anthology. If there is no semicolon in the # original title, do not consider titles that contain # semicolons. if (not ' ; ' in must_resemble_title and ' ; ' in title and threshold > 0): cls.log.debug("SEMICOLON DISQUALIFICATION: %s", title) return None # Apply restrictions. If they're not met, return None. if 'language' in restrictions and language: # We know which language this record is for. Match it # against the language used in the Edition we're # matching against. restrict_to_language = set(restrictions['language']) if language != restrict_to_language: # This record is for a book in a different language cls.log.debug("WRONG LANGUAGE: %s", language) return None if 'authors' in restrictions: restrict_to_authors = restrictions['authors'] if restrict_to_authors and isinstance(restrict_to_authors[0], Contributor): restrict_to_authors = [ x.sort_name for x in restrict_to_authors ] primary_author = None for a, roles in authors_and_roles: if Contributor.PRIMARY_AUTHOR_ROLE in roles: primary_author = a break if (not primary_author or (primary_author not in restrict_to_authors and primary_author.sort_name not in restrict_to_authors)): # None of the given authors showed up as the # primary author of this book. They may have had # some other role in it, or the book may be about # them, or incorporate their work, but this book # is not *by* them. return None author_names = ", ".join([x.sort_name for x, y in authors_and_roles]) return title, authors_and_roles, language UNUSED_MEDIA = set([ "itemtype-intmm", "itemtype-msscr", "itemtype-artchap-artcl", "itemtype-jrnl", "itemtype-map", "itemtype-vis", "itemtype-jrnl-digital", "itemtype-image-2d", "itemtype-artchap-digital", "itemtype-intmm-digital", "itemtype-archv", "itemtype-msscr-digital", "itemtype-game", "itemtype-web-digital", "itemtype-map-digital", ]) @classmethod def extract_edition(cls, _db, work_tag, existing_authors, **restrictions): """Create a new Edition object with information about a work (identified by OCLC Work ID). """ # TODO: 'pswid' is what it's called in older representations. # That code can be removed once we replace all representations. oclc_work_id = unicode(work_tag.get('owi') or work_tag.get('pswid')) # if oclc_work_id: # print " owi: %s" % oclc_work_id # else: # print " No owi in %s" % etree.tostring(work_tag) if not oclc_work_id: raise ValueError("Work has no owi") item_type = work_tag.get("itemtype") if (item_type.startswith('itemtype-book') or item_type.startswith('itemtype-compfile')): medium = Edition.BOOK_MEDIUM elif item_type.startswith('itemtype-audiobook' ) or item_type.startswith('itemtype-music'): # Pretty much all Gutenberg texts, even the audio texts, # are based on a book, and the ones that aren't # (recordings of individual songs) probably aren't in OCLC # anyway. So we just want to get the books. medium = Edition.AUDIO_MEDIUM medium = None elif item_type.startswith('itemtype-video'): #medium = Edition.VIDEO_MEDIUM medium = None elif item_type in cls.UNUSED_MEDIA: medium = None else: medium = None # Only create Editions for books with a recognized medium if medium is None: return None, False result = cls._extract_basic_info(_db, work_tag, existing_authors, **restrictions) if not result: # This record did not meet one of the restrictions. return None, False title, authors_and_roles, language = result # Record some extra OCLC-specific information editions = work_tag.get('editions') holdings = work_tag.get('holdings') # Get an identifier for this work. identifier, ignore = Identifier.for_foreign_id(_db, Identifier.OCLC_WORK, oclc_work_id) data_source = DataSource.lookup(_db, DataSource.OCLC) identifier.add_measurement(data_source, Measurement.HOLDINGS, holdings) identifier.add_measurement(data_source, Measurement.PUBLISHED_EDITIONS, editions) # Create a Edition for source + identifier edition, new = get_one_or_create(_db, Edition, data_source=data_source, primary_identifier=identifier, create_method_kwargs=dict( title=title, language=language, )) # Get the most popular Dewey and LCC classification for this # work. for tag_name, subject_type in (("ddc", Subject.DDC), ("lcc", Subject.LCC)): tag = cls._xpath1(work_tag, "//oclc:%s/oclc:mostPopular" % tag_name) if tag is not None: id = tag.get('nsfa') or tag.get('sfa') weight = int(tag.get('holdings')) identifier.classify(data_source, subject_type, id, weight=weight) # Find FAST subjects for the work. for heading in cls._xpath(work_tag, "//oclc:fast//oclc:heading"): id = heading.get('ident') weight = int(heading.get('heldby')) value = heading.text identifier.classify(data_source, Subject.FAST, id, value, weight) # Associate the authors with the Edition. for contributor, roles in authors_and_roles: edition.add_contributor(contributor, roles) return edition, new @classmethod def extract_edition_record(cls, _db, edition_tag, existing_authors, **restrictions): """Create a new Edition object with information about an edition of a book (identified by OCLC Number). """ oclc_number = unicode(edition_tag.get('oclc')) try: int(oclc_number) except ValueError, e: # This record does not have a valid OCLC number. return None, False # Fill in some basic information about this new record. result = cls._extract_basic_info(_db, edition_tag, existing_authors, **restrictions) if not result: # This record did not meet one of the restrictions. return None, False title, authors_and_roles, language = result # Add a couple extra bits of OCLC-specific information. extra = { OCLC.HOLDING_COUNT: edition_tag.get('holdings'), OCLC.FORMAT: edition_tag.get('itemtype'), } # Get an identifier for this edition. identifier, ignore = Identifier.for_foreign_id(_db, Identifier.OCLC_NUMBER, oclc_number) # Create a Edition for source + identifier data_source = DataSource.lookup(_db, DataSource.OCLC) edition_record, new = get_one_or_create(_db, Edition, data_source=data_source, primary_identifier=identifier, create_method_kwargs=dict( title=title, language=language, subjects=subjects, extra=extra, )) subjects = {} for subject_type, oclc_code in ((Subject.LCC, "050"), (Subject.DDC, "082")): classification = cls._xpath1( edition_tag, "oclc:classifications/oclc:class[@tag=%s]" % oclc_code) if classification is not None: value = classification.get("nsfa") or classification.get('sfa') identifier.classify(data_source, subject_type, value) # Associated each contributor with the new record. for author, roles in authors_and_roles: edition_record.add_contributor(author, roles) return edition_record, new
url = "%s/libraries/%s/patrons/%s" % (api.base_url, api.library_id, patron) response = api.request(url) return response.status_code == 200 # See if we have multiple RB Digital collections rb_digital_collections = _db.query(Collection.id)\ .select_from(ExternalIntegration)\ .join(ExternalIntegration.collections)\ .filter(ExternalIntegration.protocol == ExternalIntegration.RB_DIGITAL)\ .all() # We don't have to do any validation just update the credentials table if len(rb_digital_collections) == 1: rb_collection_id = rb_digital_collections[0][0] source = DataSource.lookup(_db, DataSource.RB_DIGITAL) update_statement = update(Credential)\ .where(and_( Credential.data_source_id == source.id, Credential.type == Credential.IDENTIFIER_FROM_REMOTE_SERVICE ))\ .values(collection_id=rb_collection_id) _db.execute(update_statement) # We have multiple RBDigital integration and we don't know which credential # belongs to each one. Have to check each credential against RBDigital API. else: rb_api = [] for collection_id in rb_digital_collections: collection = Collection.by_id(_db, collection_id[0]) rb_api.append(RBDigitalAPI(_db, collection))
def setup(self): super(TestURNLookupController, self).setup() self.controller = URNLookupController(self._db) self.source = DataSource.lookup(self._db, DataSource.INTERNAL_PROCESSING)
def test_fulfill(self): patron = self._patron() data_source = DataSource.lookup(self._db, "Biblioboard", autocreate=True) edition, pool = self._edition( identifier_type=Identifier.URI, data_source_name=data_source.name, with_license_pool=True, collection=self.collection, ) # This pool doesn't have an acquisition link, so # we can't fulfill it yet. assert_raises(CannotFulfill, self.api.fulfill, patron, "1234", pool, Representation.EPUB_MEDIA_TYPE) # Set up an epub acquisition link for the pool. url = self._url link, ignore = pool.identifier.add_link( Hyperlink.GENERIC_OPDS_ACQUISITION, url, data_source, Representation.EPUB_MEDIA_TYPE, ) pool.set_delivery_mechanism( Representation.EPUB_MEDIA_TYPE, DeliveryMechanism.NO_DRM, RightsStatus.IN_COPYRIGHT, link.resource, ) # Set the API's auth url so it doesn't have to get it - # that's tested in test_get_token. self.api.auth_url = "http://auth" token_response = json.dumps({ "access_token": "token", "expires_in": 60 }) self.api.queue_response(200, content=token_response) fulfillment_time = datetime.datetime.utcnow() fulfillment_info = self.api.fulfill(patron, "1234", pool, Representation.EPUB_MEDIA_TYPE) eq_(self.collection.id, fulfillment_info.collection_id) eq_(data_source.name, fulfillment_info.data_source_name) eq_(Identifier.URI, fulfillment_info.identifier_type) eq_(pool.identifier.identifier, fulfillment_info.identifier) eq_(None, fulfillment_info.content_link) eq_(DeliveryMechanism.BEARER_TOKEN, fulfillment_info.content_type) bearer_token_document = json.loads(fulfillment_info.content) expires_in = bearer_token_document['expires_in'] assert expires_in < 60 eq_("Bearer", bearer_token_document['token_type']) eq_("token", bearer_token_document['access_token']) eq_(url, bearer_token_document['location']) # The FulfillmentInfo's content_expires is approximately the # time you get if you add the number of seconds until the # bearer token expires to the time at which the title was # originally fulfilled. expect_expiration = fulfillment_time + datetime.timedelta( seconds=expires_in) assert abs((fulfillment_info.content_expires - expect_expiration).total_seconds()) < 5
def data_source(self): return DataSource.lookup(self._db, DataSource.CONTENT_CAFE)
def test_recursively_equivalent_identifiers(self): # We start with a Gutenberg book. gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG) record, ignore = Edition.for_foreign_id(self._db, gutenberg, Identifier.GUTENBERG_ID, "100") gutenberg_id = record.primary_identifier # We use OCLC Classify to do a title/author lookup. oclc = DataSource.lookup(self._db, DataSource.OCLC) search_id, ignore = Identifier.for_foreign_id(self._db, Identifier.OCLC_WORK, "60010") gutenberg_id.equivalent_to(oclc, search_id, 1) # The title/author lookup associates the search term with two # different OCLC Numbers. oclc_id, ignore = Identifier.for_foreign_id(self._db, Identifier.OCLC_NUMBER, "9999") oclc_id_2, ignore = Identifier.for_foreign_id(self._db, Identifier.OCLC_NUMBER, "1000") search_id.equivalent_to(oclc, oclc_id, 1) search_id.equivalent_to(oclc, oclc_id_2, 1) # We then use OCLC Linked Data to connect one of the OCLC # Numbers with an ISBN. linked_data = DataSource.lookup(self._db, DataSource.OCLC_LINKED_DATA) isbn_id, ignore = Identifier.for_foreign_id(self._db, Identifier.ISBN, "900100434X") oclc_id.equivalent_to(linked_data, isbn_id, 1) # As it turns out, we have an Overdrive work record... overdrive = DataSource.lookup(self._db, DataSource.OVERDRIVE) overdrive_record, ignore = Edition.for_foreign_id( self._db, overdrive, Identifier.OVERDRIVE_ID, "{111-222}") overdrive_id = overdrive_record.primary_identifier # ...which is tied (by Overdrive) to the same ISBN. overdrive_id.equivalent_to(overdrive, isbn_id, 1) # Finally, here's a completely unrelated Edition, which # will not be showing up. gutenberg2, ignore = Edition.for_foreign_id(self._db, gutenberg, Identifier.GUTENBERG_ID, "200") gutenberg2.title = "Unrelated Gutenberg record." levels = [ record.equivalent_identifiers(policy=PresentationCalculationPolicy( equivalent_identifier_levels=i)) for i in range(0, 5) ] # At level 0, the only identifier found is the Gutenberg ID. assert set([gutenberg_id]) == set(levels[0]) # At level 1, we pick up the title/author lookup. assert set([gutenberg_id, search_id]) == set(levels[1]) # At level 2, we pick up the title/author lookup and the two # OCLC Numbers. assert set([gutenberg_id, search_id, oclc_id, oclc_id_2]) == set(levels[2]) # At level 3, we also pick up the ISBN. assert set([gutenberg_id, search_id, oclc_id, oclc_id_2, isbn_id]) == set(levels[3]) # At level 4, the recursion starts to go in the other # direction: we pick up the Overdrive ID that's equivalent to # the same ISBN as the OCLC Number. assert set([ gutenberg_id, search_id, oclc_id, oclc_id_2, isbn_id, overdrive_id ]) == set(levels[4])
def source(self): return DataSource.lookup(self._db, DataSource.NOVELIST)
def custom_lists(self, identifier_type, identifier): self.require_librarian(flask.request.library) library = flask.request.library work = self.load_work(library, identifier_type, identifier) if isinstance(work, ProblemDetail): return work staff_data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF) if flask.request.method == "GET": lists = [] for entry in work.custom_list_entries: list = entry.customlist lists.append(dict(id=list.id, name=list.name)) return dict(custom_lists=lists) if flask.request.method == "POST": lists = flask.request.form.get("lists") if lists: lists = json.loads(lists) else: lists = [] affected_lanes = set() # Remove entries for lists that were not in the submitted form. submitted_ids = [l.get("id") for l in lists if l.get("id")] for entry in work.custom_list_entries: if entry.list_id not in submitted_ids: list = entry.customlist list.remove_entry(work) for lane in Lane.affected_by_customlist(list): affected_lanes.add(lane) # Add entries for any new lists. for list_info in lists: id = list_info.get("id") name = list_info.get("name") if id: is_new = False list = get_one(self._db, CustomList, id=int(id), name=name, library=library, data_source=staff_data_source) if not list: self._db.rollback() return MISSING_CUSTOM_LIST.detailed( _("Could not find list \"%(list_name)s\"", list_name=name)) else: list, is_new = create(self._db, CustomList, name=name, data_source=staff_data_source, library=library) list.created = datetime.now() entry, was_new = list.add_entry(work, featured=True) if was_new: for lane in Lane.affected_by_customlist(list): affected_lanes.add(lane) # If any list changes affected lanes, update their sizes. # NOTE: This may not make a difference until the # works are actually re-indexed. for lane in affected_lanes: lane.update_size(self._db, self.search_engine) return Response(unicode(_("Success")), 200)