def unaffiliated_collection(cls, _db): """Find a special metadata-wrangler-specific Collection whose catalog contains identifiers that came in through anonymous lookup. """ return Collection.by_name_and_protocol( _db, "Unaffiliated Identifiers", DataSource.INTERNAL_PROCESSING )
def collection(self): """Returns an associated Collection object :return: Associated Collection object :rtype: Collection """ return Collection.by_id(self._db, id=self._collection_id)
def look_up_collection_by_id(self, identifier): """Find the collection to display self test results or run self tests for; display an error message if a collection with this ID turns out not to exist""" collection = Collection.by_id(self._db, identifier) if not collection: return NO_SUCH_COLLECTION return collection
def create_overdrive_api(self, overdrive_api_class): collection, is_new = Collection.by_name_and_protocol( self._db, self.DEFAULT_OVERDRIVE_COLLECTION_NAME, ExternalIntegration.OVERDRIVE) if is_new: raise ValueError( 'Default Overdrive collection has not been configured.') return overdrive_api_class(self._db, collection)
def process_get(self): collections_db = self._db.query(Collection).order_by( Collection.name).all() ConfigurationSetting.cache_warm(self._db) Collection.cache_warm(self._db, lambda: collections_db) protocols = self._get_collection_protocols() user = flask.request.admin collections = [] protocolClass = None for collection_object in collections_db: if not user or not user.can_see_collection(collection_object): continue collection_dict = self.collection_to_dict(collection_object) if collection_object.protocol in [ p.get("name") for p in protocols ]: [protocol] = [ p for p in protocols if p.get("name") == collection_object.protocol ] libraries = self.load_libraries(collection_object, user, protocol) collection_dict["libraries"] = libraries settings = self.load_settings( protocol.get("settings"), collection_object, collection_dict.get("settings"), ) collection_dict["settings"] = settings protocolClass = self.find_protocol_class(collection_object) collection_dict[ "self_test_results"] = self._get_prior_test_results( collection_object, protocolClass) collection_dict[ "marked_for_deletion"] = collection_object.marked_for_deletion collections.append(collection_dict) return dict( collections=collections, protocols=protocols, )
def run(self): """Explain every Overdrive collection and, for each one, all of its Advantage collections. """ collections = Collection.by_protocol(self._db, ExternalIntegration.OVERDRIVE) for collection in collections: self.explain_main_collection(collection) print
def run(self): """Explain every Overdrive collection and, for each one, all of its Advantage collections. """ collections = Collection.by_protocol( self._db, ExternalIntegration.OVERDRIVE ) for collection in collections: self.explain_main_collection(collection) print
def __init__(self, _db, collection=None, *args, **kwargs): self.responses = [] self.requests = [] library = DatabaseTest.make_default_library(_db) collection, ignore = Collection.by_name_and_protocol( _db, name="Test Enki Collection", protocol=EnkiAPI.ENKI) collection.protocol = EnkiAPI.ENKI collection.external_account_id = u'c' library.collections.append(collection) super(MockEnkiAPI, self).__init__(_db, collection, *args, **kwargs)
def authenticated_collection_from_request(self, required=True): header = request.authorization if header: client_id, client_secret = header.username, header.password collection = Collection.authenticate(self._db, client_id, client_secret) if collection: return collection if not required and not header: # In the case that authentication is not required # (i.e. URN lookup) return None instead of an error. return None return INVALID_CREDENTIALS
def all(cls, _db, **kwargs): if cls.PROTOCOL and cls.DATA_SOURCE_NAME: qu = Collection.by_protocol(_db, cls.PROTOCOL) qu = qu.join(ExternalIntegration.settings).filter( ConfigurationSetting.key == Collection.DATA_SOURCE_NAME_SETTING, ConfigurationSetting.value == cls.DATA_SOURCE_NAME).order_by( func.random()) for collection in qu: yield cls(collection, **kwargs) else: for collection in super(OPDSImportCoverageProvider, cls).all(_db, **kwargs): yield collection
def run(self, name): if not name: ValueError("No name provided. Could not create collection.") name = " ".join(name) print "Creating collection %s... " % name collection, plaintext_client_secret = Collection.register(self._db, name) print collection print ("RECORD THE FOLLOWING AUTHENTICATION DETAILS. " "The client secret cannot be recovered.") print "-" * 40 print "CLIENT ID: %s" % collection.client_id print "CLIENT SECRET: %s" % plaintext_client_secret
def collection_from_details(_db, client, collection_details): if not (client and isinstance(client, IntegrationClient)): return None if collection_details: # A DataSource may be sent for collections with the # ExternalIntegration.OPDS_IMPORT protocol. data_source_name = request.args.get('data_source') if data_source_name: data_source_name = urllib.unquote(data_source_name) collection, ignore = Collection.from_metadata_identifier( _db, collection_details, data_source=data_source_name) return collection return None
def collection_from_details(_db, client, collection_details): if not (client and isinstance(client, IntegrationClient)): return None if collection_details: # A DataSource may be sent for collections with the # ExternalIntegration.OPDS_IMPORT protocol. data_source_name = request.args.get('data_source') if data_source_name: data_source_name = urllib.unquote(data_source_name) collection, ignore = Collection.from_metadata_identifier( _db, collection_details, data_source=data_source_name ) return collection return None
def add_items(self, collection_details): """Adds identifiers to a Collection's catalog""" client = authenticated_client_from_request(self._db) if isinstance(client, ProblemDetail): return client collection, ignore = Collection.from_metadata_identifier( self._db, collection_details) urns = request.args.getlist('urn') messages = [] for urn in urns: message = None identifier = None try: identifier, ignore = Identifier.parse_urn(self._db, urn) except Exception as e: identifier = None if not identifier: message = OPDSMessage(urn, INVALID_URN.status_code, INVALID_URN.detail) else: status = HTTP_OK description = "Already in catalog" if identifier not in collection.catalog: collection.catalog_identifier(self._db, identifier) status = HTTP_CREATED description = "Successfully added" message = OPDSMessage(urn, status, description) messages.append(message) title = "%s Catalog Item Additions for %s" % (collection.protocol, client.url) url = cdn_url_for("add", collection_metadata_identifier=collection.name, urn=urns) addition_feed = AcquisitionFeed(self._db, title, url, [], VerboseAnnotator, precomposed_entries=messages) return feed_response(addition_feed)
def remove_items(self, collection_details): """Removes identifiers from a Collection's catalog""" client = authenticated_client_from_request(self._db) if isinstance(client, ProblemDetail): return client collection, ignore = Collection.from_metadata_identifier( self._db, collection_details) urns = request.args.getlist('urn') messages = [] for urn in urns: message = None identifier = None try: identifier, ignore = Identifier.parse_urn(self._db, urn) except Exception as e: identifier = None if not identifier: message = OPDSMessage(urn, INVALID_URN.status_code, INVALID_URN.detail) else: if identifier in collection.catalog: collection.catalog.remove(identifier) message = OPDSMessage(urn, HTTP_OK, "Successfully removed") else: message = OPDSMessage(urn, HTTP_NOT_FOUND, "Not in catalog") messages.append(message) title = "%s Catalog Item Removal for %s" % (collection.protocol, client.url) url = cdn_url_for("remove", collection_metadata_identifier=collection.name, urn=urns) removal_feed = AcquisitionFeed(self._db, title, url, [], VerboseAnnotator, precomposed_entries=messages) return feed_response(removal_feed)
def __init__(self, _db, collection=None, *args, **kwargs): self.responses = [] self.requests = [] library = DatabaseTest.make_default_library(_db) if not collection: collection, ignore = Collection.by_name_and_protocol( _db, name="Test Enki Collection", protocol=EnkiAPI.ENKI ) collection.protocol = EnkiAPI.ENKI if collection not in library.collections: library.collections.append(collection) # Set the "Enki library ID" variable between the default library # and this Enki collection. ConfigurationSetting.for_library_and_externalintegration( _db, self.ENKI_LIBRARY_ID_KEY, library, collection.external_integration ).value = "c" super(MockEnkiAPI, self).__init__(_db, collection, *args, **kwargs)
def _get_lcp_collection(self, patron, collection_name): """Returns an LCP collection for a specified library NOTE: We assume that there is only ONE LCP collection per library :param patron: Patron object :type patron: core.model.patron.Patron :param collection_name: Name of the collection :type collection_name: string :return: LCP collection :rtype: core.model.collection.Collection """ db = Session.object_session(patron) lcp_collection, _ = Collection.by_name_and_protocol( db, collection_name, ExternalIntegration.LCP) if not lcp_collection or lcp_collection not in patron.library.collections: return MISSING_COLLECTION return lcp_collection
def __init__(self, _db, collection=None, *args, **kwargs): self.responses = [] self.requests = [] library = DatabaseTest.make_default_library(_db) if not collection: collection, ignore = Collection.by_name_and_protocol( _db, name="Test Enki Collection", protocol=EnkiAPI.ENKI ) collection.protocol=EnkiAPI.ENKI if collection not in library.collections: library.collections.append(collection) # Set the "Enki library ID" variable between the default library # and this Enki collection. ConfigurationSetting.for_library_and_externalintegration( _db, self.ENKI_LIBRARY_ID_KEY, library, collection.external_integration ).value = 'c' super(MockEnkiAPI, self).__init__( _db, collection, *args, **kwargs )
def collection(self): return Collection.by_id(self._db, id=self.collection_id)
from sqlalchemy.orm import aliased from sqlalchemy import and_ from core.model import ( Collection, Hyperlink, LicensePool, Representation, Resource, production_session, ) from api.odl import SharedODLAPI try: _db = production_session() for collection in Collection.by_protocol(_db, SharedODLAPI.NAME): borrow_link = aliased(Hyperlink) open_link = aliased(Hyperlink) pools = _db.query(LicensePool).join( borrow_link, LicensePool.identifier_id == borrow_link.identifier_id, ).join( open_link, LicensePool.identifier_id == open_link.identifier_id, ).join( Resource, borrow_link.resource_id == Resource.id, ).join( Representation, Resource.representation_id == Representation.id,
# Create Collections generated by OPDS import with importer classes. opds_importers = { FeedbooksOPDSImporter : DataSource.FEEDBOOKS, UnglueItImporter : DataSource.UNGLUE_IT, } for importer_class, data_source_name in opds_importers.items(): OPDSImportScript(importer_class, data_source_name, _db=_db) # Create a StandardEbooks Collection. OPDSImportScript(object(), DataSource.STANDARD_EBOOKS, _db=_db, collection_data=dict(url=u'https://standardebooks.org/opds/all')) # Create a Gutenberg Collection. gutenberg, is_new = Collection.by_name_and_protocol( _db, DataSource.GUTENBERG, ExternalIntegration.GUTENBERG ) if not gutenberg.data_source: gutenberg.external_integration.set_setting( Collection.DATA_SOURCE_NAME_SETTING, DataSource.GUTENBERG ) if is_new: library = Library.default(_db) gutenberg.libraries.append(library) logging.info('CREATED Collection for %s: %r' % ( DataSource.GUTENBERG, gutenberg)) _db.commit() # Alright, all the Collections have been created. Let's update the # LicensePools now.
from core.model import ( Collection, ConfigurationSetting, DataSource, ExternalIntegration, production_session, ) log = logging.getLogger(name="Metadata Wrangler configuration import") try: _db = production_session() # Get all of the OPDS_IMPORT collections. collections = Collection.by_protocol(_db, ExternalIntegration.OPDS_IMPORT) for collection in collections: opds_url = collection.external_account_id if not opds_url: decoded_collection, ignore = Collection.from_metadata_identifier( _db, collection.name) opds_url = decoded_collection.external_account_id if not opds_url: # This shouldn't happen. log.warn('Could not find external_account_id for %r' % collection) continue if opds_url and collection == decoded_collection: log.info('Added URL "%s" to collection %r',
from sqlalchemy.orm import aliased from sqlalchemy import and_ from core.model import ( Collection, Hyperlink, LicensePool, Representation, Resource, production_session, ) from api.odl import SharedODLAPI try: _db = production_session() for collection in Collection.by_protocol(_db, SharedODLAPI.NAME): borrow_link = aliased(Hyperlink) open_link = aliased(Hyperlink) pools = _db.query( LicensePool ).join( borrow_link, LicensePool.identifier_id==borrow_link.identifier_id, ).join( open_link, LicensePool.identifier_id==open_link.identifier_id, ).join( Resource, borrow_link.resource_id==Resource.id, ).join(
def collection(self, _db): """Find the Collection to which this object belongs.""" return Collection.by_id(_db, self.collection_id)
def unaffiliated_collection(cls, _db): """Find a special metadata-wrangler-specific Collection whose catalog contains identifiers that came in through anonymous lookup. """ return Collection.by_name_and_protocol(_db, "Unaffiliated Identifiers", DataSource.INTERNAL_PROCESSING)
class URNLookupController(CoreURNLookupController): UNRESOLVABLE_IDENTIFIER = "I can't gather information about an identifier of this type." IDENTIFIER_REGISTERED = "You're the first one to ask about this identifier. I'll try to find out about it." WORKING_TO_RESOLVE_IDENTIFIER = "I'm working to locate a source for this identifier." SUCCESS_DID_NOT_RESULT_IN_PRESENTATION_READY_WORK = "Something's wrong. I have a record of covering this identifier but there's no presentation-ready work to show you." OPERATION = CoverageRecord.RESOLVE_IDENTIFIER_OPERATION NO_WORK_DONE_EXCEPTION = u'No work done yet' log = logging.getLogger("URN lookup controller") def presentation_ready_work_for(self, identifier): """Either return a presentation-ready work associated with the given `identifier`, or return None. """ pools = identifier.licensed_through if not pools: return None # All LicensePools for a given Identifier have the same Work. work = pools[0].work if not work or not work.presentation_ready: return None return work def can_resolve_identifier(self, identifier): """A chance to determine whether resolution should proceed.""" # We can resolve any ISBN and any Overdrive ID. # # We can resolve any Gutenberg ID by looking it up in the open-access # content server. # # We can attempt to resolve URIs by looking them up in the # open-access content server, though there's no guarantee # it will work. if identifier.type in (Identifier.ISBN, Identifier.OVERDRIVE_ID, Identifier.GUTENBERG_ID, Identifier.URI): return True # We can resolve any identifier that's associated with a # presentation-ready work, since the resolution has already # been done--no need to speculate about how. work = self.presentation_ready_work_for(identifier) if work is None: return False return True def process_urn(self, urn, collection_details=None, **kwargs): """Turn a URN into a Work suitable for use in an OPDS feed. """ try: identifier, is_new = Identifier.parse_urn(self._db, urn) except ValueError, e: identifier = None if not identifier: # Not a well-formed URN. return self.add_message(urn, 400, INVALID_URN.detail) if not self.can_resolve_identifier(identifier): return self.add_message(urn, HTTP_NOT_FOUND, self.UNRESOLVABLE_IDENTIFIER) # We are at least willing to try to resolve this Identifier. # If a Collection was provided by an authenticated IntegrationClient, # this Identifier is part of the Collection's catalog. client = authenticated_client_from_request(self._db, required=False) if client and collection_details: collection, ignore = Collection.from_metadata_identifier( self._db, collection_details) collection.catalog_identifier(self._db, identifier) if (identifier.type == Identifier.ISBN and not identifier.work): # There's not always enough information about an ISBN to # create a full Work. If not, we scrape together the cover # and description information and force the entry. return self.make_opds_entry_from_metadata_lookups(identifier) # All other identifiers need to be associated with a # presentation-ready Work for the lookup to succeed. If there # isn't one, we need to register it as unresolved. work = self.presentation_ready_work_for(identifier) if work: # The work has been done. return self.add_work(identifier, work) # Work remains to be done. return self.register_identifier_as_unresolved(urn, identifier)
def updates_feed(self, collection_details): client = authenticated_client_from_request(self._db) if isinstance(client, ProblemDetail): return client collection, ignore = Collection.from_metadata_identifier( self._db, collection_details) last_update_time = request.args.get('last_update_time', None) if last_update_time: last_update_time = datetime.strptime(last_update_time, "%Y-%m-%dT%H:%M:%SZ") updated_works = collection.works_updated_since(self._db, last_update_time) pagination = load_pagination_from_request() works = pagination.apply(updated_works).all() title = "%s Collection Updates for %s" % (collection.protocol, client.url) def update_url(time=last_update_time, page=None): kw = dict(_external=True, collection_metadata_identifier=collection_details) if time: kw.update({'last_update_time': last_update_time}) if page: kw.update(page.items()) return cdn_url_for("updates", **kw) entries = [] for work in works[:]: entry = work.verbose_opds_entry or work.simple_opds_entry entry = etree.fromstring(entry) if entry: entries.append(entry) works.remove(work) works = [(work.identifier, work) for work in works] update_feed = LookupAcquisitionFeed(self._db, title, update_url(), works, VerboseAnnotator, precomposed_entries=entries) if len(updated_works.all()) > pagination.size + pagination.offset: update_feed.add_link_to_feed( update_feed.feed, rel="next", href=update_url(page=pagination.next_page)) if pagination.offset > 0: update_feed.add_link_to_feed( update_feed.feed, rel="first", href=update_url(page=pagination.first_page)) previous_page = pagination.previous_page if previous_page: update_feed.add_link_to_feed(update_feed.feed, rel="previous", href=update_url(page=previous_page)) return feed_response(update_feed)
from core.model import ( Collection, ConfigurationSetting, DataSource, ExternalIntegration, production_session, ) log = logging.getLogger(name="Metadata Wrangler configuration import") try: _db = production_session() # Get all of the OPDS_IMPORT collections. collections = Collection.by_protocol(_db, ExternalIntegration.OPDS_IMPORT) for collection in collections: opds_url = collection.external_account_id if not opds_url: decoded_collection, ignore = Collection.from_metadata_identifier( _db, collection.name ) opds_url = decoded_collection.external_account_id if not opds_url: # This shouldn't happen. log.warn( 'Could not find external_account_id for %r' % collection ) continue
rb_collection_id = rb_digital_collections[0][0] source = DataSource.lookup(_db, DataSource.RB_DIGITAL) update_statement = update(Credential)\ .where(and_( Credential.data_source_id == source.id, Credential.type == Credential.IDENTIFIER_FROM_REMOTE_SERVICE ))\ .values(collection_id=rb_collection_id) _db.execute(update_statement) # We have multiple RBDigital integration and we don't know which credential # belongs to each one. Have to check each credential against RBDigital API. else: rb_api = [] for collection_id in rb_digital_collections: collection = Collection.by_id(_db, collection_id[0]) rb_api.append(RBDigitalAPI(_db, collection)) source = DataSource.lookup(_db, DataSource.RB_DIGITAL) credentials = _db.query(Credential)\ .filter( Credential.data_source_id == source.id, Credential.type == Credential.IDENTIFIER_FROM_REMOTE_SERVICE, Credential.credential != None ) for credential in credentials: for api in rb_api: if check_patron_id(api, credential.credential): credential.collection = api.collection break # Remove credentials stored with none as the credential