def unaffiliated_collection(cls, _db):
"""Find a special metadata-wrangler-specific Collection whose catalog
contains identifiers that came in through anonymous lookup.
"""
return Collection.by_name_and_protocol(
_db, "Unaffiliated Identifiers", DataSource.INTERNAL_PROCESSING
)
def collection(self):
"""Returns an associated Collection object
:return: Associated Collection object
:rtype: Collection
"""
return Collection.by_id(self._db, id=self._collection_id)
def look_up_collection_by_id(self, identifier):
"""Find the collection to display self test results or run self tests for;
display an error message if a collection with this ID turns out not to exist"""
collection = Collection.by_id(self._db, identifier)
if not collection:
return NO_SUCH_COLLECTION
return collection
def create_overdrive_api(self, overdrive_api_class):
collection, is_new = Collection.by_name_and_protocol(
self._db, self.DEFAULT_OVERDRIVE_COLLECTION_NAME,
ExternalIntegration.OVERDRIVE)
if is_new:
raise ValueError(
'Default Overdrive collection has not been configured.')
return overdrive_api_class(self._db, collection)
def process_get(self):
collections_db = self._db.query(Collection).order_by(
Collection.name).all()
ConfigurationSetting.cache_warm(self._db)
Collection.cache_warm(self._db, lambda: collections_db)
protocols = self._get_collection_protocols()
user = flask.request.admin
collections = []
protocolClass = None
for collection_object in collections_db:
if not user or not user.can_see_collection(collection_object):
continue
collection_dict = self.collection_to_dict(collection_object)
if collection_object.protocol in [
p.get("name") for p in protocols
]:
[protocol] = [
p for p in protocols
if p.get("name") == collection_object.protocol
]
libraries = self.load_libraries(collection_object, user,
protocol)
collection_dict["libraries"] = libraries
settings = self.load_settings(
protocol.get("settings"),
collection_object,
collection_dict.get("settings"),
)
collection_dict["settings"] = settings
protocolClass = self.find_protocol_class(collection_object)
collection_dict[
"self_test_results"] = self._get_prior_test_results(
collection_object, protocolClass)
collection_dict[
"marked_for_deletion"] = collection_object.marked_for_deletion
collections.append(collection_dict)
return dict(
collections=collections,
protocols=protocols,
)
def run(self):
"""Explain every Overdrive collection and, for each one, all of its
Advantage collections.
"""
collections = Collection.by_protocol(self._db,
ExternalIntegration.OVERDRIVE)
for collection in collections:
self.explain_main_collection(collection)
print
def run(self):
"""Explain every Overdrive collection and, for each one, all of its
Advantage collections.
"""
collections = Collection.by_protocol(
self._db, ExternalIntegration.OVERDRIVE
)
for collection in collections:
self.explain_main_collection(collection)
print
def __init__(self, _db, collection=None, *args, **kwargs):
self.responses = []
self.requests = []
library = DatabaseTest.make_default_library(_db)
collection, ignore = Collection.by_name_and_protocol(
_db, name="Test Enki Collection", protocol=EnkiAPI.ENKI)
collection.protocol = EnkiAPI.ENKI
collection.external_account_id = u'c'
library.collections.append(collection)
super(MockEnkiAPI, self).__init__(_db, collection, *args, **kwargs)
def authenticated_collection_from_request(self, required=True):
header = request.authorization
if header:
client_id, client_secret = header.username, header.password
collection = Collection.authenticate(self._db, client_id, client_secret)
if collection:
return collection
if not required and not header:
# In the case that authentication is not required
# (i.e. URN lookup) return None instead of an error.
return None
return INVALID_CREDENTIALS
def all(cls, _db, **kwargs):
if cls.PROTOCOL and cls.DATA_SOURCE_NAME:
qu = Collection.by_protocol(_db, cls.PROTOCOL)
qu = qu.join(ExternalIntegration.settings).filter(
ConfigurationSetting.key ==
Collection.DATA_SOURCE_NAME_SETTING,
ConfigurationSetting.value == cls.DATA_SOURCE_NAME).order_by(
func.random())
for collection in qu:
yield cls(collection, **kwargs)
else:
for collection in super(OPDSImportCoverageProvider,
cls).all(_db, **kwargs):
yield collection
def run(self, name):
if not name:
ValueError("No name provided. Could not create collection.")
name = " ".join(name)
print "Creating collection %s... " % name
collection, plaintext_client_secret = Collection.register(self._db, name)
print collection
print ("RECORD THE FOLLOWING AUTHENTICATION DETAILS. "
"The client secret cannot be recovered.")
print "-" * 40
print "CLIENT ID: %s" % collection.client_id
print "CLIENT SECRET: %s" % plaintext_client_secret
def collection_from_details(_db, client, collection_details):
if not (client and isinstance(client, IntegrationClient)):
return None
if collection_details:
# A DataSource may be sent for collections with the
# ExternalIntegration.OPDS_IMPORT protocol.
data_source_name = request.args.get('data_source')
if data_source_name:
data_source_name = urllib.unquote(data_source_name)
collection, ignore = Collection.from_metadata_identifier(
_db, collection_details, data_source=data_source_name)
return collection
return None
def collection_from_details(_db, client, collection_details):
if not (client and isinstance(client, IntegrationClient)):
return None
if collection_details:
# A DataSource may be sent for collections with the
# ExternalIntegration.OPDS_IMPORT protocol.
data_source_name = request.args.get('data_source')
if data_source_name:
data_source_name = urllib.unquote(data_source_name)
collection, ignore = Collection.from_metadata_identifier(
_db, collection_details, data_source=data_source_name
)
return collection
return None
def add_items(self, collection_details):
"""Adds identifiers to a Collection's catalog"""
client = authenticated_client_from_request(self._db)
if isinstance(client, ProblemDetail):
return client
collection, ignore = Collection.from_metadata_identifier(
self._db, collection_details)
urns = request.args.getlist('urn')
messages = []
for urn in urns:
message = None
identifier = None
try:
identifier, ignore = Identifier.parse_urn(self._db, urn)
except Exception as e:
identifier = None
if not identifier:
message = OPDSMessage(urn, INVALID_URN.status_code,
INVALID_URN.detail)
else:
status = HTTP_OK
description = "Already in catalog"
if identifier not in collection.catalog:
collection.catalog_identifier(self._db, identifier)
status = HTTP_CREATED
description = "Successfully added"
message = OPDSMessage(urn, status, description)
messages.append(message)
title = "%s Catalog Item Additions for %s" % (collection.protocol,
client.url)
url = cdn_url_for("add",
collection_metadata_identifier=collection.name,
urn=urns)
addition_feed = AcquisitionFeed(self._db,
title,
url, [],
VerboseAnnotator,
precomposed_entries=messages)
return feed_response(addition_feed)
def remove_items(self, collection_details):
"""Removes identifiers from a Collection's catalog"""
client = authenticated_client_from_request(self._db)
if isinstance(client, ProblemDetail):
return client
collection, ignore = Collection.from_metadata_identifier(
self._db, collection_details)
urns = request.args.getlist('urn')
messages = []
for urn in urns:
message = None
identifier = None
try:
identifier, ignore = Identifier.parse_urn(self._db, urn)
except Exception as e:
identifier = None
if not identifier:
message = OPDSMessage(urn, INVALID_URN.status_code,
INVALID_URN.detail)
else:
if identifier in collection.catalog:
collection.catalog.remove(identifier)
message = OPDSMessage(urn, HTTP_OK, "Successfully removed")
else:
message = OPDSMessage(urn, HTTP_NOT_FOUND,
"Not in catalog")
messages.append(message)
title = "%s Catalog Item Removal for %s" % (collection.protocol,
client.url)
url = cdn_url_for("remove",
collection_metadata_identifier=collection.name,
urn=urns)
removal_feed = AcquisitionFeed(self._db,
title,
url, [],
VerboseAnnotator,
precomposed_entries=messages)
return feed_response(removal_feed)
def __init__(self, _db, collection=None, *args, **kwargs):
self.responses = []
self.requests = []
library = DatabaseTest.make_default_library(_db)
if not collection:
collection, ignore = Collection.by_name_and_protocol(
_db, name="Test Enki Collection", protocol=EnkiAPI.ENKI
)
collection.protocol = EnkiAPI.ENKI
if collection not in library.collections:
library.collections.append(collection)
# Set the "Enki library ID" variable between the default library
# and this Enki collection.
ConfigurationSetting.for_library_and_externalintegration(
_db, self.ENKI_LIBRARY_ID_KEY, library, collection.external_integration
).value = "c"
super(MockEnkiAPI, self).__init__(_db, collection, *args, **kwargs)
def _get_lcp_collection(self, patron, collection_name):
"""Returns an LCP collection for a specified library
NOTE: We assume that there is only ONE LCP collection per library
:param patron: Patron object
:type patron: core.model.patron.Patron
:param collection_name: Name of the collection
:type collection_name: string
:return: LCP collection
:rtype: core.model.collection.Collection
"""
db = Session.object_session(patron)
lcp_collection, _ = Collection.by_name_and_protocol(
db, collection_name, ExternalIntegration.LCP)
if not lcp_collection or lcp_collection not in patron.library.collections:
return MISSING_COLLECTION
return lcp_collection
def __init__(self, _db, collection=None, *args, **kwargs):
self.responses = []
self.requests = []
library = DatabaseTest.make_default_library(_db)
if not collection:
collection, ignore = Collection.by_name_and_protocol(
_db, name="Test Enki Collection", protocol=EnkiAPI.ENKI
)
collection.protocol=EnkiAPI.ENKI
if collection not in library.collections:
library.collections.append(collection)
# Set the "Enki library ID" variable between the default library
# and this Enki collection.
ConfigurationSetting.for_library_and_externalintegration(
_db, self.ENKI_LIBRARY_ID_KEY, library,
collection.external_integration
).value = 'c'
super(MockEnkiAPI, self).__init__(
_db, collection, *args, **kwargs
)
def collection(self):
return Collection.by_id(self._db, id=self.collection_id)
from sqlalchemy.orm import aliased
from sqlalchemy import and_
from core.model import (
Collection,
Hyperlink,
LicensePool,
Representation,
Resource,
production_session,
)
from api.odl import SharedODLAPI
try:
_db = production_session()
for collection in Collection.by_protocol(_db, SharedODLAPI.NAME):
borrow_link = aliased(Hyperlink)
open_link = aliased(Hyperlink)
pools = _db.query(LicensePool).join(
borrow_link,
LicensePool.identifier_id == borrow_link.identifier_id,
).join(
open_link,
LicensePool.identifier_id == open_link.identifier_id,
).join(
Resource,
borrow_link.resource_id == Resource.id,
).join(
Representation,
Resource.representation_id == Representation.id,
# Create Collections generated by OPDS import with importer classes.
opds_importers = {
FeedbooksOPDSImporter : DataSource.FEEDBOOKS,
UnglueItImporter : DataSource.UNGLUE_IT,
}
for importer_class, data_source_name in opds_importers.items():
OPDSImportScript(importer_class, data_source_name, _db=_db)
# Create a StandardEbooks Collection.
OPDSImportScript(object(), DataSource.STANDARD_EBOOKS, _db=_db,
collection_data=dict(url=u'https://standardebooks.org/opds/all'))
# Create a Gutenberg Collection.
gutenberg, is_new = Collection.by_name_and_protocol(
_db, DataSource.GUTENBERG, ExternalIntegration.GUTENBERG
)
if not gutenberg.data_source:
gutenberg.external_integration.set_setting(
Collection.DATA_SOURCE_NAME_SETTING, DataSource.GUTENBERG
)
if is_new:
library = Library.default(_db)
gutenberg.libraries.append(library)
logging.info('CREATED Collection for %s: %r' % (
DataSource.GUTENBERG, gutenberg))
_db.commit()
# Alright, all the Collections have been created. Let's update the
# LicensePools now.
from core.model import (
Collection,
ConfigurationSetting,
DataSource,
ExternalIntegration,
production_session,
)
log = logging.getLogger(name="Metadata Wrangler configuration import")
try:
_db = production_session()
# Get all of the OPDS_IMPORT collections.
collections = Collection.by_protocol(_db, ExternalIntegration.OPDS_IMPORT)
for collection in collections:
opds_url = collection.external_account_id
if not opds_url:
decoded_collection, ignore = Collection.from_metadata_identifier(
_db, collection.name)
opds_url = decoded_collection.external_account_id
if not opds_url:
# This shouldn't happen.
log.warn('Could not find external_account_id for %r' %
collection)
continue
if opds_url and collection == decoded_collection:
log.info('Added URL "%s" to collection %r',
from sqlalchemy.orm import aliased
from sqlalchemy import and_
from core.model import (
Collection,
Hyperlink,
LicensePool,
Representation,
Resource,
production_session,
)
from api.odl import SharedODLAPI
try:
_db = production_session()
for collection in Collection.by_protocol(_db, SharedODLAPI.NAME):
borrow_link = aliased(Hyperlink)
open_link = aliased(Hyperlink)
pools = _db.query(
LicensePool
).join(
borrow_link,
LicensePool.identifier_id==borrow_link.identifier_id,
).join(
open_link,
LicensePool.identifier_id==open_link.identifier_id,
).join(
Resource,
borrow_link.resource_id==Resource.id,
).join(
def collection(self, _db):
"""Find the Collection to which this object belongs."""
return Collection.by_id(_db, self.collection_id)
def unaffiliated_collection(cls, _db):
"""Find a special metadata-wrangler-specific Collection whose catalog
contains identifiers that came in through anonymous lookup.
"""
return Collection.by_name_and_protocol(_db, "Unaffiliated Identifiers",
DataSource.INTERNAL_PROCESSING)
class URNLookupController(CoreURNLookupController):
UNRESOLVABLE_IDENTIFIER = "I can't gather information about an identifier of this type."
IDENTIFIER_REGISTERED = "You're the first one to ask about this identifier. I'll try to find out about it."
WORKING_TO_RESOLVE_IDENTIFIER = "I'm working to locate a source for this identifier."
SUCCESS_DID_NOT_RESULT_IN_PRESENTATION_READY_WORK = "Something's wrong. I have a record of covering this identifier but there's no presentation-ready work to show you."
OPERATION = CoverageRecord.RESOLVE_IDENTIFIER_OPERATION
NO_WORK_DONE_EXCEPTION = u'No work done yet'
log = logging.getLogger("URN lookup controller")
def presentation_ready_work_for(self, identifier):
"""Either return a presentation-ready work associated with the
given `identifier`, or return None.
"""
pools = identifier.licensed_through
if not pools:
return None
# All LicensePools for a given Identifier have the same Work.
work = pools[0].work
if not work or not work.presentation_ready:
return None
return work
def can_resolve_identifier(self, identifier):
"""A chance to determine whether resolution should proceed."""
# We can resolve any ISBN and any Overdrive ID.
#
# We can resolve any Gutenberg ID by looking it up in the open-access
# content server.
#
# We can attempt to resolve URIs by looking them up in the
# open-access content server, though there's no guarantee
# it will work.
if identifier.type in (Identifier.ISBN, Identifier.OVERDRIVE_ID,
Identifier.GUTENBERG_ID, Identifier.URI):
return True
# We can resolve any identifier that's associated with a
# presentation-ready work, since the resolution has already
# been done--no need to speculate about how.
work = self.presentation_ready_work_for(identifier)
if work is None:
return False
return True
def process_urn(self, urn, collection_details=None, **kwargs):
"""Turn a URN into a Work suitable for use in an OPDS feed.
"""
try:
identifier, is_new = Identifier.parse_urn(self._db, urn)
except ValueError, e:
identifier = None
if not identifier:
# Not a well-formed URN.
return self.add_message(urn, 400, INVALID_URN.detail)
if not self.can_resolve_identifier(identifier):
return self.add_message(urn, HTTP_NOT_FOUND,
self.UNRESOLVABLE_IDENTIFIER)
# We are at least willing to try to resolve this Identifier.
# If a Collection was provided by an authenticated IntegrationClient,
# this Identifier is part of the Collection's catalog.
client = authenticated_client_from_request(self._db, required=False)
if client and collection_details:
collection, ignore = Collection.from_metadata_identifier(
self._db, collection_details)
collection.catalog_identifier(self._db, identifier)
if (identifier.type == Identifier.ISBN and not identifier.work):
# There's not always enough information about an ISBN to
# create a full Work. If not, we scrape together the cover
# and description information and force the entry.
return self.make_opds_entry_from_metadata_lookups(identifier)
# All other identifiers need to be associated with a
# presentation-ready Work for the lookup to succeed. If there
# isn't one, we need to register it as unresolved.
work = self.presentation_ready_work_for(identifier)
if work:
# The work has been done.
return self.add_work(identifier, work)
# Work remains to be done.
return self.register_identifier_as_unresolved(urn, identifier)
def updates_feed(self, collection_details):
client = authenticated_client_from_request(self._db)
if isinstance(client, ProblemDetail):
return client
collection, ignore = Collection.from_metadata_identifier(
self._db, collection_details)
last_update_time = request.args.get('last_update_time', None)
if last_update_time:
last_update_time = datetime.strptime(last_update_time,
"%Y-%m-%dT%H:%M:%SZ")
updated_works = collection.works_updated_since(self._db,
last_update_time)
pagination = load_pagination_from_request()
works = pagination.apply(updated_works).all()
title = "%s Collection Updates for %s" % (collection.protocol,
client.url)
def update_url(time=last_update_time, page=None):
kw = dict(_external=True,
collection_metadata_identifier=collection_details)
if time:
kw.update({'last_update_time': last_update_time})
if page:
kw.update(page.items())
return cdn_url_for("updates", **kw)
entries = []
for work in works[:]:
entry = work.verbose_opds_entry or work.simple_opds_entry
entry = etree.fromstring(entry)
if entry:
entries.append(entry)
works.remove(work)
works = [(work.identifier, work) for work in works]
update_feed = LookupAcquisitionFeed(self._db,
title,
update_url(),
works,
VerboseAnnotator,
precomposed_entries=entries)
if len(updated_works.all()) > pagination.size + pagination.offset:
update_feed.add_link_to_feed(
update_feed.feed,
rel="next",
href=update_url(page=pagination.next_page))
if pagination.offset > 0:
update_feed.add_link_to_feed(
update_feed.feed,
rel="first",
href=update_url(page=pagination.first_page))
previous_page = pagination.previous_page
if previous_page:
update_feed.add_link_to_feed(update_feed.feed,
rel="previous",
href=update_url(page=previous_page))
return feed_response(update_feed)
def collection(self):
return Collection.by_id(self._db, id=self.collection_id)
from core.model import (
Collection,
ConfigurationSetting,
DataSource,
ExternalIntegration,
production_session,
)
log = logging.getLogger(name="Metadata Wrangler configuration import")
try:
_db = production_session()
# Get all of the OPDS_IMPORT collections.
collections = Collection.by_protocol(_db, ExternalIntegration.OPDS_IMPORT)
for collection in collections:
opds_url = collection.external_account_id
if not opds_url:
decoded_collection, ignore = Collection.from_metadata_identifier(
_db, collection.name
)
opds_url = decoded_collection.external_account_id
if not opds_url:
# This shouldn't happen.
log.warn(
'Could not find external_account_id for %r' % collection
)
continue
rb_collection_id = rb_digital_collections[0][0]
source = DataSource.lookup(_db, DataSource.RB_DIGITAL)
update_statement = update(Credential)\
.where(and_(
Credential.data_source_id == source.id,
Credential.type == Credential.IDENTIFIER_FROM_REMOTE_SERVICE
))\
.values(collection_id=rb_collection_id)
_db.execute(update_statement)
# We have multiple RBDigital integration and we don't know which credential
# belongs to each one. Have to check each credential against RBDigital API.
else:
rb_api = []
for collection_id in rb_digital_collections:
collection = Collection.by_id(_db, collection_id[0])
rb_api.append(RBDigitalAPI(_db, collection))
source = DataSource.lookup(_db, DataSource.RB_DIGITAL)
credentials = _db.query(Credential)\
.filter(
Credential.data_source_id == source.id,
Credential.type == Credential.IDENTIFIER_FROM_REMOTE_SERVICE,
Credential.credential != None
)
for credential in credentials:
for api in rb_api:
if check_patron_id(api, credential.credential):
credential.collection = api.collection
break
# Remove credentials stored with none as the credential
