def _integration(self, _db, id, name, protocol, goal):
"""Find or create the ExternalIntegration referred to."""
if not id and not name and not (protocol and goal):
raise ValueError(
"An integration must by identified by either ID, name, or the combination of protocol and goal."
)
integration = None
if id:
integration = get_one(
_db, ExternalIntegration, ExternalIntegration.id==id
)
if not integration:
raise ValueError("No integration with ID %s." % id)
if name:
integration = get_one(_db, ExternalIntegration, name=name)
if not integration and not (protocol and goal):
raise ValueError(
'No integration with name "%s". To create it, you must also provide protocol and goal.' % name
)
if not integration and (protocol and goal):
integration, is_new = get_one_or_create(
_db, ExternalIntegration, protocol=protocol, goal=goal
)
if name:
integration.name = name
return integration
def do_run(self, _db=None, cmd_args=None, output=sys.stdout):
_db = _db or self._db
args = self.parse_command_line(_db, cmd_args=cmd_args)
if args.name:
name = args.name
integration = get_one(_db, ExternalIntegration, name=name)
if not integration:
integration = get_one(_db, ExternalIntegration, id=name)
if integration:
integrations = [integration]
else:
output.write(
"Could not locate integration by name or ID: %s\n" % args
)
integrations = []
else:
integrations = _db.query(ExternalIntegration).order_by(
ExternalIntegration.name, ExternalIntegration.id).all()
if not integrations:
output.write("No integrations found.\n")
for integration in integrations:
output.write(
"\n".join(
integration.explain(include_secrets=args.show_secrets)
)
)
output.write("\n")
def to_collection(self, _db):
"""Find or create a Collection object for this Overdrive Advantage
account.
:return: a 2-tuple of Collections (primary Overdrive
collection, Overdrive Advantage collection)
"""
# First find the parent Collection.
parent = get_one(_db,
Collection,
external_account_id=self.parent_library_id,
protocol=Collection.OVERDRIVE)
if not parent:
# Without the parent's credentials we can't access the child.
raise ValueError(
"Cannot create a Collection whose parent does not already exist."
)
name = parent.name + " / " + self.name
child, ignore = get_one_or_create(_db,
Collection,
parent_id=parent.id,
protocol=Collection.OVERDRIVE,
external_account_id=self.library_id,
create_method_kwargs=dict(name=name))
# Set or update the name of the collection to reflect the name of
# the library, just in case that name has changed.
child.name = name
return parent, child
def ensure_coverage(self, item, force=False):
"""Ensure coverage for one specific item.
:param item: This should always be an Identifier, but this
code will also work if it's an Edition. (The Edition's
.primary_identifier will be covered.)
:param force: Run the coverage code even if an existing
coverage record for this item was created after
`self.cutoff_time`.
:return: Either a coverage record or a CoverageFailure.
TODO: This could be abstracted and moved to BaseCoverageProvider.
"""
if isinstance(item, Identifier):
identifier = item
else:
identifier = item.primary_identifier
coverage_record = get_one(
self._db, CoverageRecord,
identifier=identifier,
data_source=self.data_source,
operation=self.operation,
on_multiple='interchangeable',
)
if not force and not self.should_update(coverage_record):
return coverage_record
counts, records = self.process_batch_and_handle_results(
[identifier]
)
if records:
coverage_record = records[0]
else:
coverage_record = None
return coverage_record
def collection(self):
"""Retrieve the Collection object associated with this
Monitor.
"""
if not self.collection_id:
return None
return get_one(self._db, Collection, id=self.collection_id)
def ensure_coverage(self, item, force=False):
"""Ensure coverage for one specific item.
TODO: Could potentially be moved into BaseCoverageProvider.
:param force: Run the coverage code even if an existing
coverage record for this item was created after
`self.cutoff_time`.
:return: Either a coverage record or a CoverageFailure.
"""
if isinstance(item, Identifier):
identifier = item
else:
identifier = item.primary_identifier
coverage_record = get_one(
self._db,
CoverageRecord,
identifier=identifier,
data_source=self.output_source,
operation=self.operation,
on_multiple='interchangeable',
)
if not force and not self.should_update(coverage_record):
return coverage_record
counts, records = self.process_batch_and_handle_results([identifier])
if records:
coverage_record = records[0]
else:
coverage_record = None
return coverage_record
def collection(self):
"""Retrieve the Collection object associated with this
Monitor.
"""
if not self.collection_id:
return None
return get_one(self._db, Collection, id=self.collection_id)
def update_work_for_edition(self,
edition,
even_if_no_author=False,
immediately_presentation_ready=False):
work = None
# Find a pool for this edition. If we have CirculationData, a pool was created
# when we imported the edition. If there was already a pool from a different data
# source, that's fine too.
pool = get_one(self._db,
LicensePool,
identifier=edition.primary_identifier)
if pool:
# Note: pool.calculate_work will call self.set_presentation_edition(),
# which will find editions attached to same Identifier.
work, is_new_work = pool.calculate_work(
even_if_no_author=even_if_no_author)
# Note: if pool.calculate_work found or made a work, it already called work.calculate_presentation()
if work:
if immediately_presentation_ready:
# We want this book to be presentation-ready
# immediately upon import. As long as no crucial
# information is missing (like language or title),
# this will do it.
work.set_presentation_ready_based_on_content()
return pool, work
def confirm(self, resource_id, secret):
"""Confirm a secret for a URI, or don't.
:return: A Response containing a simple HTML document.
"""
if not secret:
return self.html_response(404, _("No confirmation code provided"))
if not resource_id:
return self.html_response(404, _("No resource ID provided"))
validation = get_one(self._db, Validation, secret=secret)
resource = get_one(self._db, Resource, id=resource_id)
if not resource:
return self.html_response(404, _("No such resource"))
if not validation:
# The secret is invalid. This might be because the secret
# is wrong, or because the Resource has already been
# validated.
#
# Let's eliminate the 'Resource has already been validated'
# possibility and take care of the other case next.
if resource and resource.validation and resource.validation.success:
return self.html_response(
200, _("This URI has already been validated."))
if (not validation or not validation.resource
or validation.resource.id != resource_id):
# For whatever reason the resource ID and secret don't match.
# A generic error that doesn't reveal information is appropriate
# in all cases.
error = _("Confirmation code %r not found") % secret
return self.html_response(404, error)
# At this point we know that the resource has not been
# confirmed, and that the secret matches the resource. The
# only other problem might be that the validation has expired.
if not validation.active:
error = _(
"Confirmation code %r has expired. Re-register to get another code."
) % secret
return self.html_response(400, error)
validation.mark_as_successful()
resource = validation.resource
message = _("You successfully confirmed %s.") % resource.href
return self.html_response(200, message)
def _validated_at(self, hyperlink):
validated_at = "Not validated"
if hyperlink and hyperlink.resource:
validation = get_one(self._db,
Validation,
resource=hyperlink.resource)
if validation:
return validation.started_at
return validated_at
def libraries(self, library_name=None):
"""Find all libraries on which this script should operate.
:param library_name: The library name passed in on the command line,
if any.
"""
if library_name:
library = get_one(self._db, Library, name=library_name)
if not library:
raise ValueError("No library with name %r" % library_name)
return [library]
return self.all_libraries
def do_run(self):
existing_timestamp = get_one(self._db, Timestamp, service=self.name)
if existing_timestamp:
raise Exception(
"Timestamp for Database Migration script already exists")
migrations = self.fetch_migration_files()[0]
most_recent_migration = self.sort_migrations(migrations)[-1]
initial_timestamp = Timestamp.stamp(self._db, self.name)
self.update_timestamp(initial_timestamp, most_recent_migration)
def test_success(self):
script = ConfigureCollectionScript()
l1, ignore = create(
self._db, Library, name="Library 1", short_name="L1",
)
l2, ignore = create(
self._db, Library, name="Library 2", short_name="L2",
)
l3, ignore = create(
self._db, Library, name="Library 3", short_name="L3",
)
self._db.commit()
# Create a collection, set all its attributes, set a custom
# setting, and associate it with two libraries.
output = StringIO()
script.do_run(
self._db, ["--name=New Collection", "--protocol=Overdrive",
"--library=L2", "--library=L1",
"--setting=library_id=1234",
"--external-account-id=acctid",
"--url=url",
"--username=username",
"--password=password",
], output
)
# The collection was created and configured properly.
collection = get_one(self._db, Collection)
eq_("New Collection", collection.name)
eq_("url", collection.external_integration.url)
eq_("acctid", collection.external_account_id)
eq_("username", collection.external_integration.username)
eq_("password", collection.external_integration.password)
# Two libraries now have access to the collection.
eq_([collection], l1.collections)
eq_([collection], l2.collections)
eq_([], l3.collections)
# One CollectionSetting was set on the collection.
[setting] = collection.external_integration.settings
eq_("library_id", setting.key)
eq_("1234", setting.value)
# The output explains the collection settings.
expect = ("Configuration settings stored.\n"
+ "\n".join(collection.explain()) + "\n")
eq_(expect, output.getvalue())
def test_add_settings(self):
script = ConfigureIntegrationScript()
output = StringIO()
script.do_run(self._db, [
"--protocol=aprotocol",
"--goal=agoal",
"--setting=akey=avalue",
], output)
# An ExternalIntegration was created and configured.
integration = get_one(self._db,
ExternalIntegration,
protocol="aprotocol",
goal="agoal")
expect_output = "Configuration settings stored.\n" + "\n".join(
integration.explain()) + "\n"
eq_(expect_output, output.getvalue())
def nlu_interface(nlu_inputs, sess, model):
# nlu_output = {'nlu_result':{'intent':'', 'slots':[]}}
# CONFIRM_LIST, REJECT_LIST = get_y_n()
nlu_inputs = nlu_inputs.strip().replace(' ', '')
assert type(nlu_inputs) == str
inputs = cut_sentence(nlu_inputs)
id_inputs = data_utils.nlu_input_to_token_ids(inputs, input_vocab_path,
data_utils.tab_tokenizer)
_inputs, _labels, _sequence_length = model.get_one([[id_inputs, [0]]], 0)
# pdb.set_trace()
_, step_loss, logits, summary = model.step(sess, _inputs, _labels,
_sequence_length, True)
label_logit = logits['label'][0]
predict_label_ids = np.argmax(label_logit, 1)[:_sequence_length[0]]
predict_label = [rev_label_vocab[x] for x in predict_label_ids]
nlu_output = '\t'.join(predict_label)
return nlu_output
def do_run(self):
args = self.parse_command_line()
last_run_date = args.last_run_date
last_run_counter = args.last_run_counter
existing_timestamp = get_one(self._db, Timestamp, service=self.name)
if last_run_date:
last_run_datetime = self.parse_time(last_run_date)
if existing_timestamp:
existing_timestamp.timestamp = last_run_datetime
if last_run_counter:
existing_timestamp.counter = last_run_counter
else:
existing_timestamp, ignore = get_one_or_create(
self._db, Timestamp,
service=self.name,
timestamp=last_run_datetime
)
if existing_timestamp:
migrations, migrations_by_dir = self.fetch_migration_files()
new_migrations = self.get_new_migrations(
existing_timestamp, migrations
)
if new_migrations:
# Log the new migrations.
print "%d new migrations found." % len(new_migrations)
for migration in new_migrations:
print " - %s" % migration
self.run_migrations(
new_migrations, migrations_by_dir, existing_timestamp
)
else:
print "No new migrations found. Your database is up-to-date."
else:
print ""
print (
"NO TIMESTAMP FOUND. Run script with timestamp that indicates"
" the last migration run against this database."
)
self.arg_parser().print_help()
def initialize(cls, _db):
"""Find or create a local analytics service.
"""
# If a local analytics service already exists, return it.
local_analytics = get_one(_db,
ExternalIntegration,
protocol=cls.__module__,
goal=ExternalIntegration.ANALYTICS_GOAL)
# If a local analytics service already exists, don't create a
# default one. Otherwise, create it with default name of
# "Local Analytics".
if not local_analytics:
local_analytics, ignore = create(
_db,
ExternalIntegration,
protocol=cls.__module__,
goal=ExternalIntegration.ANALYTICS_GOAL,
name=str(cls.NAME))
return local_analytics
def run(self, cmd_args=None):
parsed = self.parse_command_line(self._db, cmd_args)
name = parsed.name
authentication_url = parsed.authentication_url
opds = parsed.opds
web = parsed.web
description = parsed.description
aliases = parsed.alias
places = parsed.place
short_name = parsed.short_name
shared_secret = parsed.shared_secret
library, is_new = get_one_or_create(
self._db, Library, authentication_url=authentication_url
)
if name:
library.name = name
if opds:
library.opds_url = opds
if web:
library.web_url = web
if description:
library.description = description
if short_name:
library.short_name = short_name
if shared_secret:
library.shared_secret = shared_secret
if aliases:
for alias in aliases:
get_one_or_create(self._db, LibraryAlias, library=library,
name=alias, language='eng')
if places:
for place_external_id in places:
place = get_one(self._db, Place, external_id=place_external_id)
get_one_or_create(
self._db, ServiceArea, library=library, place=place
)
self._db.commit()
def register(self, do_get=HTTP.debuggable_get):
if flask.request.method == 'GET':
document = self.registration_document
return self.catalog_response(document)
auth_url = flask.request.form.get("url")
self.log.info("Got request to register %s", auth_url)
if not auth_url:
return NO_AUTH_URL
integration_contact_uri = flask.request.form.get("contact")
integration_contact_email = integration_contact_uri
shared_secret = None
auth_header = flask.request.headers.get('Authorization')
if auth_header and isinstance(
auth_header, basestring) and "bearer" in auth_header.lower():
shared_secret = auth_header.split(' ', 1)[1]
self.log.info("Incoming shared secret: %s...", shared_secret[:4])
# If 'stage' is not provided, it means the client doesn't make the
# testing/production distinction. We have to assume they want
# production -- otherwise they wouldn't bother registering.
library_stage = flask.request.form.get("stage")
self.log.info("Incoming stage: %s", library_stage)
library_stage = library_stage or Library.PRODUCTION_STAGE
# NOTE: This is commented out until we can say that
# registration requires providing a contact email and expect
# every new library to be on a circulation manager that can meet
# this requirement.
#
#integration_contact_email = self._required_email_address(
# integration_contact_uri,
# "Invalid or missing configuration contact email address"
#)
if isinstance(integration_contact_email, ProblemDetail):
return integration_contact_email
# Registration is a complex multi-step process. Start a subtransaction
# so we can back out of the whole thing if any part of it fails.
__transaction = self._db.begin_nested()
library = None
elevated_permissions = False
if shared_secret:
# Look up a library by the provided shared secret. This
# will let us handle the case where the library has
# changed URLs (auth_url does not match
# library.authentication_url) but the shared secret is the
# same.
library = get_one(self._db, Library, shared_secret=shared_secret)
if not library:
__transaction.rollback()
return AUTHENTICATION_FAILURE.detailed(
_("Provided shared secret is invalid"))
# This gives the requestor an elevated level of permissions.
elevated_permissions = True
library_is_new = False
if library.authentication_url != auth_url:
# The library's authentication URL has changed,
# e.g. moved from HTTP to HTTPS. The registration
# includes a valid shared secret, so it's okay to
# modify the corresponding database field.
#
# We want to do this before the registration, so that
# we request the new URL instead of the old one.
library.authentication_url = auth_url
if not library:
# Either this is a library at a known authentication URL
# or it's a brand new library.
library, library_is_new = get_one_or_create(
self._db, Library, authentication_url=auth_url)
registrar = LibraryRegistrar(self._db, do_get=do_get)
result = registrar.register(library, library_stage)
if isinstance(result, ProblemDetail):
__transaction.rollback()
return result
# At this point registration (or re-registration) has
# succeeded, so we won't be rolling back the subtransaction
# that created the Library.
__transaction.commit()
auth_document, hyperlinks_to_create = result
# Now that we've completed the registration process, we
# know the opds_url -- it's the 'start' link found in
# the auth_document.
#
# Registration will fail if this link is missing or the
# URL doesn't work, so we can assume this is valid.
opds_url = auth_document.root['href']
if library_is_new:
# The library was just created, so it had no opds_url.
# Set it now.
library.opds_url = opds_url
# The registration process may have queued up a number of
# Hyperlinks that needed to be created (taken from the
# library's authentication document), but we also need to
# create a hyperlink for the integration contact provided with
# the registration request itself.
if integration_contact_email:
hyperlinks_to_create.append((Hyperlink.INTEGRATION_CONTACT_REL,
[integration_contact_email]))
reset_shared_secret = False
if elevated_permissions:
# If you have elevated permissions you may ask for the
# shared secret to be reset.
reset_shared_secret = flask.request.form.get(
"reset_shared_secret", False)
if library.opds_url != opds_url:
# The library's OPDS URL has changed, e.g. moved from
# HTTP to HTTPS. Since we have elevated permissions,
# it's okay to modify the corresponding database
# field.
library.opds_url = opds_url
for rel, candidates in hyperlinks_to_create:
hyperlink, is_modified = library.set_hyperlink(rel, *candidates)
if is_modified:
# We need to send an email to this email address about
# what just happened. This is either so the receipient
# can confirm that the address works, or to inform
# them a new library is using their address.
try:
hyperlink.notify(self.emailer, self.app.url_for)
except SMTPException, e:
# We were unable to send the email.
return INTEGRATION_ERROR.detailed(
_("SMTP error while sending email to %(address)s",
address=hyperlink.resource.href))
def get_collection(major, minor):
item = get_one(Conf.db,
DisplayItem,
beacon_major_id=int(major),
beacon_minor_id=int(minor))
return flask.jsonify(**item.json)
