def test_settings(self):
script = ConfigureSiteScript()
output = StringIO()
script.do_run(self._db, [
"--setting=setting1=value1",
"--setting=setting2=[1,2,\"3\"]",
"--setting=secret_setting=secretvalue",
], output)
# The secret was set, but is not shown.
eq_(
"""Current site-wide settings:
setting1='value1'
setting2='[1,2,"3"]'
""", output.getvalue())
eq_("value1",
ConfigurationSetting.sitewide(self._db, "setting1").value)
eq_('[1,2,"3"]',
ConfigurationSetting.sitewide(self._db, "setting2").value)
eq_("secretvalue",
ConfigurationSetting.sitewide(self._db, "secret_setting").value)
# If we run again with --show-secrets, the secret is shown.
output = StringIO()
script.do_run(self._db, ["--show-secrets"], output)
eq_(
"""Current site-wide settings:
secret_setting='secretvalue'
setting1='value1'
setting2='[1,2,"3"]'
""", output.getvalue())
def teardown(self):
# Close the session.
self._db.close()
# Roll back all database changes that happened during this
# test, whether in the session that was just closed or some
# other session.
self.transaction.rollback()
# Remove any database objects cached in the model classes but
# associated with the now-rolled-back session.
Collection.reset_cache()
ConfigurationSetting.reset_cache()
DataSource.reset_cache()
DeliveryMechanism.reset_cache()
ExternalIntegration.reset_cache()
Genre.reset_cache()
Library.reset_cache()
# Also roll back any record of those changes in the
# Configuration instance.
for key in [
Configuration.SITE_CONFIGURATION_LAST_UPDATE,
Configuration.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE
]:
if key in Configuration.instance:
del (Configuration.instance[key])
if self.search_mock:
self.search_mock.stop()
def teardown(self):
# Close the session.
self._db.close()
# Roll back all database changes that happened during this
# test, whether in the session that was just closed or some
# other session.
self.transaction.rollback()
# Remove any database objects cached in the model classes but
# associated with the now-rolled-back session.
Collection.reset_cache()
ConfigurationSetting.reset_cache()
DataSource.reset_cache()
DeliveryMechanism.reset_cache()
ExternalIntegration.reset_cache()
Genre.reset_cache()
Library.reset_cache()
# Also roll back any record of those changes in the
# Configuration instance.
for key in [
Configuration.SITE_CONFIGURATION_LAST_UPDATE,
Configuration.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE
]:
if key in Configuration.instance:
del(Configuration.instance[key])
if self.search_mock:
self.search_mock.stop()
def from_configuration(cls, _db, testing=False):
from model import (ExternalIntegration, ConfigurationSetting)
(internal_log_level, internal_log_format, database_log_level,
message_template) = cls._defaults(testing)
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
internal = ExternalIntegration.lookup(
_db, ExternalIntegration.INTERNAL_LOGGING, goal)
if internal:
internal_log_format = (internal.setting(cls.LOG_FORMAT).value
or internal_log_format)
message_template = (internal.setting(
cls.LOG_MESSAGE_TEMPLATE).value or message_template)
internal_log_level = (ConfigurationSetting.sitewide(
_db, Configuration.LOG_LEVEL).value or internal_log_level)
database_log_level = (ConfigurationSetting.sitewide(
_db, Configuration.DATABASE_LOG_LEVEL).value
or database_log_level)
app_name = ConfigurationSetting.sitewide(
_db, Configuration.LOG_APP_NAME).value or app_name
handler = logging.StreamHandler()
cls.set_formatter(handler, internal_log_format, message_template,
app_name)
return (handler, internal_log_level, database_log_level)
def test_from_configuration(self):
cls = LogConfiguration
config = Configuration
m = cls.from_configuration
# When logging is configured on initial startup, with no
# database connection, these are the defaults.
internal_log_level, database_log_level, [handler] = m(None,
testing=False)
eq_(cls.INFO, internal_log_level)
eq_(cls.WARN, database_log_level)
assert isinstance(handler.formatter, JSONFormatter)
# The same defaults hold when there is a database connection
# but nothing is actually configured.
internal_log_level, database_log_level, [handler] = m(self._db,
testing=False)
eq_(cls.INFO, internal_log_level)
eq_(cls.WARN, database_log_level)
assert isinstance(handler.formatter, JSONFormatter)
# Let's set up a Loggly integration and change the defaults.
loggly = self.loggly_integration()
internal = self._external_integration(
protocol=ExternalIntegration.INTERNAL_LOGGING,
goal=ExternalIntegration.LOGGING_GOAL)
ConfigurationSetting.sitewide(self._db,
config.LOG_LEVEL).value = config.ERROR
internal.setting(
SysLogger.LOG_FORMAT).value = SysLogger.TEXT_LOG_FORMAT
ConfigurationSetting.sitewide(
self._db, config.DATABASE_LOG_LEVEL).value = config.DEBUG
ConfigurationSetting.sitewide(self._db,
config.LOG_APP_NAME).value = "test app"
template = "%(filename)s:%(message)s"
internal.setting(SysLogger.LOG_MESSAGE_TEMPLATE).value = template
internal_log_level, database_log_level, handlers = m(self._db,
testing=False)
eq_(cls.ERROR, internal_log_level)
eq_(cls.DEBUG, database_log_level)
[loggly_handler
] = [x for x in handlers if isinstance(x, LogglyHandler)]
eq_("http://example.com/a_token/", loggly_handler.url)
eq_("test app", loggly_handler.formatter.app_name)
[stream_handler
] = [x for x in handlers if isinstance(x, logging.StreamHandler)]
assert isinstance(stream_handler.formatter, UTF8Formatter)
eq_(template, stream_handler.formatter._fmt)
# If testing=True, then the database configuration is ignored,
# and the log setup is one that's appropriate for display
# alongside unit test output.
internal_log_level, database_log_level, [handler] = m(self._db,
testing=True)
eq_(cls.INFO, internal_log_level)
eq_(cls.WARN, database_log_level)
eq_(SysLogger.DEFAULT_MESSAGE_TEMPLATE, handler.formatter._fmt)
def site_configuration_last_update(cls,
_db,
known_value=None,
timeout=None):
"""Check when the site configuration was last updated.
Updates Configuration.instance[Configuration.SITE_CONFIGURATION_LAST_UPDATE].
It's the application's responsibility to periodically check
this value and reload the configuration if appropriate.
:param known_value: We know when the site configuration was
last updated--it's this timestamp. Use it instead of checking
with the database.
:param timeout: We will only call out to the database once in
this number of seconds. If we are asked again before this
number of seconds elapses, we will assume site configuration
has not changed.
:return: a datetime object.
"""
now = datetime.datetime.utcnow()
if _db and timeout is None:
from model import ConfigurationSetting
timeout = ConfigurationSetting.sitewide(
_db, cls.SITE_CONFIGURATION_TIMEOUT).value
if timeout is None:
timeout = 60
last_check = cls.instance.get(
cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE)
if (not known_value and last_check
and (now - last_check).total_seconds() < timeout):
# We went to the database less than [timeout] seconds ago.
# Assume there has been no change.
return cls._site_configuration_last_update()
# Ask the database when was the last time the site
# configuration changed. Specifically, this is the last time
# site_configuration_was_changed() (defined in model.py) was
# called.
if not known_value:
from model import Timestamp
known_value = Timestamp.value(_db, cls.SITE_CONFIGURATION_CHANGED,
None)
if not known_value:
# The site configuration has never changed.
last_update = None
else:
last_update = known_value
# Update the Configuration object's record of the last update time.
cls.instance[cls.SITE_CONFIGURATION_LAST_UPDATE] = last_update
# Whether that record changed or not, the time at which we
# _checked_ is going to be set to the current time.
cls.instance[cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE] = now
return last_update
def __init__(self, _db, title, url, libraries, annotator=None,
live=True, url_for=None):
"""Turn a list of libraries into a catalog."""
if not annotator:
annotator = Annotator()
# To save bandwidth, omit logos from large feeds. What 'large'
# means is customizable.
include_logos = not (self._feed_is_large(_db, libraries))
self.catalog = dict(metadata=dict(title=title), catalogs=[])
self.add_link_to_catalog(self.catalog, rel="self",
href=url, type=self.OPDS_TYPE)
web_client_uri_template = ConfigurationSetting.sitewide(
_db, Configuration.WEB_CLIENT_URL
).value
for library in libraries:
if not isinstance(library, tuple):
library = (library,)
self.catalog["catalogs"].append(
self.library_catalog(
*library, url_for=url_for,
include_logo=include_logos,
web_client_uri_template=web_client_uri_template
)
)
annotator.annotate_catalog(self, live=live)
def ils_name_setting(cls, _db, collection, library):
"""Find the ConfigurationSetting controlling the ILS name
for the given collection and library.
"""
return ConfigurationSetting.for_library_and_externalintegration(
_db, cls.ILS_NAME_KEY, library, collection.external_integration
)
def from_configuration(cls, _db, testing=False):
"""Return the logging policy as configured in the database.
:param _db: A database connection. If None, the default
logging policy will be used.
:param testing: A boolean indicating whether a unit test is
happening right now. If True, the database configuration will
be ignored in favor of a known test-friendly policy. (It's
okay to pass in False during a test *of this method*.)
:return: A 3-tuple (internal_log_level, database_log_level,
handlers). `internal_log_level` is the log level to be used
for most log messages. `database_log_level` is the log level
to be applied to the loggers for the database connector and
other verbose third-party libraries. `handlers` is a list of
Handler objects that will be associated with the top-level
logger.
"""
log_level = cls.DEFAULT_LOG_LEVEL
database_log_level = cls.DEFAULT_DATABASE_LOG_LEVEL
if _db and not testing:
log_level = (
ConfigurationSetting.sitewide(_db, Configuration.LOG_LEVEL).value
or log_level
)
database_log_level = (
ConfigurationSetting.sitewide(_db, Configuration.DATABASE_LOG_LEVEL).value
or database_log_level
)
loggers = [SysLogger, Loggly, CloudwatchLogs]
handlers = []
errors = []
for logger in loggers:
try:
handler = logger.from_configuration(_db, testing)
if handler:
handlers.append(handler)
except Exception, e:
errors.append(
"Error creating logger %s %s" % (logger.NAME, unicode(e))
)
def test_library_catalogs(self):
l1 = self._library("The New York Public Library")
l2 = self._library("Brooklyn Public Library")
class TestAnnotator(object):
def annotate_catalog(self, catalog_obj, live=True):
catalog_obj.catalog['metadata'][
'random'] = "Random text inserted by annotator."
# This template will be used to construct a web client link
# for each library.
template = "http://web/{uuid}"
ConfigurationSetting.sitewide(
self._db, Configuration.WEB_CLIENT_URL).value = template
catalog = OPDSCatalog(self._db,
"A Catalog!",
"http://url/", [l1, l2],
TestAnnotator(),
url_for=self.mock_url_for)
catalog = unicode(catalog)
parsed = json.loads(catalog)
# The catalog is labeled appropriately.
eq_("A Catalog!", parsed['metadata']['title'])
[self_link] = parsed['links']
eq_("http://url/", self_link['href'])
eq_("self", self_link['rel'])
# The annotator modified the catalog in passing.
eq_("Random text inserted by annotator.", parsed['metadata']['random'])
# Each library became a catalog in the catalogs collection.
eq_([l1.name, l2.name],
[x['metadata']['title'] for x in parsed['catalogs']])
# Each library has a link to its web catalog.
l1_links, l2_links = [
library['links'] for library in parsed['catalogs']
]
[l1_web] = [
link['href'] for link in l1_links if link['type'] == 'text/html'
]
eq_(l1_web, template.replace("{uuid}", l1.internal_urn))
[l2_web] = [
link['href'] for link in l2_links if link['type'] == 'text/html'
]
eq_(l2_web, template.replace("{uuid}", l2.internal_urn))
def registration_document(self):
"""Serve a document that describes the registration process,
notably the terms of service for that process.
The terms of service are hosted elsewhere; we only know the
URL of the page they're stored.
"""
document = dict()
# The terms of service may be encapsulated in a link to
# a web page.
terms_of_service_url = ConfigurationSetting.sitewide(
self._db, Configuration.REGISTRATION_TERMS_OF_SERVICE_URL).value
type = "text/html"
rel = "terms-of-service"
if terms_of_service_url:
OPDSCatalog.add_link_to_catalog(
document,
rel=rel,
type=type,
href=terms_of_service_url,
)
# And/or the terms of service may be described in
# human-readable HTML, which we'll present as a data: link.
terms_of_service_html = ConfigurationSetting.sitewide(
self._db, Configuration.REGISTRATION_TERMS_OF_SERVICE_HTML).value
if terms_of_service_html:
encoded = base64.b64encode(terms_of_service_html)
terms_of_service_link = "data:%s;base64,%s" % (type, encoded)
OPDSCatalog.add_link_to_catalog(document,
rel=rel,
type=type,
href=terms_of_service_link)
return document
def do_run(self, _db=None, cmd_args=None, output=sys.stdout):
_db = _db or self._db
args = self.parse_command_line(_db, cmd_args=cmd_args)
if args.setting:
for setting in args.setting:
key, value = self._parse_setting(setting)
ConfigurationSetting.sitewide(_db, key).value = value
settings = _db.query(ConfigurationSetting).filter(
ConfigurationSetting.library_id==None).filter(
ConfigurationSetting.external_integration==None
).order_by(ConfigurationSetting.key)
output.write("Current site-wide settings:\n")
for setting in settings:
if args.show_secrets or not setting.is_secret:
output.write("%s='%s'\n" % (setting.key, setting.value))
_db.commit()
def from_configuration(cls, _db, testing=False):
settings = None
cloudwatch = None
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
settings = ExternalIntegration.lookup(
_db, ExternalIntegration.CLOUDWATCH, goal)
app_name = ConfigurationSetting.sitewide(
_db, Configuration.LOG_APP_NAME).value or app_name
if settings:
cloudwatch = cls.get_handler(settings, testing)
cls.set_formatter(cloudwatch, app_name)
return cloudwatch
def from_configuration(cls, _db, testing=False):
loggly = None
from model import (ExternalIntegration, ConfigurationSetting)
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
loggly = ExternalIntegration.lookup(
_db, ExternalIntegration.LOGGLY, goal
)
app_name = ConfigurationSetting.sitewide(_db, Configuration.LOG_APP_NAME).value or app_name
if loggly:
loggly = Loggly.loggly_handler(loggly)
cls.set_formatter(loggly, app_name)
return loggly
def test_feed_is_large(self):
# Verify that the _feed_is_large helper method
# works whether it's given a Python list or a SQLAlchemy query.
setting = ConfigurationSetting.sitewide(self._db,
Configuration.LARGE_FEED_SIZE)
setting.value = 2
m = OPDSCatalog._feed_is_large
query = self._db.query(Library)
# There are no libraries, and the limit is 2, so a feed of libraries would not be large.
eq_(0, query.count())
eq_(False, m(self._db, query))
# Make some libraries, and the feed becomes large.
[self._library() for x in range(2)]
eq_(True, m(self._db, query))
# It also works with a list.
eq_(True, m(self._db, [1, 2]))
eq_(False, m(self._db, [1]))
def _feed_is_large(cls, _db, libraries):
"""Determine whether a prospective feed is 'large' per a sitewide setting.
:param _db: A database session
:param libraries: A list of libraries (or anything else that might be
going into a feed).
"""
large_feed_size = ConfigurationSetting.sitewide(
_db, Configuration.LARGE_FEED_SIZE
).int_value
if large_feed_size is None:
# No limit
return False
if isinstance(libraries, Query):
# This is a SQLAlchemy query.
size = libraries.count()
else:
# This is something like a normal Python list.
size = len(libraries)
return size >= large_feed_size
def test_success(self):
us = self._place(type=Place.NATION, abbreviated_name='US')
library = self._library()
s = SetCoverageAreaScript(_db=self._db)
# Setting a service area with no focus area assigns that
# service area to the library.
args = [
"--library=%s" % library.name,
'--service-area={"US": "everywhere"}'
]
s.run(args)
[area] = library.service_areas
eq_(us, area.place)
# Setting a focus area and not a service area treats 'everywhere'
# as the service area.
uk = self._place(type=Place.NATION, abbreviated_name='UK')
args = [
"--library=%s" % library.name, '--focus-area={"UK": "everywhere"}'
]
s.run(args)
places = [x.place for x in library.service_areas]
eq_(2, len(places))
assert uk in places
assert Place.everywhere(self._db) in places
# The library's former ServiceAreas have been removed.
assert us not in places
# If a default nation is set, you can name a single place as
# your service area.
ConfigurationSetting.sitewide(
self._db, Configuration.DEFAULT_NATION_ABBREVIATION).value = "US"
ut = self._place(type=Place.STATE, abbreviated_name='UT', parent=us)
args = ["--library=%s" % library.name, '--service-area=UT']
s.run(args)
[area] = library.service_areas
eq_(ut, area.place)
def test_large_feeds_treated_differently(self):
# The libraries in large feeds are converted to JSON in ways
# that omit large chunks of data such as inline logos.
# In this test, a feed with 2 or more items is considered
# 'large'. Any smaller feed is considered 'small'.
setting = ConfigurationSetting.sitewide(self._db,
Configuration.LARGE_FEED_SIZE)
setting.value = 2
class Mock(OPDSCatalog):
def library_catalog(*args, **kwargs):
# Every time library_catalog is called, record whether
# we were asked to include a logo.
return kwargs['include_logo']
# Every item in the large feed resulted in a call with
# include_logo=False.
large_feed = Mock(self._db, "title", "url", ["it's", "large"])
large_catalog = large_feed.catalog['catalogs']
eq_([False, False], large_catalog)
# Every item in the large feed resulted in a call with
# include_logo=True.
small_feed = Mock(self._db, "title", "url", ["small"])
small_catalog = small_feed.catalog['catalogs']
eq_([True], small_catalog)
# Make it so even a feed with one item is 'large'.
setting.value = 1
small_feed = Mock(self._db, "title", "url", ["small"])
small_catalog = small_feed.catalog['catalogs']
eq_([False], small_catalog)
# Try it with a query that returns no results. No catalogs
# are included at all.
small_feed = Mock(self._db, "title", "url", self._db.query(Library))
small_catalog = small_feed.catalog['catalogs']
eq_([], small_catalog)
def from_configuration(cls, _db, testing=False):
(internal_log_format, message_template) = cls._defaults(testing)
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
internal = ExternalIntegration.lookup(
_db, ExternalIntegration.INTERNAL_LOGGING, goal)
if internal:
internal_log_format = (internal.setting(cls.LOG_FORMAT).value
or internal_log_format)
message_template = (internal.setting(
cls.LOG_MESSAGE_TEMPLATE).value or message_template)
app_name = ConfigurationSetting.sitewide(
_db, Configuration.LOG_APP_NAME).value or app_name
handler = logging.StreamHandler()
cls.set_formatter(handler,
log_format=internal_log_format,
message_template=message_template,
app_name=app_name)
return handler
directory, "registry-admin.js")
@app.route('/admin/static/registry-admin.css')
@returns_problem_detail
def admin_css():
directory = os.path.join(os.path.abspath(os.path.dirname(__file__)),
"node_modules", "simplified-registry-admin",
"dist")
return app.library_registry.static_files.static_file(
directory, "registry-admin.css")
if __name__ == '__main__':
debug = True
if len(sys.argv) > 1:
url = sys.argv[1]
else:
url = ConfigurationSetting.sitewide(_db, Configuration.BASE_URL).value
url = url or u'http://localhost:7000/'
scheme, netloc, path, parameters, query, fragment = urlparse.urlparse(url)
if ':' in netloc:
host, port = netloc.split(':')
port = int(port)
else:
host = netloc
port = 80
app.library_registry.log.info("Starting app on %s:%s", host, port)
app.run(debug=debug, host=host, port=port)
def site_configuration_last_update(cls, _db, known_value=None,
timeout=None):
"""Check when the site configuration was last updated.
Updates Configuration.instance[Configuration.SITE_CONFIGURATION_LAST_UPDATE].
It's the application's responsibility to periodically check
this value and reload the configuration if appropriate.
:param known_value: We know when the site configuration was
last updated--it's this timestamp. Use it instead of checking
with the database.
:param timeout: We will only call out to the database once in
this number of seconds. If we are asked again before this
number of seconds elapses, we will assume site configuration
has not changed.
:return: a datetime object.
"""
now = datetime.datetime.utcnow()
if _db and timeout is None:
from model import ConfigurationSetting
timeout = ConfigurationSetting.sitewide(
_db, cls.SITE_CONFIGURATION_TIMEOUT
).value
if timeout is None:
timeout = 60
last_check = cls.instance.get(
cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE
)
if (not known_value
and last_check and (now - last_check).total_seconds() < timeout):
# We went to the database less than [timeout] seconds ago.
# Assume there has been no change.
return cls._site_configuration_last_update()
# Ask the database when was the last time the site
# configuration changed. Specifically, this is the last time
# site_configuration_was_changed() (defined in model.py) was
# called.
if not known_value:
from model import Timestamp
known_value = Timestamp.value(
_db, cls.SITE_CONFIGURATION_CHANGED, service_type=None,
collection=None
)
if not known_value:
# The site configuration has never changed.
last_update = None
else:
last_update = known_value
# Update the Configuration object's record of the last update time.
cls.instance[cls.SITE_CONFIGURATION_LAST_UPDATE] = last_update
# Whether that record changed or not, the time at which we
# _checked_ is going to be set to the current time.
cls.instance[cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE] = now
return last_update
def set_secret_key(_db=None):
_db = _db or app._db
app.secret_key = ConfigurationSetting.sitewide_secret(
_db, Configuration.SECRET_KEY)