def test_settings(self): script = ConfigureSiteScript() output = StringIO() script.do_run(self._db, [ "--setting=setting1=value1", "--setting=setting2=[1,2,\"3\"]", "--setting=secret_setting=secretvalue", ], output) # The secret was set, but is not shown. eq_( """Current site-wide settings: setting1='value1' setting2='[1,2,"3"]' """, output.getvalue()) eq_("value1", ConfigurationSetting.sitewide(self._db, "setting1").value) eq_('[1,2,"3"]', ConfigurationSetting.sitewide(self._db, "setting2").value) eq_("secretvalue", ConfigurationSetting.sitewide(self._db, "secret_setting").value) # If we run again with --show-secrets, the secret is shown. output = StringIO() script.do_run(self._db, ["--show-secrets"], output) eq_( """Current site-wide settings: secret_setting='secretvalue' setting1='value1' setting2='[1,2,"3"]' """, output.getvalue())
def teardown(self): # Close the session. self._db.close() # Roll back all database changes that happened during this # test, whether in the session that was just closed or some # other session. self.transaction.rollback() # Remove any database objects cached in the model classes but # associated with the now-rolled-back session. Collection.reset_cache() ConfigurationSetting.reset_cache() DataSource.reset_cache() DeliveryMechanism.reset_cache() ExternalIntegration.reset_cache() Genre.reset_cache() Library.reset_cache() # Also roll back any record of those changes in the # Configuration instance. for key in [ Configuration.SITE_CONFIGURATION_LAST_UPDATE, Configuration.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE ]: if key in Configuration.instance: del (Configuration.instance[key]) if self.search_mock: self.search_mock.stop()
def teardown(self): # Close the session. self._db.close() # Roll back all database changes that happened during this # test, whether in the session that was just closed or some # other session. self.transaction.rollback() # Remove any database objects cached in the model classes but # associated with the now-rolled-back session. Collection.reset_cache() ConfigurationSetting.reset_cache() DataSource.reset_cache() DeliveryMechanism.reset_cache() ExternalIntegration.reset_cache() Genre.reset_cache() Library.reset_cache() # Also roll back any record of those changes in the # Configuration instance. for key in [ Configuration.SITE_CONFIGURATION_LAST_UPDATE, Configuration.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE ]: if key in Configuration.instance: del(Configuration.instance[key]) if self.search_mock: self.search_mock.stop()
def from_configuration(cls, _db, testing=False): from model import (ExternalIntegration, ConfigurationSetting) (internal_log_level, internal_log_format, database_log_level, message_template) = cls._defaults(testing) app_name = cls.DEFAULT_APP_NAME if _db and not testing: goal = ExternalIntegration.LOGGING_GOAL internal = ExternalIntegration.lookup( _db, ExternalIntegration.INTERNAL_LOGGING, goal) if internal: internal_log_format = (internal.setting(cls.LOG_FORMAT).value or internal_log_format) message_template = (internal.setting( cls.LOG_MESSAGE_TEMPLATE).value or message_template) internal_log_level = (ConfigurationSetting.sitewide( _db, Configuration.LOG_LEVEL).value or internal_log_level) database_log_level = (ConfigurationSetting.sitewide( _db, Configuration.DATABASE_LOG_LEVEL).value or database_log_level) app_name = ConfigurationSetting.sitewide( _db, Configuration.LOG_APP_NAME).value or app_name handler = logging.StreamHandler() cls.set_formatter(handler, internal_log_format, message_template, app_name) return (handler, internal_log_level, database_log_level)
def test_from_configuration(self): cls = LogConfiguration config = Configuration m = cls.from_configuration # When logging is configured on initial startup, with no # database connection, these are the defaults. internal_log_level, database_log_level, [handler] = m(None, testing=False) eq_(cls.INFO, internal_log_level) eq_(cls.WARN, database_log_level) assert isinstance(handler.formatter, JSONFormatter) # The same defaults hold when there is a database connection # but nothing is actually configured. internal_log_level, database_log_level, [handler] = m(self._db, testing=False) eq_(cls.INFO, internal_log_level) eq_(cls.WARN, database_log_level) assert isinstance(handler.formatter, JSONFormatter) # Let's set up a Loggly integration and change the defaults. loggly = self.loggly_integration() internal = self._external_integration( protocol=ExternalIntegration.INTERNAL_LOGGING, goal=ExternalIntegration.LOGGING_GOAL) ConfigurationSetting.sitewide(self._db, config.LOG_LEVEL).value = config.ERROR internal.setting( SysLogger.LOG_FORMAT).value = SysLogger.TEXT_LOG_FORMAT ConfigurationSetting.sitewide( self._db, config.DATABASE_LOG_LEVEL).value = config.DEBUG ConfigurationSetting.sitewide(self._db, config.LOG_APP_NAME).value = "test app" template = "%(filename)s:%(message)s" internal.setting(SysLogger.LOG_MESSAGE_TEMPLATE).value = template internal_log_level, database_log_level, handlers = m(self._db, testing=False) eq_(cls.ERROR, internal_log_level) eq_(cls.DEBUG, database_log_level) [loggly_handler ] = [x for x in handlers if isinstance(x, LogglyHandler)] eq_("http://example.com/a_token/", loggly_handler.url) eq_("test app", loggly_handler.formatter.app_name) [stream_handler ] = [x for x in handlers if isinstance(x, logging.StreamHandler)] assert isinstance(stream_handler.formatter, UTF8Formatter) eq_(template, stream_handler.formatter._fmt) # If testing=True, then the database configuration is ignored, # and the log setup is one that's appropriate for display # alongside unit test output. internal_log_level, database_log_level, [handler] = m(self._db, testing=True) eq_(cls.INFO, internal_log_level) eq_(cls.WARN, database_log_level) eq_(SysLogger.DEFAULT_MESSAGE_TEMPLATE, handler.formatter._fmt)
def site_configuration_last_update(cls, _db, known_value=None, timeout=None): """Check when the site configuration was last updated. Updates Configuration.instance[Configuration.SITE_CONFIGURATION_LAST_UPDATE]. It's the application's responsibility to periodically check this value and reload the configuration if appropriate. :param known_value: We know when the site configuration was last updated--it's this timestamp. Use it instead of checking with the database. :param timeout: We will only call out to the database once in this number of seconds. If we are asked again before this number of seconds elapses, we will assume site configuration has not changed. :return: a datetime object. """ now = datetime.datetime.utcnow() if _db and timeout is None: from model import ConfigurationSetting timeout = ConfigurationSetting.sitewide( _db, cls.SITE_CONFIGURATION_TIMEOUT).value if timeout is None: timeout = 60 last_check = cls.instance.get( cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE) if (not known_value and last_check and (now - last_check).total_seconds() < timeout): # We went to the database less than [timeout] seconds ago. # Assume there has been no change. return cls._site_configuration_last_update() # Ask the database when was the last time the site # configuration changed. Specifically, this is the last time # site_configuration_was_changed() (defined in model.py) was # called. if not known_value: from model import Timestamp known_value = Timestamp.value(_db, cls.SITE_CONFIGURATION_CHANGED, None) if not known_value: # The site configuration has never changed. last_update = None else: last_update = known_value # Update the Configuration object's record of the last update time. cls.instance[cls.SITE_CONFIGURATION_LAST_UPDATE] = last_update # Whether that record changed or not, the time at which we # _checked_ is going to be set to the current time. cls.instance[cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE] = now return last_update
def __init__(self, _db, title, url, libraries, annotator=None, live=True, url_for=None): """Turn a list of libraries into a catalog.""" if not annotator: annotator = Annotator() # To save bandwidth, omit logos from large feeds. What 'large' # means is customizable. include_logos = not (self._feed_is_large(_db, libraries)) self.catalog = dict(metadata=dict(title=title), catalogs=[]) self.add_link_to_catalog(self.catalog, rel="self", href=url, type=self.OPDS_TYPE) web_client_uri_template = ConfigurationSetting.sitewide( _db, Configuration.WEB_CLIENT_URL ).value for library in libraries: if not isinstance(library, tuple): library = (library,) self.catalog["catalogs"].append( self.library_catalog( *library, url_for=url_for, include_logo=include_logos, web_client_uri_template=web_client_uri_template ) ) annotator.annotate_catalog(self, live=live)
def ils_name_setting(cls, _db, collection, library): """Find the ConfigurationSetting controlling the ILS name for the given collection and library. """ return ConfigurationSetting.for_library_and_externalintegration( _db, cls.ILS_NAME_KEY, library, collection.external_integration )
def from_configuration(cls, _db, testing=False): """Return the logging policy as configured in the database. :param _db: A database connection. If None, the default logging policy will be used. :param testing: A boolean indicating whether a unit test is happening right now. If True, the database configuration will be ignored in favor of a known test-friendly policy. (It's okay to pass in False during a test *of this method*.) :return: A 3-tuple (internal_log_level, database_log_level, handlers). `internal_log_level` is the log level to be used for most log messages. `database_log_level` is the log level to be applied to the loggers for the database connector and other verbose third-party libraries. `handlers` is a list of Handler objects that will be associated with the top-level logger. """ log_level = cls.DEFAULT_LOG_LEVEL database_log_level = cls.DEFAULT_DATABASE_LOG_LEVEL if _db and not testing: log_level = ( ConfigurationSetting.sitewide(_db, Configuration.LOG_LEVEL).value or log_level ) database_log_level = ( ConfigurationSetting.sitewide(_db, Configuration.DATABASE_LOG_LEVEL).value or database_log_level ) loggers = [SysLogger, Loggly, CloudwatchLogs] handlers = [] errors = [] for logger in loggers: try: handler = logger.from_configuration(_db, testing) if handler: handlers.append(handler) except Exception, e: errors.append( "Error creating logger %s %s" % (logger.NAME, unicode(e)) )
def test_library_catalogs(self): l1 = self._library("The New York Public Library") l2 = self._library("Brooklyn Public Library") class TestAnnotator(object): def annotate_catalog(self, catalog_obj, live=True): catalog_obj.catalog['metadata'][ 'random'] = "Random text inserted by annotator." # This template will be used to construct a web client link # for each library. template = "http://web/{uuid}" ConfigurationSetting.sitewide( self._db, Configuration.WEB_CLIENT_URL).value = template catalog = OPDSCatalog(self._db, "A Catalog!", "http://url/", [l1, l2], TestAnnotator(), url_for=self.mock_url_for) catalog = unicode(catalog) parsed = json.loads(catalog) # The catalog is labeled appropriately. eq_("A Catalog!", parsed['metadata']['title']) [self_link] = parsed['links'] eq_("http://url/", self_link['href']) eq_("self", self_link['rel']) # The annotator modified the catalog in passing. eq_("Random text inserted by annotator.", parsed['metadata']['random']) # Each library became a catalog in the catalogs collection. eq_([l1.name, l2.name], [x['metadata']['title'] for x in parsed['catalogs']]) # Each library has a link to its web catalog. l1_links, l2_links = [ library['links'] for library in parsed['catalogs'] ] [l1_web] = [ link['href'] for link in l1_links if link['type'] == 'text/html' ] eq_(l1_web, template.replace("{uuid}", l1.internal_urn)) [l2_web] = [ link['href'] for link in l2_links if link['type'] == 'text/html' ] eq_(l2_web, template.replace("{uuid}", l2.internal_urn))
def registration_document(self): """Serve a document that describes the registration process, notably the terms of service for that process. The terms of service are hosted elsewhere; we only know the URL of the page they're stored. """ document = dict() # The terms of service may be encapsulated in a link to # a web page. terms_of_service_url = ConfigurationSetting.sitewide( self._db, Configuration.REGISTRATION_TERMS_OF_SERVICE_URL).value type = "text/html" rel = "terms-of-service" if terms_of_service_url: OPDSCatalog.add_link_to_catalog( document, rel=rel, type=type, href=terms_of_service_url, ) # And/or the terms of service may be described in # human-readable HTML, which we'll present as a data: link. terms_of_service_html = ConfigurationSetting.sitewide( self._db, Configuration.REGISTRATION_TERMS_OF_SERVICE_HTML).value if terms_of_service_html: encoded = base64.b64encode(terms_of_service_html) terms_of_service_link = "data:%s;base64,%s" % (type, encoded) OPDSCatalog.add_link_to_catalog(document, rel=rel, type=type, href=terms_of_service_link) return document
def do_run(self, _db=None, cmd_args=None, output=sys.stdout): _db = _db or self._db args = self.parse_command_line(_db, cmd_args=cmd_args) if args.setting: for setting in args.setting: key, value = self._parse_setting(setting) ConfigurationSetting.sitewide(_db, key).value = value settings = _db.query(ConfigurationSetting).filter( ConfigurationSetting.library_id==None).filter( ConfigurationSetting.external_integration==None ).order_by(ConfigurationSetting.key) output.write("Current site-wide settings:\n") for setting in settings: if args.show_secrets or not setting.is_secret: output.write("%s='%s'\n" % (setting.key, setting.value)) _db.commit()
def from_configuration(cls, _db, testing=False): settings = None cloudwatch = None app_name = cls.DEFAULT_APP_NAME if _db and not testing: goal = ExternalIntegration.LOGGING_GOAL settings = ExternalIntegration.lookup( _db, ExternalIntegration.CLOUDWATCH, goal) app_name = ConfigurationSetting.sitewide( _db, Configuration.LOG_APP_NAME).value or app_name if settings: cloudwatch = cls.get_handler(settings, testing) cls.set_formatter(cloudwatch, app_name) return cloudwatch
def from_configuration(cls, _db, testing=False): loggly = None from model import (ExternalIntegration, ConfigurationSetting) app_name = cls.DEFAULT_APP_NAME if _db and not testing: goal = ExternalIntegration.LOGGING_GOAL loggly = ExternalIntegration.lookup( _db, ExternalIntegration.LOGGLY, goal ) app_name = ConfigurationSetting.sitewide(_db, Configuration.LOG_APP_NAME).value or app_name if loggly: loggly = Loggly.loggly_handler(loggly) cls.set_formatter(loggly, app_name) return loggly
def test_feed_is_large(self): # Verify that the _feed_is_large helper method # works whether it's given a Python list or a SQLAlchemy query. setting = ConfigurationSetting.sitewide(self._db, Configuration.LARGE_FEED_SIZE) setting.value = 2 m = OPDSCatalog._feed_is_large query = self._db.query(Library) # There are no libraries, and the limit is 2, so a feed of libraries would not be large. eq_(0, query.count()) eq_(False, m(self._db, query)) # Make some libraries, and the feed becomes large. [self._library() for x in range(2)] eq_(True, m(self._db, query)) # It also works with a list. eq_(True, m(self._db, [1, 2])) eq_(False, m(self._db, [1]))
def _feed_is_large(cls, _db, libraries): """Determine whether a prospective feed is 'large' per a sitewide setting. :param _db: A database session :param libraries: A list of libraries (or anything else that might be going into a feed). """ large_feed_size = ConfigurationSetting.sitewide( _db, Configuration.LARGE_FEED_SIZE ).int_value if large_feed_size is None: # No limit return False if isinstance(libraries, Query): # This is a SQLAlchemy query. size = libraries.count() else: # This is something like a normal Python list. size = len(libraries) return size >= large_feed_size
def test_success(self): us = self._place(type=Place.NATION, abbreviated_name='US') library = self._library() s = SetCoverageAreaScript(_db=self._db) # Setting a service area with no focus area assigns that # service area to the library. args = [ "--library=%s" % library.name, '--service-area={"US": "everywhere"}' ] s.run(args) [area] = library.service_areas eq_(us, area.place) # Setting a focus area and not a service area treats 'everywhere' # as the service area. uk = self._place(type=Place.NATION, abbreviated_name='UK') args = [ "--library=%s" % library.name, '--focus-area={"UK": "everywhere"}' ] s.run(args) places = [x.place for x in library.service_areas] eq_(2, len(places)) assert uk in places assert Place.everywhere(self._db) in places # The library's former ServiceAreas have been removed. assert us not in places # If a default nation is set, you can name a single place as # your service area. ConfigurationSetting.sitewide( self._db, Configuration.DEFAULT_NATION_ABBREVIATION).value = "US" ut = self._place(type=Place.STATE, abbreviated_name='UT', parent=us) args = ["--library=%s" % library.name, '--service-area=UT'] s.run(args) [area] = library.service_areas eq_(ut, area.place)
def test_large_feeds_treated_differently(self): # The libraries in large feeds are converted to JSON in ways # that omit large chunks of data such as inline logos. # In this test, a feed with 2 or more items is considered # 'large'. Any smaller feed is considered 'small'. setting = ConfigurationSetting.sitewide(self._db, Configuration.LARGE_FEED_SIZE) setting.value = 2 class Mock(OPDSCatalog): def library_catalog(*args, **kwargs): # Every time library_catalog is called, record whether # we were asked to include a logo. return kwargs['include_logo'] # Every item in the large feed resulted in a call with # include_logo=False. large_feed = Mock(self._db, "title", "url", ["it's", "large"]) large_catalog = large_feed.catalog['catalogs'] eq_([False, False], large_catalog) # Every item in the large feed resulted in a call with # include_logo=True. small_feed = Mock(self._db, "title", "url", ["small"]) small_catalog = small_feed.catalog['catalogs'] eq_([True], small_catalog) # Make it so even a feed with one item is 'large'. setting.value = 1 small_feed = Mock(self._db, "title", "url", ["small"]) small_catalog = small_feed.catalog['catalogs'] eq_([False], small_catalog) # Try it with a query that returns no results. No catalogs # are included at all. small_feed = Mock(self._db, "title", "url", self._db.query(Library)) small_catalog = small_feed.catalog['catalogs'] eq_([], small_catalog)
def from_configuration(cls, _db, testing=False): (internal_log_format, message_template) = cls._defaults(testing) app_name = cls.DEFAULT_APP_NAME if _db and not testing: goal = ExternalIntegration.LOGGING_GOAL internal = ExternalIntegration.lookup( _db, ExternalIntegration.INTERNAL_LOGGING, goal) if internal: internal_log_format = (internal.setting(cls.LOG_FORMAT).value or internal_log_format) message_template = (internal.setting( cls.LOG_MESSAGE_TEMPLATE).value or message_template) app_name = ConfigurationSetting.sitewide( _db, Configuration.LOG_APP_NAME).value or app_name handler = logging.StreamHandler() cls.set_formatter(handler, log_format=internal_log_format, message_template=message_template, app_name=app_name) return handler
directory, "registry-admin.js") @app.route('/admin/static/registry-admin.css') @returns_problem_detail def admin_css(): directory = os.path.join(os.path.abspath(os.path.dirname(__file__)), "node_modules", "simplified-registry-admin", "dist") return app.library_registry.static_files.static_file( directory, "registry-admin.css") if __name__ == '__main__': debug = True if len(sys.argv) > 1: url = sys.argv[1] else: url = ConfigurationSetting.sitewide(_db, Configuration.BASE_URL).value url = url or u'http://localhost:7000/' scheme, netloc, path, parameters, query, fragment = urlparse.urlparse(url) if ':' in netloc: host, port = netloc.split(':') port = int(port) else: host = netloc port = 80 app.library_registry.log.info("Starting app on %s:%s", host, port) app.run(debug=debug, host=host, port=port)
def site_configuration_last_update(cls, _db, known_value=None, timeout=None): """Check when the site configuration was last updated. Updates Configuration.instance[Configuration.SITE_CONFIGURATION_LAST_UPDATE]. It's the application's responsibility to periodically check this value and reload the configuration if appropriate. :param known_value: We know when the site configuration was last updated--it's this timestamp. Use it instead of checking with the database. :param timeout: We will only call out to the database once in this number of seconds. If we are asked again before this number of seconds elapses, we will assume site configuration has not changed. :return: a datetime object. """ now = datetime.datetime.utcnow() if _db and timeout is None: from model import ConfigurationSetting timeout = ConfigurationSetting.sitewide( _db, cls.SITE_CONFIGURATION_TIMEOUT ).value if timeout is None: timeout = 60 last_check = cls.instance.get( cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE ) if (not known_value and last_check and (now - last_check).total_seconds() < timeout): # We went to the database less than [timeout] seconds ago. # Assume there has been no change. return cls._site_configuration_last_update() # Ask the database when was the last time the site # configuration changed. Specifically, this is the last time # site_configuration_was_changed() (defined in model.py) was # called. if not known_value: from model import Timestamp known_value = Timestamp.value( _db, cls.SITE_CONFIGURATION_CHANGED, service_type=None, collection=None ) if not known_value: # The site configuration has never changed. last_update = None else: last_update = known_value # Update the Configuration object's record of the last update time. cls.instance[cls.SITE_CONFIGURATION_LAST_UPDATE] = last_update # Whether that record changed or not, the time at which we # _checked_ is going to be set to the current time. cls.instance[cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE] = now return last_update
def set_secret_key(_db=None): _db = _db or app._db app.secret_key = ConfigurationSetting.sitewide_secret( _db, Configuration.SECRET_KEY)