def from_configuration(cls, _db, testing=False):
"""Return the logging policy as configured in the database.
:param _db: A database connection. If None, the default
logging policy will be used.
:param testing: A boolean indicating whether a unit test is
happening right now. If True, the database configuration will
be ignored in favor of a known test-friendly policy. (It's
okay to pass in False during a test *of this method*.)
:return: A 3-tuple (internal_log_level, database_log_level,
handlers). `internal_log_level` is the log level to be used
for most log messages. `database_log_level` is the log level
to be applied to the loggers for the database connector and
other verbose third-party libraries. `handlers` is a list of
Handler objects that will be associated with the top-level
logger.
"""
# Establish defaults, in case the database is not initialized or
# it is initialized but logging is not configured.
(internal_log_level, internal_log_format, database_log_level,
message_template) = cls._defaults(testing)
handlers = []
from model import ExternalIntegration
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
internal = ExternalIntegration.lookup(
_db, ExternalIntegration.INTERNAL_LOGGING, goal)
loggly = ExternalIntegration.lookup(_db,
ExternalIntegration.LOGGLY,
goal)
if internal:
internal_log_level = internal.setting(
cls.LOG_LEVEL).setdefault(internal_log_level)
internal_log_format = internal.setting(
cls.LOG_FORMAT).setdefault(internal_log_format)
database_log_level = internal.setting(
cls.DATABASE_LOG_LEVEL).setdefault(database_log_level)
message_template = internal.setting(
cls.LOG_MESSAGE_TEMPLATE).setdefault(message_template)
if loggly:
handlers.append(cls.loggly_handler(loggly))
# handlers is either empty or it contains a loggly handler.
# Let's also add a handler that logs to standard error.
handlers.append(logging.StreamHandler())
for handler in handlers:
cls.set_formatter(handler, internal_log_format, message_template)
return internal_log_level, database_log_level, handlers
def from_configuration(cls, _db, testing=False):
from model import (ExternalIntegration, ConfigurationSetting)
(internal_log_level, internal_log_format, database_log_level,
message_template) = cls._defaults(testing)
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
internal = ExternalIntegration.lookup(
_db, ExternalIntegration.INTERNAL_LOGGING, goal)
if internal:
internal_log_format = (internal.setting(cls.LOG_FORMAT).value
or internal_log_format)
message_template = (internal.setting(
cls.LOG_MESSAGE_TEMPLATE).value or message_template)
internal_log_level = (ConfigurationSetting.sitewide(
_db, Configuration.LOG_LEVEL).value or internal_log_level)
database_log_level = (ConfigurationSetting.sitewide(
_db, Configuration.DATABASE_LOG_LEVEL).value
or database_log_level)
app_name = ConfigurationSetting.sitewide(
_db, Configuration.LOG_APP_NAME).value or app_name
handler = logging.StreamHandler()
cls.set_formatter(handler, internal_log_format, message_template,
app_name)
return (handler, internal_log_level, database_log_level)
def __init__(self, _db, url=None, works_index=None):
self.log = logging.getLogger("External search index")
self.works_index = None
self.works_alias = None
integration = None
if not _db:
raise CannotLoadConfiguration(
"Cannot load Elasticsearch configuration without a database.",
)
if not url or not works_index:
integration = ExternalIntegration.lookup(
_db,
ExternalIntegration.ELASTICSEARCH,
goal=ExternalIntegration.SEARCH_GOAL)
if not integration:
raise CannotLoadConfiguration(
"No Elasticsearch integration configured.")
url = url or integration.url
if not works_index:
setting = integration.setting(self.WORKS_INDEX_KEY)
works_index = setting.value_or_default(
self.DEFAULT_WORKS_INDEX)
if not url:
raise CannotLoadConfiguration(
"No URL configured to Elasticsearch server.")
if not ExternalSearchIndex.__client:
use_ssl = url.startswith('https://')
self.log.info(
"Connecting to index %s in Elasticsearch cluster at %s",
works_index, url)
ExternalSearchIndex.__client = Elasticsearch(url,
use_ssl=use_ssl,
timeout=20,
maxsize=25)
self.indices = self.__client.indices
self.search = self.__client.search
self.index = self.__client.index
self.delete = self.__client.delete
self.exists = self.__client.exists
# Sets self.works_index and self.works_alias values.
# Document upload runs against the works_index.
# Search queries run against works_alias.
if works_index and integration:
self.set_works_index_and_alias(works_index)
self.update_integration_settings(integration)
def bulk(docs, **kwargs):
return elasticsearch_bulk(self.__client, docs, **kwargs)
self.bulk = bulk
def from_config(cls, library):
_db = Session.object_session(library)
integration = ExternalIntegration.lookup(
_db,
ExternalIntegration.MARC_EXPORT,
ExternalIntegration.CATALOG_GOAL,
library=library)
if not integration:
raise CannotLoadConfiguration(
"No MARC export service is configured for this library")
return cls(_db, library, integration)
def from_config(cls, library):
_db = Session.object_session(library)
integration = ExternalIntegration.lookup(
_db, ExternalIntegration.MARC_EXPORT,
ExternalIntegration.CATALOG_GOAL, library=library
)
if not integration:
raise CannotLoadConfiguration(
"No MARC export service is configured for this library"
)
return cls(_db, library, integration)
def from_configuration(cls, _db, testing=False):
settings = None
cloudwatch = None
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
settings = ExternalIntegration.lookup(
_db, ExternalIntegration.CLOUDWATCH, goal)
app_name = ConfigurationSetting.sitewide(
_db, Configuration.LOG_APP_NAME).value or app_name
if settings:
cloudwatch = cls.get_handler(settings, testing)
cls.set_formatter(cloudwatch, app_name)
return cloudwatch
def from_configuration(cls, _db, testing=False):
loggly = None
from model import (ExternalIntegration, ConfigurationSetting)
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
loggly = ExternalIntegration.lookup(
_db, ExternalIntegration.LOGGLY, goal
)
app_name = ConfigurationSetting.sitewide(_db, Configuration.LOG_APP_NAME).value or app_name
if loggly:
loggly = Loggly.loggly_handler(loggly)
cls.set_formatter(loggly, app_name)
return loggly
def test_run(self):
cmd_args = [
"--vendor-id=LIBR",
"--node-value=abc12",
"--delegate=http://server1/AdobeAuth/",
"--delegate=http://server2/AdobeAuth/",
]
script = ConfigureVendorIDScript(self._db)
script.do_run(self._db, cmd_args=cmd_args)
# The ExternalIntegration is properly configured.
integration = ExternalIntegration.lookup(
self._db, ExternalIntegration.ADOBE_VENDOR_ID,
ExternalIntegration.DRM_GOAL)
eq_("LIBR", integration.setting(Configuration.ADOBE_VENDOR_ID).value)
eq_(
"abc12",
integration.setting(
Configuration.ADOBE_VENDOR_ID_NODE_VALUE).value)
eq_(["http://server1/AdobeAuth/", "http://server2/AdobeAuth/"],
integration.setting(
Configuration.ADOBE_VENDOR_ID_DELEGATE_URL).json_value)
# The script won't run if --node-value or --delegate have obviously
# wrong values.
cmd_args = [
"--vendor-id=LIBR",
"--node-value=not a hex number",
]
assert_raises_regexp(ValueError,
"invalid literal for int",
script.do_run,
self._db,
cmd_args=cmd_args)
cmd_args = [
"--vendor-id=LIBR",
"--node-value=abce",
"--delegate=http://random-site/",
]
assert_raises_regexp(ValueError,
"Invalid delegate: http://random-site/",
script.do_run,
self._db,
cmd_args=cmd_args)
def vendor_id(cls, _db):
"""Look up the Adobe Vendor ID configuration for this registry.
:return: a 3-tuple (vendor ID, node value, [delegates])
"""
from model import ExternalIntegration
integration = ExternalIntegration.lookup(
_db, ExternalIntegration.ADOBE_VENDOR_ID,
ExternalIntegration.DRM_GOAL)
if not integration:
return None, None, []
setting = integration.setting(cls.ADOBE_VENDOR_ID_DELEGATE_URL)
delegates = []
try:
delegates = setting.json_value or []
except ValueError, e:
cls.log.warn("Invalid Adobe Vendor ID delegates configured.")
def _external_integration(self,
protocol,
goal=None,
settings=None,
libraries=None,
**kwargs):
integration = None
if not libraries:
integration, ignore = get_one_or_create(self._db,
ExternalIntegration,
protocol=protocol,
goal=goal)
else:
if not isinstance(libraries, list):
libraries = [libraries]
# Try to find an existing integration for one of the given
# libraries.
for library in libraries:
integration = ExternalIntegration.lookup(self._db,
protocol,
goal,
library=libraries[0])
if integration:
break
if not integration:
# Otherwise, create a brand new integration specifically
# for the library.
integration = ExternalIntegration(
protocol=protocol,
goal=goal,
)
integration.libraries.extend(libraries)
self._db.add(integration)
for attr, value in kwargs.items():
setattr(integration, attr, value)
settings = settings or dict()
for key, value in settings.items():
integration.set_setting(key, value)
return integration
def _external_integration(self, protocol, goal=None, settings=None,
libraries=None, **kwargs
):
integration = None
if not libraries:
integration, ignore = get_one_or_create(
self._db, ExternalIntegration, protocol=protocol, goal=goal
)
else:
if not isinstance(libraries, list):
libraries = [libraries]
# Try to find an existing integration for one of the given
# libraries.
for library in libraries:
integration = ExternalIntegration.lookup(
self._db, protocol, goal, library=libraries[0]
)
if integration:
break
if not integration:
# Otherwise, create a brand new integration specifically
# for the library.
integration = ExternalIntegration(
protocol=protocol, goal=goal,
)
integration.libraries.extend(libraries)
self._db.add(integration)
for attr, value in kwargs.items():
setattr(integration, attr, value)
settings = settings or dict()
for key, value in settings.items():
integration.set_setting(key, value)
return integration
def from_configuration(cls, _db, testing=False):
(internal_log_format, message_template) = cls._defaults(testing)
app_name = cls.DEFAULT_APP_NAME
if _db and not testing:
goal = ExternalIntegration.LOGGING_GOAL
internal = ExternalIntegration.lookup(
_db, ExternalIntegration.INTERNAL_LOGGING, goal)
if internal:
internal_log_format = (internal.setting(cls.LOG_FORMAT).value
or internal_log_format)
message_template = (internal.setting(
cls.LOG_MESSAGE_TEMPLATE).value or message_template)
app_name = ConfigurationSetting.sitewide(
_db, Configuration.LOG_APP_NAME).value or app_name
handler = logging.StreamHandler()
cls.set_formatter(handler,
log_format=internal_log_format,
message_template=message_template,
app_name=app_name)
return handler
import sys
import logging
from nose.tools import set_trace
bin_dir = os.path.split(__file__)[0]
package_dir = os.path.join(bin_dir, "..")
sys.path.append(os.path.abspath(package_dir))
from model import (
production_session,
ExternalIntegration as EI,
)
_db = production_session()
try:
integration = EI.lookup(_db, EI.METADATA_WRANGLER, EI.METADATA_GOAL)
if integration:
for setting in integration.settings:
if setting.key == 'username':
# A username (or client_id) is no longer required.
_db.delete(setting)
if setting.key == 'password':
# The password (previously client_secret) must be reset to
# register for a shared_secret.
setting.value = None
_db.commit()
_db.close()
except Exception as e:
_db.close()
raise e
def search_integration(cls, _db):
"""Look up the ExternalIntegration for ElasticSearch."""
return ExternalIntegration.lookup(
_db, ExternalIntegration.ELASTICSEARCH,
goal=ExternalIntegration.SEARCH_GOAL
)