def __init__(self, *args, **kwargs):
"""Initialize the site configuration.
Args:
*args (tuple):
Positional arguments to pass to the parent constructor.
**kwargs (dict):
Keyword arguments to pass to the parent constructor.
"""
super(SiteConfiguration, self).__init__(*args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer('%s:siteconfig:%s:generation' %
(self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def __init__(self, key):
self.key = key
self.pkg_resources = None
self._extension_classes = {}
self._extension_instances = {}
self._load_errors = {}
# State synchronization
self._gen_sync = GenerationSynchronizer('extensionmgr:%s:gen' % key)
self._load_lock = threading.Lock()
self._shutdown_lock = threading.Lock()
self._block_sync_gen = False
self.dynamic_urls = DynamicURLResolver()
# Extension middleware instances, ordered by dependencies.
self.middleware = []
# Wrap the INSTALLED_APPS and TEMPLATE_CONTEXT_PROCESSORS settings
# to allow for ref-counted add/remove operations.
self._installed_apps_setting = SettingListWrapper(
'INSTALLED_APPS', 'installed app')
self._context_processors_setting = SettingListWrapper(
'TEMPLATE_CONTEXT_PROCESSORS', 'context processor')
instance_id = id(self)
_extension_managers[instance_id] = self
def test_is_expired_after_other_process_updates(self):
"""Testing IntegrationManager.is_expired after another process updates
the configuration state
"""
manager = IntegrationManager(IntegrationConfig)
self.assertFalse(manager.is_expired())
gen_sync = GenerationSynchronizer(manager._gen_sync.cache_key, normalize_cache_key=False)
gen_sync.mark_updated()
self.assertTrue(manager.is_expired())
def test_is_expired_after_other_process_updates(self):
"""Testing IntegrationManager.is_expired after another process updates
the configuration state
"""
manager = IntegrationManager(IntegrationConfig)
self.assertFalse(manager.is_expired())
gen_sync = GenerationSynchronizer(manager._gen_sync.cache_key,
normalize_cache_key=False)
gen_sync.mark_updated()
self.assertTrue(manager.is_expired())
def __init__(self, config_model):
"""Initialize the integration manager.
Args:
config_model (type):
The model used to store configuration data. This must be a
subclass of
:py:class:`djblets.integrations.models.BaseIntegrationConfig`.
"""
# Check that the Django environment is set up for integrations to
# properly function.
if 'djblets.integrations' not in settings.INSTALLED_APPS:
raise ImproperlyConfigured(
'IntegrationManager requires djblets.integrations to be '
'listed in settings.INSTALLED_APPS.')
middleware = 'djblets.integrations.middleware.IntegrationsMiddleware'
if middleware not in settings.MIDDLEWARE_CLASSES:
raise ImproperlyConfigured(
'IntegrationManager requires %s to be listed in '
'settings.MIDDLEWARE_CLASSES' % middleware)
self.config_model = config_model
key = ('integrationmgr:%s.%s' %
(self.__class__.__module__, self.__class__.__name__))
self._integration_classes = {}
self._integration_configs = {}
self._integration_instances = {}
self._lock = threading.Lock()
self._needs_recalc = False
self._gen_sync = GenerationSynchronizer('%s:gen' % key)
instance_id = id(self)
_integration_managers[instance_id] = self
# Listen for any config model saves/deletes, so we can mark whether
# a reload of state is needed.
dispatch_uid = '%s:%s' % (key, id(self))
post_delete.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
post_save.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
def __init__(self, key):
self.key = key
self.pkg_resources = None
self._extension_classes = {}
self._extension_instances = {}
self._load_errors = {}
# State synchronization
self._gen_sync = GenerationSynchronizer('extensionmgr:%s:gen' % key)
self._load_lock = threading.Lock()
self._shutdown_lock = threading.Lock()
self._block_sync_gen = False
self.dynamic_urls = DynamicURLResolver()
# Extension middleware instances, ordered by dependencies.
self.middleware = []
# Wrap the INSTALLED_APPS and TEMPLATE_CONTEXT_PROCESSORS settings
# to allow for ref-counted add/remove operations.
self._installed_apps_setting = SettingListWrapper(
'INSTALLED_APPS',
'installed app')
self._context_processors_setting = SettingListWrapper(
'TEMPLATE_CONTEXT_PROCESSORS',
'context processor')
instance_id = id(self)
_extension_managers[instance_id] = self
def __init__(self, *args, **kwargs):
"""Initialize the site configuration.
Args:
*args (tuple):
Positional arguments to pass to the parent constructor.
**kwargs (dict):
Keyword arguments to pass to the parent constructor.
"""
super(SiteConfiguration, self).__init__(*args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer(
'%s:siteconfig:%s:generation' % (self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def __init__(self, *args, **kwargs):
models.Model.__init__(self, *args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer('%s:siteconfig:%s:generation' %
(self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def __init__(self, config_model):
"""Initialize the integration manager.
Args:
config_model (type):
The model used to store configuration data. This must be a
subclass of
:py:class:`djblets.integrations.models.BaseIntegrationConfig`.
"""
# Check that the Django environment is set up for integrations to
# properly function.
if 'djblets.integrations' not in settings.INSTALLED_APPS:
raise ImproperlyConfigured(
'IntegrationManager requires djblets.integrations to be '
'listed in settings.INSTALLED_APPS.'
)
middleware = 'djblets.integrations.middleware.IntegrationsMiddleware'
if middleware not in settings.MIDDLEWARE_CLASSES:
raise ImproperlyConfigured(
'IntegrationManager requires %s to be listed in '
'settings.MIDDLEWARE_CLASSES'
% middleware
)
self.config_model = config_model
key = ('integrationmgr:%s.%s'
% (self.__class__.__module__, self.__class__.__name__))
self.enabled = True
self._integration_classes = {}
self._integration_configs = {}
self._integration_instances = {}
self._lock = threading.Lock()
self._needs_recalc = False
self._gen_sync = GenerationSynchronizer('%s:gen' % key)
instance_id = id(self)
_integration_managers[instance_id] = self
# Listen for any config model saves/deletes, so we can mark whether
# a reload of state is needed.
dispatch_uid = '%s:%s' % (key, id(self))
post_delete.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
post_save.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
def __init__(self, *args, **kwargs):
models.Model.__init__(self, *args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer(
'%s:siteconfig:%s:generation' % (self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
class GenerationSynchronizerTests(TestCase):
"""Unit tests for djblets.cache.synchronizer.GenerationSynchronizer."""
def setUp(self):
super(GenerationSynchronizerTests, self).setUp()
self.gen_sync = GenerationSynchronizer('test-synchronizer')
def test_initial_state(self):
"""Testing GenerationSynchronizer initial state"""
self.assertIsNotNone(self.gen_sync.sync_gen)
def test_is_expired_when_expired(self):
"""Testing GenerationSynchronizer.is_expired when expired"""
cache.set(self.gen_sync.cache_key, self.gen_sync.sync_gen + 1)
self.assertTrue(self.gen_sync.is_expired())
def test_is_expired_when_not_expired(self):
"""Testing GenerationSynchronizer.is_expired when not expired"""
self.assertFalse(self.gen_sync.is_expired())
def test_refresh(self):
"""Testing GenerationSynchronizer.refresh"""
new_sync_gen = self.gen_sync.sync_gen + 1
cache.set(self.gen_sync.cache_key, new_sync_gen)
self.gen_sync.refresh()
self.assertEqual(self.gen_sync.sync_gen, new_sync_gen)
def test_clear(self):
"""Testing GenerationSynchronizer.clear"""
self.gen_sync.clear()
self.assertEqual(cache.get(self.gen_sync.cache_key), None)
def test_mark_updated(self):
"""Testing GenerationSynchronizer.mark_updated"""
sync_gen = self.gen_sync.sync_gen
self.gen_sync.mark_updated()
self.assertEqual(self.gen_sync.sync_gen, sync_gen + 1)
self.assertEqual(cache.get(self.gen_sync.cache_key),
self.gen_sync.sync_gen)
class GenerationSynchronizerTests(TestCase):
"""Unit tests for djblets.cache.synchronizer.GenerationSynchronizer."""
def setUp(self):
super(GenerationSynchronizerTests, self).setUp()
self.gen_sync = GenerationSynchronizer('test-synchronizer')
def test_initial_state(self):
"""Testing GenerationSynchronizer initial state"""
self.assertIsNotNone(self.gen_sync.sync_gen)
def test_is_expired_when_expired(self):
"""Testing GenerationSynchronizer.is_expired when expired"""
cache.set(self.gen_sync.cache_key, self.gen_sync.sync_gen + 1)
self.assertTrue(self.gen_sync.is_expired())
def test_is_expired_when_not_expired(self):
"""Testing GenerationSynchronizer.is_expired when not expired"""
self.assertFalse(self.gen_sync.is_expired())
def test_refresh(self):
"""Testing GenerationSynchronizer.refresh"""
new_sync_gen = self.gen_sync.sync_gen + 1
cache.set(self.gen_sync.cache_key, new_sync_gen)
self.gen_sync.refresh()
self.assertEqual(self.gen_sync.sync_gen, new_sync_gen)
def test_clear(self):
"""Testing GenerationSynchronizer.clear"""
self.gen_sync.clear()
self.assertEqual(cache.get(self.gen_sync.cache_key), None)
def test_mark_updated(self):
"""Testing GenerationSynchronizer.mark_updated"""
sync_gen = self.gen_sync.sync_gen
self.gen_sync.mark_updated()
self.assertEqual(self.gen_sync.sync_gen, sync_gen + 1)
self.assertEqual(cache.get(self.gen_sync.cache_key),
self.gen_sync.sync_gen)
class ExtensionManager(object):
"""A manager for all extensions.
ExtensionManager manages the extensions available to a project. It can
scan for new extensions, enable or disable them, determine dependencies,
install into the database, and uninstall.
An installed extension is one that has been installed by a Python package
on the system.
A registered extension is one that has been installed and information then
placed in the database. This happens automatically after scanning for
an installed extension. The registration data stores whether or not it's
enabled, and stores various pieces of information on the extension.
An enabled extension is one that is actively enabled and hooked into the
project.
Each project should have one ExtensionManager.
"""
#: Whether to explicitly install static media files from packages.
#:
#: By default, we install static media files if :django:setting:`DEBUG`
#: is ``True``. Subclasses can override this to factor in other settings,
#: if needed.
should_install_static_media = not settings.DEBUG
#: The key in the settings indicating the last known configured version.
#:
#: This setting is used to differentiate the version of an extension
#: installed locally on a system and the last version that completed
#: database-related installation/upgrade steps.
#:
#: This should not be changed by subclasses, and generally is not needed
#: outside of this class.
VERSION_SETTINGS_KEY = '_extension_installed_version'
def __init__(self, key):
self.key = key
self.pkg_resources = None
self._extension_classes = {}
self._extension_instances = {}
self._load_errors = {}
# State synchronization
self._gen_sync = GenerationSynchronizer('extensionmgr:%s:gen' % key)
self._load_lock = threading.Lock()
self._shutdown_lock = threading.Lock()
self._block_sync_gen = False
self.dynamic_urls = DynamicURLResolver()
# Extension middleware instances, ordered by dependencies.
self.middleware = []
# Wrap the INSTALLED_APPS and TEMPLATE_CONTEXT_PROCESSORS settings
# to allow for ref-counted add/remove operations.
self._installed_apps_setting = SettingListWrapper(
'INSTALLED_APPS',
'installed app')
if hasattr(settings, 'TEMPLATES'):
# Django >= 1.7
self._context_processors_setting = SettingListWrapper(
'context_processors',
'context processor',
parent_dict=settings.TEMPLATES[0]['OPTIONS'])
else:
self._context_processors_setting = SettingListWrapper(
'TEMPLATE_CONTEXT_PROCESSORS',
'context processor')
instance_id = id(self)
_extension_managers[instance_id] = self
def get_url_patterns(self):
"""Return the URL patterns for the Extension Manager.
This should be included in the root urlpatterns for the site.
Returns:
list:
The list of URL patterns for the Extension Manager.
"""
return [self.dynamic_urls]
def is_expired(self):
"""Returns whether or not the extension state is possibly expired.
Extension state covers the lists of extensions and each extension's
configuration. It can expire if the state synchronization value
falls out of cache or is changed.
Each ExtensionManager has its own state synchronization cache key.
Returns:
bool:
Whether the state has expired.
"""
return self._gen_sync.is_expired()
def clear_sync_cache(self):
"""Clear the extension synchronization state.
This will force every process to reload the extension list and
settings.
"""
self._gen_sync.clear()
def get_absolute_url(self):
return reverse('extension-list')
def get_can_disable_extension(self, registered_extension):
extension_id = registered_extension.class_name
return (registered_extension.extension_class is not None and
(self.get_enabled_extension(extension_id) is not None or
extension_id in self._load_errors))
def get_can_enable_extension(self, registered_extension):
return (registered_extension.extension_class is not None and
self.get_enabled_extension(
registered_extension.class_name) is None)
def get_enabled_extension(self, extension_id):
"""Returns an enabled extension with the given ID."""
if extension_id in self._extension_instances:
return self._extension_instances[extension_id]
return None
def get_enabled_extensions(self):
"""Returns the list of all enabled extensions."""
return list(self._extension_instances.values())
def get_installed_extensions(self):
"""Returns the list of all installed extensions."""
return list(self._extension_classes.values())
def get_installed_extension(self, extension_id):
"""Returns the installed extension with the given ID."""
if extension_id not in self._extension_classes:
raise InvalidExtensionError(extension_id)
return self._extension_classes[extension_id]
def get_dependent_extensions(self, dependency_extension_id):
"""Returns a list of all extensions required by an extension."""
if dependency_extension_id not in self._extension_instances:
raise InvalidExtensionError(dependency_extension_id)
dependency = self.get_installed_extension(dependency_extension_id)
result = []
for extension_id, extension in six.iteritems(self._extension_classes):
if extension_id == dependency_extension_id:
continue
for ext_requirement in extension.info.requirements:
if ext_requirement == dependency:
result.append(extension_id)
return result
def enable_extension(self, extension_id):
"""Enables an extension.
Enabling an extension will install any data files the extension
may need, any tables in the database, perform any necessary
database migrations, and then will start up the extension.
"""
if extension_id in self._extension_instances:
# It's already enabled.
return
if extension_id not in self._extension_classes:
if extension_id in self._load_errors:
raise EnablingExtensionError(
_('There was an error loading this extension'),
self._load_errors[extension_id],
needs_reload=True)
raise InvalidExtensionError(extension_id)
ext_class = self._extension_classes[extension_id]
# Enable extension dependencies
for requirement_id in ext_class.requirements:
self.enable_extension(requirement_id)
extension = self._init_extension(ext_class)
ext_class.registration.enabled = True
ext_class.registration.save()
clear_template_caches()
self._bump_sync_gen()
self._recalculate_middleware()
return extension
def disable_extension(self, extension_id):
"""Disables an extension.
Disabling an extension will remove any data files the extension
installed and then shut down the extension and all of its hooks.
It will not delete any data from the database.
"""
has_load_error = extension_id in self._load_errors
if not has_load_error:
if extension_id not in self._extension_instances:
# It's not enabled.
return
if extension_id not in self._extension_classes:
raise InvalidExtensionError(extension_id)
extension = self._extension_instances[extension_id]
for dependent_id in self.get_dependent_extensions(extension_id):
self.disable_extension(dependent_id)
self._uninstall_extension(extension)
self._uninit_extension(extension)
self._unregister_static_bundles(extension)
registration = extension.registration
else:
del self._load_errors[extension_id]
if extension_id in self._extension_classes:
# The class was loadable, so it just couldn't be instantiated.
# Update the registration on the class.
ext_class = self._extension_classes[extension_id]
registration = ext_class.registration
else:
registration = RegisteredExtension.objects.get(
class_name=extension_id)
registration.enabled = False
registration.save(update_fields=['enabled'])
clear_template_caches()
self._bump_sync_gen()
self._recalculate_middleware()
def install_extension(self, install_url, package_name):
"""Install an extension from a remote source.
Installs an extension from a remote URL containing the
extension egg. Installation may fail if a malformed install_url
or package_name is passed, which will cause an InstallExtensionError
exception to be raised. It is also assumed that the extension is not
already installed.
"""
try:
easy_install.main(["-U", install_url])
# Update the entry points.
dist = pkg_resources.get_distribution(package_name)
dist.activate()
pkg_resources.working_set.add(dist)
except pkg_resources.DistributionNotFound:
raise InstallExtensionError(_("Invalid package name."))
except SystemError:
raise InstallExtensionError(
_('Installation failed (probably malformed URL).'))
# Refresh the extension manager.
self.load(True)
def load(self, full_reload=False):
"""
Loads all known extensions, initializing any that are recorded as
being enabled.
If this is called a second time, it will refresh the list of
extensions, adding new ones and removing deleted ones.
If full_reload is passed, all state is cleared and we reload all
extensions and state from scratch.
"""
with self._load_lock:
self._block_sync_gen = True
self._load_extensions(full_reload)
self._block_sync_gen = False
def shutdown(self):
"""Shut down the extension manager and all of its extensions."""
with self._shutdown_lock:
self._clear_extensions()
def _load_extensions(self, full_reload=False):
if full_reload:
# We're reloading everything, so nuke all the cached copies.
self._clear_extensions()
clear_template_caches()
self._load_errors = {}
# Preload all the RegisteredExtension objects
registered_extensions = {}
for registered_ext in RegisteredExtension.objects.all():
registered_extensions[registered_ext.class_name] = registered_ext
found_extensions = {}
found_registrations = {}
registrations_to_fetch = []
find_registrations = False
extensions_changed = False
for entrypoint in self._entrypoint_iterator():
registered_ext = None
try:
ext_class = entrypoint.load()
except Exception as e:
logger.exception('Error loading extension %s: %s',
entrypoint.name, e)
extension_id = '%s.%s' % (entrypoint.module_name,
'.'.join(entrypoint.attrs))
self._store_load_error(extension_id, e)
continue
# A class's extension ID is its class name. We want to
# make this easier for users to access by giving it an 'id'
# variable, which will be accessible both on the class and on
# instances.
class_name = ext_class.id = "%s.%s" % (ext_class.__module__,
ext_class.__name__)
self._extension_classes[class_name] = ext_class
found_extensions[class_name] = ext_class
# Don't override the info if we've previously loaded this
# class.
if not getattr(ext_class, 'info', None):
ext_class.info = ExtensionInfo.create_from_entrypoint(
entrypoint, ext_class)
registered_ext = registered_extensions.get(class_name)
if registered_ext:
found_registrations[class_name] = registered_ext
if not hasattr(ext_class, 'registration'):
find_registrations = True
else:
registrations_to_fetch.append(
(class_name, entrypoint.dist.project_name))
find_registrations = True
if find_registrations:
if registrations_to_fetch:
stored_registrations = list(
RegisteredExtension.objects.filter(
class_name__in=registrations_to_fetch))
# Go through the list of registrations found in the database
# and mark them as found for later processing.
for registered_ext in stored_registrations:
class_name = registered_ext.class_name
found_registrations[class_name] = registered_ext
# Go through each registration we still need and couldn't find,
# and create an entry in the database. These are going to be
# newly discovered extensions.
for class_name, ext_name in registrations_to_fetch:
if class_name not in found_registrations:
try:
registered_ext = RegisteredExtension.objects.create(
class_name=class_name,
name=ext_name)
except IntegrityError:
# An entry was created since we last looked up
# anything. Fetch it from the database.
registered_ext = RegisteredExtension.objects.get(
class_name=class_name)
found_registrations[class_name] = registered_ext
# Now we have all the RegisteredExtension instances. Go through
# and initialize each of them.
for class_name, registered_ext in six.iteritems(found_registrations):
ext_class = found_extensions[class_name]
ext_class.registration = registered_ext
if (ext_class.registration.enabled and
ext_class.id not in self._extension_instances):
try:
self._init_extension(ext_class)
except EnablingExtensionError:
# When in debug mode, we want this error to be noticed.
# However, in production, it shouldn't break the whole
# server, so continue on.
if not settings.DEBUG:
continue
extensions_changed = True
# At this point, if we're reloading, it's possible that the user
# has removed some extensions. Go through and remove any that we
# can no longer find.
#
# While we're at it, since we're at a point where we've seen all
# extensions, we can set the ExtensionInfo.requirements for
# each extension
for class_name, ext_class in six.iteritems(self._extension_classes):
if class_name not in found_extensions:
if class_name in self._extension_instances:
self.disable_extension(class_name)
del self._extension_classes[class_name]
extensions_changed = True
else:
ext_class.info.requirements = \
[self.get_installed_extension(requirement_id)
for requirement_id in ext_class.requirements]
# Add the sync generation if it doesn't already exist.
self._gen_sync.refresh()
settings.AJAX_SERIAL = self._gen_sync.sync_gen
if extensions_changed:
self._recalculate_middleware()
def _clear_extensions(self):
"""Clear the entire list of known extensions.
This will bring the ExtensionManager back to the state where
it doesn't yet know about any extensions, requiring a re-load.
"""
for extension in self.get_enabled_extensions():
# Make sure this is actually an enabled extension, and not one
# that's already been shut down by another instance of this
# ExtensionManager (which should only happen in tests):
if hasattr(extension.__class__, 'info'):
self._uninit_extension(extension)
for extension_class in self.get_installed_extensions():
if hasattr(extension_class, 'info'):
delattr(extension_class, 'info')
if hasattr(extension_class, 'registration'):
delattr(extension_class, 'registration')
self._extension_classes = {}
self._extension_instances = {}
def _init_extension(self, ext_class):
"""Initializes an extension.
This will register the extension, install any URLs that it may need,
and make it available in Django's list of apps. It will then notify
that the extension has been initialized.
"""
extension_id = ext_class.id
assert extension_id not in self._extension_instances
try:
extension = ext_class(extension_manager=self)
except Exception as e:
logger.exception('Unable to initialize extension %s: %s',
ext_class, e)
error_details = self._store_load_error(extension_id, e)
raise EnablingExtensionError(
_('Error initializing extension: %s') % e,
error_details)
if extension_id in self._load_errors:
del self._load_errors[extension_id]
self._extension_instances[extension_id] = extension
if extension.has_admin_site:
self._init_admin_site(extension)
# Installing the urls must occur after _init_admin_site(). The urls
# for the admin site will not be generated until it is called.
self._install_admin_urls(extension)
self._register_static_bundles(extension)
extension.info.installed = extension.registration.installed
extension.info.enabled = True
self._add_to_installed_apps(extension)
self._context_processors_setting.add_list(extension.context_processors)
clear_template_tag_caches()
ext_class.instance = extension
try:
self.install_extension_media(ext_class)
except InstallExtensionError as e:
raise EnablingExtensionError(e.message, e.load_error)
# Check if the version information stored along with the extension is
# stale. If so, we may need to perform some updates.
cur_version = ext_class.info.version
if ext_class.registration.installed:
old_version = extension.settings.get(self.VERSION_SETTINGS_KEY)
else:
old_version = None
if (not old_version or
pkg_resources.parse_version(old_version) <
pkg_resources.parse_version(cur_version)):
# We may need to update the database.
self._sync_database(ext_class)
# Record this version so we don't update the database again.
extension.settings.set(self.VERSION_SETTINGS_KEY, cur_version)
extension.settings.save()
elif (old_version and
pkg_resources.parse_version(old_version) >
pkg_resources.parse_version(cur_version)):
logging.warning('The version of the "%s" extension installed on '
'the server is older than the version recorded '
'in the database! Upgrades will not be performed.',
ext_class)
# Mark the extension as installed.
ext_class.registration.installed = True
ext_class.registration.save()
extension_initialized.send(self, ext_class=extension)
return extension
def _sync_database(self, ext_class):
"""Synchronize extension-provided models to the database.
This will create any database tables that need to be created and
the perform a database migration, if needed.
Args:
ext_class (type):
The extension class owning the database models.
"""
if apps:
# Django >= 1.7
call_command('migrate', run_syncdb=True, verbosity=0,
interactive=False)
else:
# Django == 1.6
loading.cache.loaded = False
call_command('syncdb', verbosity=0, interactive=False)
# Run evolve to do any table modification.
self._migrate_extension_models(ext_class)
def _uninit_extension(self, extension):
"""Uninitializes the extension.
This will shut down the extension, remove any URLs, remove it from
Django's list of apps, and send a signal saying the extension was
shut down.
"""
extension.shutdown()
if hasattr(extension, "admin_urlpatterns"):
self.dynamic_urls.remove_patterns(
extension.admin_urlpatterns)
if hasattr(extension, "admin_site_urlpatterns"):
self.dynamic_urls.remove_patterns(
extension.admin_site_urlpatterns)
if hasattr(extension, 'admin_site'):
del extension.admin_site
self._context_processors_setting.remove_list(
extension.context_processors)
self._remove_from_installed_apps(extension)
clear_template_tag_caches()
extension.info.enabled = False
extension_uninitialized.send(self, ext_class=extension)
del self._extension_instances[extension.id]
extension.__class__.instance = None
def _store_load_error(self, extension_id, err):
"""Stores and returns a load error for the extension ID."""
error_details = '%s\n\n%s' % (err, traceback.format_exc())
self._load_errors[extension_id] = error_details
return error_details
def install_extension_media(self, ext_class, force=False):
"""Installs extension static media.
This method is a wrapper around _install_extension_media_internal to
check whether we actually need to install extension media, and avoid
contention among multiple threads/processes when doing so.
We need to install extension media if it hasn't been installed yet,
or if the version of the extension media that we installed is different
from the current version of the extension.
"""
# If we're not installing static media, it's assumed that media will
# be looked up using the static media finders instead. In that case,
# media will be served directly out of the extension's static/
# directory. We won't want to be storing version files there.
if not self.should_install_static_media:
return
lockfile = os.path.join(tempfile.gettempdir(), ext_class.id + '.lock')
media_version_dir = ext_class.info.installed_static_path
media_version_filename = os.path.join(media_version_dir, '.version')
cur_version = ext_class.info.version
# We only want to fetch the existing version information if the
# extension is already installed. We remove this key when
# disabling an extension, so if it were there, it was either
# copy/pasted, or something went wrong. Either way, we wouldn't
# be able to trust it.
if force:
logger.debug('Forcing installation of extension media for %s',
ext_class.info)
old_version = None
else:
if (ext_class.registration.installed and
os.path.exists(media_version_filename)):
# There's a media version stamp we can read from for this site.
with open(media_version_filename, 'r') as fp:
old_version = fp.read().strip()
else:
# This is either a new install, or an older one from before the
# media version stamp files.
old_version = None
if old_version == cur_version:
# Nothing to do
return
if not old_version:
logger.debug('Installing extension media for %s',
ext_class.info)
else:
logger.debug('Reinstalling extension media for %s because '
'version changed from %s',
ext_class.info, old_version)
while old_version != cur_version:
with open(lockfile, 'w') as f:
try:
locks.lock(f, locks.LOCK_EX | locks.LOCK_NB)
except IOError as e:
if e.errno in (errno.EAGAIN, errno.EACCES, errno.EINTR):
# Sleep for one second, then try again
time.sleep(1)
# See if the version has changed at all. If so, we'll
# be able to break the loop. Otherwise, we're going to
# try for the lock again.
if os.path.exists(media_version_filename):
with open(media_version_filename, 'r') as fp:
old_version = fp.read().strip()
continue
else:
raise e
self._install_extension_media_internal(ext_class)
try:
if not os.path.exists(media_version_dir):
os.makedirs(media_version_dir, 0755)
with open(media_version_filename, 'w') as fp:
fp.write('%s\n' % cur_version)
except IOError:
logging.error('Failed to write static media version file '
'"%s" for extension "%s": %s',
media_version_filename, ext_class.info, e)
old_version = cur_version
locks.unlock(f)
try:
os.unlink(lockfile)
except OSError as e:
# A "No such file or directory" (ENOENT) is most likely due to
# another thread removing the lock file before this thread could.
# It's safe to ignore. We want to handle all others, though.
if e.errno != errno.ENOENT:
logger.exception("Failed to unlock media lock file '%s' for "
"extension '%s': %s",
lockfile, ext_class.info, e)
def _install_extension_media_internal(self, ext_class):
"""Install static media for an extension.
Performs any installation necessary for an extension. If the extension
has a modern static/ directory, they will be installed into
:file:`{settings.STATIC_ROOT}/ext/`.
"""
if pkg_resources.resource_exists(ext_class.__module__, 'htdocs'):
# This is an older extension that doesn't use the static file
# support. Log a deprecation notice and then install the files.
logger.error('The %s extension uses the deprecated "htdocs" '
'directory for static files. This is no longer '
'supported. It must be updated to use a "static" '
'directory instead.',
ext_class.info.name)
ext_static_path = ext_class.info.installed_static_path
ext_static_path_exists = os.path.exists(ext_static_path)
if ext_static_path_exists:
# Also get rid of the old static contents.
shutil.rmtree(ext_static_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'static'):
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__,
'static')
shutil.copytree(extracted_path, ext_static_path, symlinks=True)
def _uninstall_extension(self, extension):
"""Uninstalls extension data.
Performs any uninstallation necessary for an extension.
This will uninstall the contents of MEDIA_ROOT/ext/ and
STATIC_ROOT/ext/.
"""
extension.settings.set(self.VERSION_SETTINGS_KEY, None)
extension.settings.save()
extension.registration.installed = False
extension.registration.save()
for path in (extension.info.installed_htdocs_path,
extension.info.installed_static_path):
if os.path.exists(path):
shutil.rmtree(path, ignore_errors=True)
def _install_admin_urls(self, extension):
"""Installs administration URLs.
This provides URLs for configuring an extension, plus any additional
admin urlpatterns that the extension provides.
"""
prefix = self.get_absolute_url()
if hasattr(settings, 'SITE_ROOT'):
prefix = prefix[len(settings.SITE_ROOT):]
# Note that we're adding to the resolve list on the root of the
# install, and prefixing it with the admin extensions path.
# The reason we're not just making this a child of our extensions
# urlconf is that everything in there gets passed an
# extension_manager variable, and we don't want to force extensions
# to handle this.
if extension.is_configurable:
urlconf = extension.admin_urlconf
if hasattr(urlconf, 'urlpatterns'):
extension.admin_urlpatterns = [
url(r'^%s%s/config/' % (prefix, extension.id),
include(urlconf.__name__)),
]
self.dynamic_urls.add_patterns(
extension.admin_urlpatterns)
if getattr(extension, 'admin_site', None):
extension.admin_site_urlpatterns = [
url(r'^%s%s/db/' % (prefix, extension.id),
include(extension.admin_site.urls)),
]
self.dynamic_urls.add_patterns(
extension.admin_site_urlpatterns)
def _register_static_bundles(self, extension):
"""Registers the extension's static bundles with Pipeline.
Each static bundle will appear as an entry in Pipeline. The
bundle name and filenames will be changed to include the extension
ID for the static file lookups.
"""
def _add_prefix(filename):
return 'ext/%s/%s' % (extension.id, filename)
def _add_bundles(pipeline_bundles, extension_bundles, default_dir,
ext):
for name, bundle in six.iteritems(extension_bundles):
new_bundle = bundle.copy()
new_bundle['source_filenames'] = [
_add_prefix(filename)
for filename in bundle.get('source_filenames', [])
]
new_bundle['output_filename'] = _add_prefix(bundle.get(
'output_filename',
'%s/%s.min%s' % (default_dir, name, ext)))
pipeline_bundles[extension.get_bundle_id(name)] = new_bundle
_add_bundles(pipeline_settings.STYLESHEETS, extension.css_bundles,
'css', '.css')
_add_bundles(pipeline_settings.JAVASCRIPT, extension.js_bundles,
'js', '.js')
def _unregister_static_bundles(self, extension):
"""Unregisters the extension's static bundles from Pipeline.
Every static bundle previously registered will be removed.
"""
def _remove_bundles(pipeline_bundles, extension_bundles):
for name, bundle in six.iteritems(extension_bundles):
try:
del pipeline_bundles[extension.get_bundle_id(name)]
except KeyError:
pass
if hasattr(settings, 'PIPELINE'):
_remove_bundles(pipeline_settings.STYLESHEETS,
extension.css_bundles)
_remove_bundles(pipeline_settings.JAVASCRIPT,
extension.js_bundles)
def _init_admin_site(self, extension):
"""Creates and initializes an admin site for an extension.
This creates the admin site and imports the extensions admin
module to register the models.
The url patterns for the admin site are generated in
_install_admin_urls().
"""
extension.admin_site = AdminSite(extension.info.app_name)
# Import the extension's admin module.
try:
admin_module_name = '%s.admin' % extension.info.app_name
if admin_module_name in sys.modules:
# If the extension has been loaded previously and
# we are re-enabling it, we must reload the module.
# Just importing again will not cause the ModelAdmins
# to be registered.
reload(sys.modules[admin_module_name])
else:
import_module(admin_module_name)
except ImportError:
mod = import_module(extension.info.app_name)
# Decide whether to bubble up this error. If the app just
# doesn't have an admin module, we can ignore the error
# attempting to import it, otherwise we want it to bubble up.
if module_has_submodule(mod, 'admin'):
raise ImportError(
"Importing admin module for extension %s failed"
% extension.info.app_name)
def _add_to_installed_apps(self, extension):
"""Add an extension's apps to the list of installed apps.
This will register each app with Django and clear any caches needed
to load the extension's modules.
Args:
extension (djblets.extensions.extension.Extension):
The extension whose apps are being added.
"""
self._installed_apps_setting.add_list(
extension.apps or [extension.info.app_name])
if apps:
apps.set_installed_apps(settings.INSTALLED_APPS)
def _remove_from_installed_apps(self, extension):
"""Remove an extension's apps from the list of installed apps.
This will unregister each app with Django and clear any caches
storing the apps' models.
Args:
extension (djblets.extensions.extension.Extension):
The extension whose apps are being removed.
"""
# Remove the extension's apps from INSTALLED_APPS.
removed_apps = self._installed_apps_setting.remove_list(
extension.apps or [extension.info.app_name])
# Now clear the apps and their modules from any caches.
if apps:
apps.unset_installed_apps()
else:
for app_name in removed_apps:
# In Django 1.6, the only apps that are registered are those
# with models. If this particular app does not have models, we
# don't want to clear any caches below. There might be another
# app with the same label that actually does have models, and
# we'd be clearing those away. An example would be
# reviewboard.hostingsvcs (which has models) and
# rbpowerpack.hostingsvcs (which does not).
#
# Django 1.6 doesn't technically allow multiple apps with the
# same label to have models (other craziness will happen), so
# we don't have to worry about that. It's not our problem.
try:
app_mod = import_module('%s.models' % app_name)
except ImportError:
# Something went very wrong. Maybe this module didn't
# exist anymore. Ignore it.
continue
# Fetch the models before we make any changes to the cache.
model_modules = {app_mod}
model_modules.update(
import_module(model.__module__)
for model in loading.get_models(app_mod)
)
# Start pruning this app from the caches.
#
# We are going to keep this in loading.cache.app_models.
# If we don't, we'll never get those modules again without
# some potentially dangerous manipulation of sys.modules and
# possibly other state. get_models() will default to ignoring
# anything in that list if the app label isn't present in
# loading.cache.app_labels, which we'll remove, so the model
# will appear as "uninstalled."
app_label = app_name.rpartition('.')[2]
loading.cache.app_labels.pop(app_label, None)
for module in model_modules:
loading.cache.app_store.pop(module, None)
# Force get_models() to recompute models for lookups, so that
# now-unregistered models aren't returned.
loading.cache._get_models_cache.clear()
def _entrypoint_iterator(self):
return pkg_resources.iter_entry_points(self.key)
def _bump_sync_gen(self):
"""Bumps the synchronization generation value.
If there's an existing synchronization generation in cache,
increment it. Otherwise, start fresh with a new one.
This will also set ``settings.AJAX_SERIAL``, which will guarantee any
cached objects that depends on templates and use this serial number
will be invalidated, allowing TemplateHooks and other hooks
to be re-run.
"""
# If we're in the middle of loading extension state, perhaps due to
# the sync number being bumped by another process, this flag will be
# sent in order to block any further attempts at bumping the number.
# Failure to do this can result in a loop where the number gets
# bumped by every process/thread reacting to another process/thread
# bumping the number, resulting in massive slowdown and errors.
if self._block_sync_gen:
return
self._gen_sync.mark_updated()
settings.AJAX_SERIAL = self._gen_sync.sync_gen
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
if django_evolution is None:
# Django Evolution isn't installed. Extensions with evolutions
# are not supported.
return
try:
stream = StringIO()
call_command('evolve',
verbosity=0,
interactive=False,
execute=True,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logger.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logger.exception('Error evolving extension models: %s: %s',
e, output)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _recalculate_middleware(self):
"""Recalculates the list of middleware."""
self.middleware = []
done = set()
for e in self.get_enabled_extensions():
self.middleware.extend(self._get_extension_middleware(e, done))
def _get_extension_middleware(self, extension, done):
"""Returns a list of middleware for 'extension' and its dependencies.
This is a recursive utility function initially called by
_recalculate_middleware() that ensures that middleware for all
dependencies are inserted before that of the given extension. It
also ensures that each extension's middleware is inserted only once.
"""
middleware = []
if extension in done:
return middleware
done.add(extension)
for req in extension.requirements:
e = self.get_enabled_extension(req)
if e:
middleware.extend(self._get_extension_middleware(e, done))
middleware.extend(extension.middleware_instances)
return middleware
def setUp(self):
super(GenerationSynchronizerTests, self).setUp()
self.gen_sync = GenerationSynchronizer('test-synchronizer')
class IntegrationManager(object):
"""Manages integrations with third-party services.
The manager keeps track of the integrations registered by extensions
or other components of an application, providing the ability to
register new ones, unregister existing ones, and list any that are
currently enabled.
It also manages the lookups of configurations for integrations, taking
care to cache the lookups for any integrations and invalidate them when
a configuration has been updated.
Attributes:
config_model (type):
The model used to store configuration data. This is a subclass of
:py:class:`djblets.integrations.models.BaseIntegrationConfig`.
enabled (bool):
The integration manager is enabled and can be used for registering,
unregistering, and otherwise using integrations. If this is
``False``, then :py:meth:`shutdown` has been called, and it should
be assumed that no integrations are registered or need to be
unregistered.
"""
def __init__(self, config_model):
"""Initialize the integration manager.
Args:
config_model (type):
The model used to store configuration data. This must be a
subclass of
:py:class:`djblets.integrations.models.BaseIntegrationConfig`.
"""
# Check that the Django environment is set up for integrations to
# properly function.
if 'djblets.integrations' not in settings.INSTALLED_APPS:
raise ImproperlyConfigured(
'IntegrationManager requires djblets.integrations to be '
'listed in settings.INSTALLED_APPS.'
)
middleware = 'djblets.integrations.middleware.IntegrationsMiddleware'
if middleware not in settings.MIDDLEWARE_CLASSES:
raise ImproperlyConfigured(
'IntegrationManager requires %s to be listed in '
'settings.MIDDLEWARE_CLASSES'
% middleware
)
self.config_model = config_model
key = ('integrationmgr:%s.%s'
% (self.__class__.__module__, self.__class__.__name__))
self.enabled = True
self._integration_classes = {}
self._integration_configs = {}
self._integration_instances = {}
self._lock = threading.Lock()
self._needs_recalc = False
self._gen_sync = GenerationSynchronizer('%s:gen' % key)
instance_id = id(self)
_integration_managers[instance_id] = self
# Listen for any config model saves/deletes, so we can mark whether
# a reload of state is needed.
dispatch_uid = '%s:%s' % (key, id(self))
post_delete.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
post_save.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
def shutdown(self):
"""Shut down the integrations on this integration manager.
This should be called when the integration manager and integrations
will no longer be used. It will shut down every integration and
unregister all integrations.
"""
for integration in self.get_integrations():
integration.disable_integration()
self._integration_classes = {}
self._integration_configs = {}
self._integration_instances = {}
self._needs_recalc = False
try:
del _integration_managers[id(self)]
except KeyError:
pass
self.enabled = False
def get_integration_classes(self):
"""Return all the integration classes that have been registered.
This is not sorted in any particular order. It is up to the caller
to determine the correct sorting order.
Yields:
type:
The registered integration classes.
"""
return six.itervalues(self._integration_classes)
def get_integration(self, integration_id):
"""Return an integration instance for a given ID.
Args:
integration_id (unicode):
The integration ID that was registered.
Returns:
djblets.integrations.integration.Integration:
The integration instance.
Raises:
djblets.integrations.errors.IntegrationNotRegisteredError:
The integration class provided wasn't registered.
"""
try:
return self._integration_instances[integration_id]
except KeyError:
raise IntegrationNotRegisteredError(integration_id)
def get_integrations(self):
"""Return all the integration instances.
This is not sorted in any particular order. It is up to the caller
to determine the correct sorting order.
Yields:
djblets.integrations.integration.Integration:
The integration instances.
"""
return six.itervalues(self._integration_instances)
def get_integration_configs(self, integration_cls=None, **filter_kwargs):
"""Return a list of saved integration configurations.
By default, all configurations will be returned for all integrations,
including configurations that are disabled. This can be filtered
down by specifying an integration class and/or by filtering by fields
in the model through keyword arguments.
Each set of results for a unique combination of integration class and
filter arguments will be cached locally, to speed up further lookups.
This cache can be flushed using :py:meth:`clear_configs_cache` or
:py:meth:`clear_all_configs_cache`, and will be automatically cleared
when cnofigurations are added, updated, or removed.
Args:
integration_cls (type, optional):
The integration class to filter by.
**filter_kwargs (dict, optional):
Keyword arguments to filter by. Each must match a field and
value on the model.
Returns:
list of djblets.integrations.models.BaseIntegrationConfig:
A list of saved integration configurations matching the query.
"""
key = self._make_config_filter_cache_key(integration_cls,
**filter_kwargs)
try:
configs = self._integration_configs[key]
except KeyError:
queryset = self.config_model.objects.all()
if integration_cls:
queryset = queryset.filter(
integration_id=integration_cls.integration_id)
if filter_kwargs:
queryset = queryset.filter(**filter_kwargs)
configs = list(queryset)
self._integration_configs[key] = configs
return configs
def clear_configs_cache(self, integration_cls=None, **filter_kwargs):
"""Clear the configuration cache matching the given filters.
This is used to clear a subset of the configs cache, matching the exact
query arguments passed to a previous call to
:py:meth:`get_integration_configs`.
To clear the entire cache, use :py:meth:`clear_all_configs_cache`.
Args:
integration_cls (type, optional):
The integration class for the filter.
**filter_kwargs (dict, optional):
Keyword arguments for the filter.
"""
key = self._make_config_filter_cache_key(integration_cls,
**filter_kwargs)
try:
del self._integration_configs[key]
except KeyError:
pass
def clear_all_configs_cache(self):
"""Clear the entire configuration cache.
This will force all future lookups to re-query the database. To
clear only a subset of the cache, use :py:meth:`clear_configs_cache`.
"""
self._integration_configs = {}
def is_expired(self):
"""Return whether the integration manager has expired state.
Returns:
bool:
``True`` if there's either expired configuration state or
integrations that need their enabled state recalculated.
"""
return self._needs_recalc or self._gen_sync.is_expired()
def check_expired(self):
"""Check for and handle expired integration state.
If the configurations of one or more integrations have been updated
by another process, or there are new integrations registered that
may need to be enabled, this method will reset the cache state and
re-calculate the integrations to enable/disable.
"""
if self.is_expired():
# We're going to check the expiration, and then only lock if it's
# expired. Following that, we'll check again.
#
# We do this in order to prevent locking unnecessarily, which could
# impact performance or cause a problem if a thread is stuck.
#
# We're checking the expiration twice to prevent every blocked
# thread from making its own attempt to reload the integrations
# state the first thread holding the lock finishes.
with self._lock:
# Check again, since another thread may have already
# reloaded.
if self.is_expired():
self._gen_sync.refresh()
self.clear_all_configs_cache()
self._recalc_enabled_integrations()
def register_integration_class(self, integration_cls):
"""Register a class for an integration.
This will instantiate the integration and make it available for new
configurations.
Args:
integration_cls (type):
The integration class to register.
Returns:
djblets.integrations.integration.Integration:
The new instance of the registered integration class.
Raises:
djblets.integrations.errors.IntegrationAlreadyRegisteredError:
The integration class was already registered.
djblets.integrations.errors.IntegrationConstructionError:
Error initializing an instance of the integration. The
integration will not be registered.
"""
if not integration_cls.integration_id:
# A pre-existing integration ID doesn't exist, so create one
# based on the class path.
integration_cls.integration_id = '%s.%s' % (
integration_cls.__module__,
integration_cls.__name__)
integration_id = integration_cls.integration_id
if integration_id in self._integration_classes:
raise IntegrationAlreadyRegisteredError(integration_id)
try:
# We're going to instantiate the integration, but we won't
# initialize it until later when we know that there are
# configurations available.
integration = integration_cls(self)
self._integration_instances[integration_id] = integration
self._integration_classes[integration_id] = integration_cls
except Exception as e:
# This should never happen, unless the subclass overrode
# __init__.
raise IntegrationRegistrationError(
'Unexpected error when constructing integration %s: %s'
% (integration_cls.__name__, e))
# Flag that we need to recalculate the list of enabled integrations.
# The next time a request is served, the middleware will perform the
# recalculation. We do it this way instead of during registration in
# order to cut back on the number of queries needed.
self._needs_recalc = True
return integration
def unregister_integration_class(self, integration_cls):
"""Unregister a class for an integration.
The integration instance will be shut down, and the integration will
no longer be made available for any further configuration.
If there is an error shutting down the integration, the output will
be logged, but no error will be returned.
Args:
integration_cls (type):
The integration class to unregister.
Raises:
djblets.integrations.errors.IntegrationNotRegisteredError:
The integration class was not previously registered.
"""
integration_id = integration_cls.integration_id
if (not integration_id or
integration_id not in self._integration_classes):
raise IntegrationNotRegisteredError(integration_id)
integration = self.get_integration(integration_id)
if integration.enabled:
try:
integration.disable_integration()
except Exception as e:
logging.exception(
'Unexpected error when shutting down integration %r: %s',
integration_cls, e)
del self._integration_classes[integration_id]
del self._integration_instances[integration_id]
def _recalc_enabled_integrations(self):
"""Recalculate the enabled states of all integrations.
The list of enabled configurations for integrations will be queried
from the database. Based on this, the desired enabled state of each
integration will be calculated. Those that are disabled but have
enabled configurations will be enabled, and those that are enabled
but no longer have enabled configurations will be disabled.
This allows us to keep memory requirements and event handling at a
minimum for any integrations not currently in use.
"""
enabled_integration_ids = set(
self.config_model.objects
.filter(enabled=True)
.distinct()
.values_list('integration_id', flat=True)
)
for integration in self.get_integrations():
should_enable = (integration.integration_id in
enabled_integration_ids)
if should_enable != integration.enabled:
if should_enable:
integration.enable_integration()
else:
integration.disable_integration()
self._needs_recalc = False
def _make_config_filter_cache_key(self, integration_cls, **filter_kwargs):
"""Return a cache key for a config query filter.
Args:
integration_cls (type):
The integration class used for the query.
**filter_kwargs (dict):
The filter keyword arguments used for the query.
Returns:
unicode:
The resulting cache key.
"""
if integration_cls:
return '%s:%s' % (integration_cls.integration_id, filter_kwargs)
else:
return '*:%s' % (filter_kwargs,)
def _on_config_changes(self, **kwargs):
"""Handler for when configuration state changes.
This will force the list of integrations to recalculate on this
process and others when a configuration is created, saved, or deleted.
Args:
**kwargs (dict):
Keyword arguments passed to the signal.
"""
self._needs_recalc = True
self._gen_sync.mark_updated()
class SiteConfiguration(models.Model):
"""
Configuration data for a site. The version and all persistent settings
are stored here.
The usual way to retrieve a SiteConfiguration is to use
:py:meth:`get_current`.
"""
site = models.ForeignKey(Site, related_name="config")
version = models.CharField(max_length=20)
#: A JSON dictionary field of settings stored for a site.
settings = JSONField()
objects = SiteConfigurationManager()
def __init__(self, *args, **kwargs):
models.Model.__init__(self, *args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer(
'%s:siteconfig:%s:generation' % (self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def get(self, key, default=None):
"""
Retrieves a setting. If the setting is not found, the default value
will be returned. This is represented by the default parameter, if
passed in, or a global default if set.
"""
if default is None and self.id in _DEFAULTS:
default = _DEFAULTS[self.id].get(key, None)
return self.settings.get(key, default)
def set(self, key, value):
"""
Sets a setting. The key should be a string, but the value can be
any native Python object.
"""
self.settings[key] = value
def add_defaults(self, defaults_dict):
"""Add a dictionary of defaults.
These defaults will be used when calling :py:meth:`get`, if that
setting wasn't saved in the database.
"""
if self.id not in _DEFAULTS:
_DEFAULTS[self.id] = {}
_DEFAULTS[self.id].update(defaults_dict)
def add_default(self, key, default_value):
"""
Adds a single default setting.
"""
self.add_defaults({key: default_value})
def get_defaults(self):
"""
Returns all default settings registered with this SiteConfiguration.
"""
if self.id not in _DEFAULTS:
_DEFAULTS[self.id] = {}
return _DEFAULTS[self.id]
def is_expired(self):
"""Return whether or not this SiteConfiguration is expired.
If the configuration is expired, it will need to be reloaded before
accessing any settings.
Returns:
bool:
Whether or not the current state is expired.
"""
return self._gen_sync.is_expired()
def save(self, clear_caches=True, **kwargs):
self._gen_sync.mark_updated()
if clear_caches:
# The cached siteconfig might be stale now. We'll want a refresh.
# Also refresh the Site cache, since callers may get this from
# Site.config.
SiteConfiguration.objects.clear_cache()
Site.objects.clear_cache()
super(SiteConfiguration, self).save(**kwargs)
def __str__(self):
return "%s (version %s)" % (six.text_type(self.site), self.version)
class Meta:
# Djblets 0.9+ sets an app label of "djblets_siteconfig" on
# Django 1.7+, which would affect the table name. We need to retain
# the old name for backwards-compatibility.
db_table = 'siteconfig_siteconfiguration'
class SiteConfiguration(models.Model):
"""
Configuration data for a site. The version and all persistent settings
are stored here.
The usual way to retrieve a SiteConfiguration is to use
:py:meth:`get_current`.
"""
site = models.ForeignKey(Site, related_name="config")
version = models.CharField(max_length=20)
#: A JSON dictionary field of settings stored for a site.
settings = JSONField()
objects = SiteConfigurationManager()
def __init__(self, *args, **kwargs):
models.Model.__init__(self, *args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer('%s:siteconfig:%s:generation' %
(self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def get(self, key, default=None):
"""
Retrieves a setting. If the setting is not found, the default value
will be returned. This is represented by the default parameter, if
passed in, or a global default if set.
"""
if default is None and self.id in _DEFAULTS:
default = _DEFAULTS[self.id].get(key, None)
return self.settings.get(key, default)
def set(self, key, value):
"""
Sets a setting. The key should be a string, but the value can be
any native Python object.
"""
self.settings[key] = value
def add_defaults(self, defaults_dict):
"""Add a dictionary of defaults.
These defaults will be used when calling :py:meth:`get`, if that
setting wasn't saved in the database.
"""
if self.id not in _DEFAULTS:
_DEFAULTS[self.id] = {}
_DEFAULTS[self.id].update(defaults_dict)
def add_default(self, key, default_value):
"""
Adds a single default setting.
"""
self.add_defaults({key: default_value})
def get_defaults(self):
"""
Returns all default settings registered with this SiteConfiguration.
"""
if self.id not in _DEFAULTS:
_DEFAULTS[self.id] = {}
return _DEFAULTS[self.id]
def is_expired(self):
"""Return whether or not this SiteConfiguration is expired.
If the configuration is expired, it will need to be reloaded before
accessing any settings.
Returns:
bool:
Whether or not the current state is expired.
"""
return self._gen_sync.is_expired()
def save(self, clear_caches=True, **kwargs):
self._gen_sync.mark_updated()
if clear_caches:
# The cached siteconfig might be stale now. We'll want a refresh.
# Also refresh the Site cache, since callers may get this from
# Site.config.
SiteConfiguration.objects.clear_cache()
Site.objects.clear_cache()
super(SiteConfiguration, self).save(**kwargs)
def __str__(self):
return "%s (version %s)" % (six.text_type(self.site), self.version)
class Meta:
# Djblets 0.9+ sets an app label of "djblets_siteconfig" on
# Django 1.7+, which would affect the table name. We need to retain
# the old name for backwards-compatibility.
db_table = 'siteconfig_siteconfiguration'
class ExtensionManager(object):
"""A manager for all extensions.
ExtensionManager manages the extensions available to a project. It can
scan for new extensions, enable or disable them, determine dependencies,
install into the database, and uninstall.
An installed extension is one that has been installed by a Python package
on the system.
A registered extension is one that has been installed and information then
placed in the database. This happens automatically after scanning for
an installed extension. The registration data stores whether or not it's
enabled, and stores various pieces of information on the extension.
An enabled extension is one that is actively enabled and hooked into the
project.
Each project should have one ExtensionManager.
"""
VERSION_SETTINGS_KEY = '_extension_installed_version'
def __init__(self, key):
self.key = key
self.pkg_resources = None
self._extension_classes = {}
self._extension_instances = {}
self._load_errors = {}
# State synchronization
self._gen_sync = GenerationSynchronizer('extensionmgr:%s:gen' % key)
self._load_lock = threading.Lock()
self._shutdown_lock = threading.Lock()
self._block_sync_gen = False
self.dynamic_urls = DynamicURLResolver()
# Extension middleware instances, ordered by dependencies.
self.middleware = []
# Wrap the INSTALLED_APPS and TEMPLATE_CONTEXT_PROCESSORS settings
# to allow for ref-counted add/remove operations.
self._installed_apps_setting = SettingListWrapper(
'INSTALLED_APPS', 'installed app')
self._context_processors_setting = SettingListWrapper(
'TEMPLATE_CONTEXT_PROCESSORS', 'context processor')
instance_id = id(self)
_extension_managers[instance_id] = self
def get_url_patterns(self):
"""Returns the URL patterns for the Extension Manager.
This should be included in the root urlpatterns for the site.
"""
return patterns('', self.dynamic_urls)
def is_expired(self):
"""Returns whether or not the extension state is possibly expired.
Extension state covers the lists of extensions and each extension's
configuration. It can expire if the state synchronization value
falls out of cache or is changed.
Each ExtensionManager has its own state synchronization cache key.
Returns:
bool:
Whether the state has expired.
"""
return self._gen_sync.is_expired()
def clear_sync_cache(self):
"""Clear the extension synchronization state.
This will force every process to reload the extension list and
settings.
"""
self._gen_sync.clear()
def get_absolute_url(self):
return reverse("djblets.extensions.views.extension_list")
def get_can_disable_extension(self, registered_extension):
extension_id = registered_extension.class_name
return (registered_extension.extension_class is not None
and (self.get_enabled_extension(extension_id) is not None
or extension_id in self._load_errors))
def get_can_enable_extension(self, registered_extension):
return (registered_extension.extension_class is not None
and self.get_enabled_extension(
registered_extension.class_name) is None)
def get_enabled_extension(self, extension_id):
"""Returns an enabled extension with the given ID."""
if extension_id in self._extension_instances:
return self._extension_instances[extension_id]
return None
def get_enabled_extensions(self):
"""Returns the list of all enabled extensions."""
return list(self._extension_instances.values())
def get_installed_extensions(self):
"""Returns the list of all installed extensions."""
return list(self._extension_classes.values())
def get_installed_extension(self, extension_id):
"""Returns the installed extension with the given ID."""
if extension_id not in self._extension_classes:
raise InvalidExtensionError(extension_id)
return self._extension_classes[extension_id]
def get_dependent_extensions(self, dependency_extension_id):
"""Returns a list of all extensions required by an extension."""
if dependency_extension_id not in self._extension_instances:
raise InvalidExtensionError(dependency_extension_id)
dependency = self.get_installed_extension(dependency_extension_id)
result = []
for extension_id, extension in six.iteritems(self._extension_classes):
if extension_id == dependency_extension_id:
continue
for ext_requirement in extension.info.requirements:
if ext_requirement == dependency:
result.append(extension_id)
return result
def enable_extension(self, extension_id):
"""Enables an extension.
Enabling an extension will install any data files the extension
may need, any tables in the database, perform any necessary
database migrations, and then will start up the extension.
"""
if extension_id in self._extension_instances:
# It's already enabled.
return
if extension_id not in self._extension_classes:
if extension_id in self._load_errors:
raise EnablingExtensionError(
_('There was an error loading this extension'),
self._load_errors[extension_id],
needs_reload=True)
raise InvalidExtensionError(extension_id)
ext_class = self._extension_classes[extension_id]
# Enable extension dependencies
for requirement_id in ext_class.requirements:
self.enable_extension(requirement_id)
extension = self._init_extension(ext_class)
ext_class.registration.enabled = True
ext_class.registration.save()
self._clear_template_cache()
self._bump_sync_gen()
self._recalculate_middleware()
return extension
def disable_extension(self, extension_id):
"""Disables an extension.
Disabling an extension will remove any data files the extension
installed and then shut down the extension and all of its hooks.
It will not delete any data from the database.
"""
has_load_error = extension_id in self._load_errors
if not has_load_error:
if extension_id not in self._extension_instances:
# It's not enabled.
return
if extension_id not in self._extension_classes:
raise InvalidExtensionError(extension_id)
extension = self._extension_instances[extension_id]
for dependent_id in self.get_dependent_extensions(extension_id):
self.disable_extension(dependent_id)
self._uninstall_extension(extension)
self._uninit_extension(extension)
self._unregister_static_bundles(extension)
registration = extension.registration
else:
del self._load_errors[extension_id]
if extension_id in self._extension_classes:
# The class was loadable, so it just couldn't be instantiated.
# Update the registration on the class.
ext_class = self._extension_classes[extension_id]
registration = ext_class.registration
else:
registration = RegisteredExtension.objects.get(
class_name=extension_id)
registration.enabled = False
registration.save(update_fields=['enabled'])
self._clear_template_cache()
self._bump_sync_gen()
self._recalculate_middleware()
def install_extension(self, install_url, package_name):
"""Install an extension from a remote source.
Installs an extension from a remote URL containing the
extension egg. Installation may fail if a malformed install_url
or package_name is passed, which will cause an InstallExtensionError
exception to be raised. It is also assumed that the extension is not
already installed.
"""
try:
easy_install.main(["-U", install_url])
# Update the entry points.
dist = pkg_resources.get_distribution(package_name)
dist.activate()
pkg_resources.working_set.add(dist)
except pkg_resources.DistributionNotFound:
raise InstallExtensionError(_("Invalid package name."))
except SystemError:
raise InstallExtensionError(
_('Installation failed (probably malformed URL).'))
# Refresh the extension manager.
self.load(True)
def load(self, full_reload=False):
"""
Loads all known extensions, initializing any that are recorded as
being enabled.
If this is called a second time, it will refresh the list of
extensions, adding new ones and removing deleted ones.
If full_reload is passed, all state is cleared and we reload all
extensions and state from scratch.
"""
with self._load_lock:
self._block_sync_gen = True
self._load_extensions(full_reload)
self._block_sync_gen = False
def shutdown(self):
"""Shut down the extension manager and all of its extensions."""
with self._shutdown_lock:
self._clear_extensions()
def _load_extensions(self, full_reload=False):
if full_reload:
# We're reloading everything, so nuke all the cached copies.
self._clear_extensions()
self._clear_template_cache()
self._load_errors = {}
# Preload all the RegisteredExtension objects
registered_extensions = {}
for registered_ext in RegisteredExtension.objects.all():
registered_extensions[registered_ext.class_name] = registered_ext
found_extensions = {}
found_registrations = {}
registrations_to_fetch = []
find_registrations = False
extensions_changed = False
for entrypoint in self._entrypoint_iterator():
registered_ext = None
try:
ext_class = entrypoint.load()
except Exception as e:
logging.exception("Error loading extension %s: %s" %
(entrypoint.name, e))
extension_id = '%s.%s' % (entrypoint.module_name, '.'.join(
entrypoint.attrs))
self._store_load_error(extension_id, e)
continue
# A class's extension ID is its class name. We want to
# make this easier for users to access by giving it an 'id'
# variable, which will be accessible both on the class and on
# instances.
class_name = ext_class.id = "%s.%s" % (ext_class.__module__,
ext_class.__name__)
self._extension_classes[class_name] = ext_class
found_extensions[class_name] = ext_class
# Don't override the info if we've previously loaded this
# class.
if not getattr(ext_class, 'info', None):
ext_class.info = ExtensionInfo.create_from_entrypoint(
entrypoint, ext_class)
registered_ext = registered_extensions.get(class_name)
if registered_ext:
found_registrations[class_name] = registered_ext
if not hasattr(ext_class, 'registration'):
find_registrations = True
else:
registrations_to_fetch.append(
(class_name, entrypoint.dist.project_name))
find_registrations = True
if find_registrations:
if registrations_to_fetch:
stored_registrations = list(
RegisteredExtension.objects.filter(
class_name__in=registrations_to_fetch))
# Go through the list of registrations found in the database
# and mark them as found for later processing.
for registered_ext in stored_registrations:
class_name = registered_ext.class_name
found_registrations[class_name] = registered_ext
# Go through each registration we still need and couldn't find,
# and create an entry in the database. These are going to be
# newly discovered extensions.
for class_name, ext_name in registrations_to_fetch:
if class_name not in found_registrations:
try:
registered_ext = RegisteredExtension.objects.create(
class_name=class_name, name=ext_name)
except IntegrityError:
# An entry was created since we last looked up
# anything. Fetch it from the database.
registered_ext = RegisteredExtension.objects.get(
class_name=class_name)
found_registrations[class_name] = registered_ext
# Now we have all the RegisteredExtension instances. Go through
# and initialize each of them.
for class_name, registered_ext in six.iteritems(found_registrations):
ext_class = found_extensions[class_name]
ext_class.registration = registered_ext
if (ext_class.registration.enabled
and ext_class.id not in self._extension_instances):
try:
self._init_extension(ext_class)
except EnablingExtensionError:
# When in debug mode, we want this error to be noticed.
# However, in production, it shouldn't break the whole
# server, so continue on.
if not settings.DEBUG:
continue
extensions_changed = True
# At this point, if we're reloading, it's possible that the user
# has removed some extensions. Go through and remove any that we
# can no longer find.
#
# While we're at it, since we're at a point where we've seen all
# extensions, we can set the ExtensionInfo.requirements for
# each extension
for class_name, ext_class in six.iteritems(self._extension_classes):
if class_name not in found_extensions:
if class_name in self._extension_instances:
self.disable_extension(class_name)
del self._extension_classes[class_name]
extensions_changed = True
else:
ext_class.info.requirements = \
[self.get_installed_extension(requirement_id)
for requirement_id in ext_class.requirements]
# Add the sync generation if it doesn't already exist.
self._gen_sync.refresh()
settings.AJAX_SERIAL = self._gen_sync.sync_gen
if extensions_changed:
self._recalculate_middleware()
def _clear_extensions(self):
"""Clear the entire list of known extensions.
This will bring the ExtensionManager back to the state where
it doesn't yet know about any extensions, requiring a re-load.
"""
for extension in self.get_enabled_extensions():
# Make sure this is actually an enabled extension, and not one
# that's already been shut down by another instance of this
# ExtensionManager (which should only happen in tests):
if hasattr(extension.__class__, 'info'):
self._uninit_extension(extension)
for extension_class in self.get_installed_extensions():
if hasattr(extension_class, 'info'):
delattr(extension_class, 'info')
if hasattr(extension_class, 'registration'):
delattr(extension_class, 'registration')
self._extension_classes = {}
self._extension_instances = {}
def _clear_template_cache(self):
"""Clears the Django template caches."""
if template_source_loaders:
# We're running in Django <=1.7.
template_loaders = template_source_loaders
elif template_engines:
template_loaders = []
for engine in template_engines.all():
template_loaders += engine.engine.template_loaders
else:
# It's valid for there to not be any loaders.
template_loaders = []
for template_loader in template_loaders:
if hasattr(template_loader, 'reset'):
template_loader.reset()
def _init_extension(self, ext_class):
"""Initializes an extension.
This will register the extension, install any URLs that it may need,
and make it available in Django's list of apps. It will then notify
that the extension has been initialized.
"""
extension_id = ext_class.id
assert extension_id not in self._extension_instances
try:
extension = ext_class(extension_manager=self)
except Exception as e:
logging.error('Unable to initialize extension %s: %s' %
(ext_class, e),
exc_info=1)
error_details = self._store_load_error(extension_id, e)
raise EnablingExtensionError(
_('Error initializing extension: %s') % e, error_details)
if extension_id in self._load_errors:
del self._load_errors[extension_id]
self._extension_instances[extension_id] = extension
if extension.has_admin_site:
self._init_admin_site(extension)
# Installing the urls must occur after _init_admin_site(). The urls
# for the admin site will not be generated until it is called.
self._install_admin_urls(extension)
self._register_static_bundles(extension)
extension.info.installed = extension.registration.installed
extension.info.enabled = True
self._add_to_installed_apps(extension)
self._context_processors_setting.add_list(extension.context_processors)
self._reset_templatetags_cache()
ext_class.instance = extension
try:
self.install_extension_media(ext_class)
except InstallExtensionError as e:
raise EnablingExtensionError(e.message, e.load_error)
extension_initialized.send(self, ext_class=extension)
return extension
def _uninit_extension(self, extension):
"""Uninitializes the extension.
This will shut down the extension, remove any URLs, remove it from
Django's list of apps, and send a signal saying the extension was
shut down.
"""
extension.shutdown()
if hasattr(extension, "admin_urlpatterns"):
self.dynamic_urls.remove_patterns(extension.admin_urlpatterns)
if hasattr(extension, "admin_site_urlpatterns"):
self.dynamic_urls.remove_patterns(extension.admin_site_urlpatterns)
if hasattr(extension, 'admin_site'):
del extension.admin_site
self._context_processors_setting.remove_list(
extension.context_processors)
self._remove_from_installed_apps(extension)
self._reset_templatetags_cache()
extension.info.enabled = False
extension_uninitialized.send(self, ext_class=extension)
del self._extension_instances[extension.id]
extension.__class__.instance = None
def _store_load_error(self, extension_id, err):
"""Stores and returns a load error for the extension ID."""
error_details = '%s\n\n%s' % (err, traceback.format_exc())
self._load_errors[extension_id] = error_details
return error_details
def _reset_templatetags_cache(self):
"""Clears the Django templatetags_modules cache."""
# We'll import templatetags_modules here because
# we want the most recent copy of templatetags_modules
from django.template.base import get_templatetags_modules
# Wipe out the contents.
if hasattr(get_templatetags_modules, 'cache_clear'):
# Django >= 1.7
get_templatetags_modules.cache_clear()
else:
# Django < 1.7
from django.template.base import templatetags_modules
del (templatetags_modules[:])
# And reload the cache
get_templatetags_modules()
def install_extension_media(self, ext_class, force=False):
"""Installs extension static media.
This method is a wrapper around _install_extension_media_internal to
check whether we actually need to install extension media, and avoid
contention among multiple threads/processes when doing so.
We need to install extension media if it hasn't been installed yet,
or if the version of the extension media that we installed is different
from the current version of the extension.
"""
lockfile = os.path.join(tempfile.gettempdir(), ext_class.id + '.lock')
extension = ext_class.instance
cur_version = ext_class.info.version
# We only want to fetch the existing version information if the
# extension is already installed. We remove this key when
# disabling an extension, so if it were there, it was either
# copy/pasted, or something went wrong. Either way, we wouldn't
# be able to trust it.
if force:
logging.debug('Forcing installation fo extension media for %s',
ext_class.info)
old_version = None
else:
if ext_class.registration.installed:
old_version = extension.settings.get(self.VERSION_SETTINGS_KEY)
else:
old_version = None
if old_version == cur_version:
# Nothing to do
return
if not old_version:
logging.debug('Installing extension media for %s',
ext_class.info)
else:
logging.debug(
'Reinstalling extension media for %s because '
'version changed from %s', ext_class.info, old_version)
while old_version != cur_version:
with open(lockfile, 'w') as f:
try:
locks.lock(f, locks.LOCK_EX | locks.LOCK_NB)
except IOError as e:
if e.errno in (errno.EAGAIN, errno.EACCES, errno.EINTR):
# Sleep for one second, then try again
time.sleep(1)
extension.settings.load()
old_version = extension.settings.get(
self.VERSION_SETTINGS_KEY)
continue
else:
raise e
self._install_extension_media_internal(ext_class)
extension.settings.set(self.VERSION_SETTINGS_KEY, cur_version)
extension.settings.save()
old_version = cur_version
locks.unlock(f)
try:
os.unlink(lockfile)
except OSError as e:
# A "No such file or directory" (ENOENT) is most likely due to
# another thread removing the lock file before this thread could.
# It's safe to ignore. We want to handle all others, though.
if e.errno != errno.ENOENT:
logging.error(
"Failed to unlock media lock file '%s' for "
"extension '%s': %s",
lockfile,
ext_class.info,
e,
exc_info=1)
def _install_extension_media_internal(self, ext_class):
"""Installs extension data.
Performs any installation necessary for an extension.
If the extension has a legacy htdocs/ directory for static media
files, they will be installed into MEDIA_ROOT/ext/, and a warning
will be logged.
If the extension has a modern static/ directory, they will be
installed into STATIC_ROOT/ext/.
"""
ext_htdocs_path = ext_class.info.installed_htdocs_path
ext_htdocs_path_exists = os.path.exists(ext_htdocs_path)
if ext_htdocs_path_exists:
# First, get rid of the old htdocs contents, so we can start
# fresh.
shutil.rmtree(ext_htdocs_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'htdocs'):
# This is an older extension that doesn't use the static file
# support. Log a deprecation notice and then install the files.
logging.warning('The %s extension uses the deprecated "htdocs" '
'directory for static files. It should be updated '
'to use a "static" directory instead.' %
ext_class.info.name)
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__, 'htdocs')
shutil.copytree(extracted_path, ext_htdocs_path, symlinks=True)
# We only want to install static media on a non-DEBUG install.
# Otherwise, we run the risk of creating a new 'static' directory and
# causing Django to look up all static files (not just from
# extensions) from there instead of from their source locations.
if not settings.DEBUG:
ext_static_path = ext_class.info.installed_static_path
ext_static_path_exists = os.path.exists(ext_static_path)
if ext_static_path_exists:
# Also get rid of the old static contents.
shutil.rmtree(ext_static_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'static'):
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__,
'static')
shutil.copytree(extracted_path, ext_static_path, symlinks=True)
# Mark the extension as installed
ext_class.registration.installed = True
ext_class.registration.save()
# Now let's build any tables that this extension might need
self._add_to_installed_apps(ext_class)
# Call syncdb to create the new tables
loading.cache.loaded = False
call_command('syncdb', verbosity=0, interactive=False)
# Run evolve to do any table modification.
self._migrate_extension_models(ext_class)
# Remove this again, since we only needed it for syncdb and
# evolve. _init_extension will add it again later in
# the install.
self._remove_from_installed_apps(ext_class)
# Mark the extension as installed
ext_class.registration.installed = True
ext_class.registration.save()
def _uninstall_extension(self, extension):
"""Uninstalls extension data.
Performs any uninstallation necessary for an extension.
This will uninstall the contents of MEDIA_ROOT/ext/ and
STATIC_ROOT/ext/.
"""
extension.settings.set(self.VERSION_SETTINGS_KEY, None)
extension.settings.save()
extension.registration.installed = False
extension.registration.save()
for path in (extension.info.installed_htdocs_path,
extension.info.installed_static_path):
if os.path.exists(path):
shutil.rmtree(path, ignore_errors=True)
def _install_admin_urls(self, extension):
"""Installs administration URLs.
This provides URLs for configuring an extension, plus any additional
admin urlpatterns that the extension provides.
"""
prefix = self.get_absolute_url()
if hasattr(settings, 'SITE_ROOT'):
prefix = prefix[len(settings.SITE_ROOT):]
# Note that we're adding to the resolve list on the root of the
# install, and prefixing it with the admin extensions path.
# The reason we're not just making this a child of our extensions
# urlconf is that everything in there gets passed an
# extension_manager variable, and we don't want to force extensions
# to handle this.
if extension.is_configurable:
urlconf = extension.admin_urlconf
if hasattr(urlconf, "urlpatterns"):
extension.admin_urlpatterns = patterns(
'', (r'^%s%s/config/' %
(prefix, extension.id), include(urlconf.__name__)))
self.dynamic_urls.add_patterns(extension.admin_urlpatterns)
if getattr(extension, 'admin_site', None):
extension.admin_site_urlpatterns = patterns(
'',
(r'^%s%s/db/' %
(prefix, extension.id), include(extension.admin_site.urls)))
self.dynamic_urls.add_patterns(extension.admin_site_urlpatterns)
def _register_static_bundles(self, extension):
"""Registers the extension's static bundles with Pipeline.
Each static bundle will appear as an entry in Pipeline. The
bundle name and filenames will be changed to include the extension
ID for the static file lookups.
"""
def _add_prefix(filename):
return 'ext/%s/%s' % (extension.id, filename)
def _add_bundles(pipeline_bundles, extension_bundles, default_dir,
ext):
for name, bundle in six.iteritems(extension_bundles):
new_bundle = bundle.copy()
new_bundle['source_filenames'] = [
_add_prefix(filename)
for filename in bundle.get('source_filenames', [])
]
new_bundle['output_filename'] = _add_prefix(
bundle.get('output_filename',
'%s/%s.min%s' % (default_dir, name, ext)))
pipeline_bundles[extension.get_bundle_id(name)] = new_bundle
_add_bundles(pipeline_settings.STYLESHEETS, extension.css_bundles,
'css', '.css')
_add_bundles(pipeline_settings.JAVASCRIPT, extension.js_bundles, 'js',
'.js')
def _unregister_static_bundles(self, extension):
"""Unregisters the extension's static bundles from Pipeline.
Every static bundle previously registered will be removed.
"""
def _remove_bundles(pipeline_bundles, extension_bundles):
for name, bundle in six.iteritems(extension_bundles):
try:
del pipeline_bundles[extension.get_bundle_id(name)]
except KeyError:
pass
if hasattr(settings, 'PIPELINE'):
_remove_bundles(pipeline_settings.STYLESHEETS,
extension.css_bundles)
_remove_bundles(pipeline_settings.JAVASCRIPT, extension.js_bundles)
def _init_admin_site(self, extension):
"""Creates and initializes an admin site for an extension.
This creates the admin site and imports the extensions admin
module to register the models.
The url patterns for the admin site are generated in
_install_admin_urls().
"""
extension.admin_site = AdminSite(extension.info.app_name)
# Import the extension's admin module.
try:
admin_module_name = '%s.admin' % extension.info.app_name
if admin_module_name in sys.modules:
# If the extension has been loaded previously and
# we are re-enabling it, we must reload the module.
# Just importing again will not cause the ModelAdmins
# to be registered.
reload(sys.modules[admin_module_name])
else:
import_module(admin_module_name)
except ImportError:
mod = import_module(extension.info.app_name)
# Decide whether to bubble up this error. If the app just
# doesn't have an admin module, we can ignore the error
# attempting to import it, otherwise we want it to bubble up.
if module_has_submodule(mod, 'admin'):
raise ImportError(
"Importing admin module for extension %s failed" %
extension.info.app_name)
def _add_to_installed_apps(self, extension):
self._installed_apps_setting.add_list(extension.apps
or [extension.info.app_name])
def _remove_from_installed_apps(self, extension):
self._installed_apps_setting.remove_list(extension.apps
or [extension.info.app_name])
def _entrypoint_iterator(self):
return pkg_resources.iter_entry_points(self.key)
def _bump_sync_gen(self):
"""Bumps the synchronization generation value.
If there's an existing synchronization generation in cache,
increment it. Otherwise, start fresh with a new one.
This will also set ``settings.AJAX_SERIAL``, which will guarantee any
cached objects that depends on templates and use this serial number
will be invalidated, allowing TemplateHooks and other hooks
to be re-run.
"""
# If we're in the middle of loading extension state, perhaps due to
# the sync number being bumped by another process, this flag will be
# sent in order to block any further attempts at bumping the number.
# Failure to do this can result in a loop where the number gets
# bumped by every process/thread reacting to another process/thread
# bumping the number, resulting in massive slowdown and errors.
if self._block_sync_gen:
return
self._gen_sync.mark_updated()
settings.AJAX_SERIAL = self._gen_sync.sync_gen
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
try:
from django_evolution.management.commands.evolve import \
Command as Evolution
except ImportError:
raise InstallExtensionError(
"Unable to migrate the extension's database tables. Django "
"Evolution is not installed.")
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0,
interactive=False,
execute=True,
hint=False,
compile_sql=False,
purge=False,
database=False,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e,
output,
exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _recalculate_middleware(self):
"""Recalculates the list of middleware."""
self.middleware = []
done = set()
for e in self.get_enabled_extensions():
self.middleware.extend(self._get_extension_middleware(e, done))
def _get_extension_middleware(self, extension, done):
"""Returns a list of middleware for 'extension' and its dependencies.
This is a recursive utility function initially called by
_recalculate_middleware() that ensures that middleware for all
dependencies are inserted before that of the given extension. It
also ensures that each extension's middleware is inserted only once.
"""
middleware = []
if extension in done:
return middleware
done.add(extension)
for req in extension.requirements:
e = self.get_enabled_extension(req)
if e:
middleware.extend(self._get_extension_middleware(e, done))
middleware.extend(extension.middleware_instances)
return middleware
def setUp(self):
super(GenerationSynchronizerTests, self).setUp()
self.gen_sync = GenerationSynchronizer('test-synchronizer')
class SiteConfiguration(models.Model):
"""Stored version and settings data for a Django site.
This stores dynamic settings for a site, along with version information,
allowing the application to alter and apply/synchronize settings across
threads, processes, and servers without restarting the server.
Consumers should not create or fetch their own instance of this class
through standard Django query functions. Instead, they should use
:py:meth:`SiteConfiguration.objects.get_current()
<djblets.siteconfig.managers.SiteConfigurationManager.get_current>`
instead. See the documentation for that method for details on how to safely
look up and use site configuration.
"""
site = models.ForeignKey(Site,
related_name='config',
on_delete=models.CASCADE)
version = models.CharField(max_length=20)
#: A JSON dictionary field of settings stored for a site.
settings = JSONField()
objects = SiteConfigurationManager()
@classmethod
def add_global_defaults(cls, defaults_dict):
"""Add a dictionary of global defaults for settings.
These defaults will be used when calling :py:meth:`get` for any setting
not stored. Defaults registered for a specific site configuration take
precedent over global defaults.
Args:
default_dict (dict):
A dictionary of defaults, mapping siteconfig settings keys to
JSON-serializable values.
"""
_GLOBAL_DEFAULTS.update(defaults_dict)
@classmethod
def add_global_default(cls, key, default_value):
"""Add a global default value for a settings key.
The default will be used when calling :py:meth:`get` for this key,
if a value is not stored. Defaults registered for a specific site
configuration take precedent over global defaults.
Args:
key (unicode):
The settings key to set the default for.
default_value (object):
The value to set as the default.
"""
cls.add_global_defaults({key: default_value})
@classmethod
def remove_global_default(self, key):
"""Remove a global default value for a settings key.
Args:
key (unicode):
The settings key to remove the default for.
"""
_GLOBAL_DEFAULTS.pop(key)
@classmethod
def clear_global_defaults(self):
"""Clear all default values for this site configuration.
This will clear only global defaults. This will not affect defaults
registered on specific site configurations.
"""
_GLOBAL_DEFAULTS.clear()
@classmethod
def get_global_defaults(cls):
"""Return all global defaults for settings.
Returns:
dict:
A dictionary of all registered global defaults for settings.
"""
return _GLOBAL_DEFAULTS
def __init__(self, *args, **kwargs):
"""Initialize the site configuration.
Args:
*args (tuple):
Positional arguments to pass to the parent constructor.
**kwargs (dict):
Keyword arguments to pass to the parent constructor.
"""
super(SiteConfiguration, self).__init__(*args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer(
'%s:siteconfig:%s:generation' % (self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def get(self, key, default=None):
"""Return the value for a setting.
If the setting is not found, the default value will be returned. This
is represented by the default parameter, if passed in, or a global
default (from :py:meth:`add_default`) if set.
If no default is available, ``None`` will be returned.
Args:
key (unicode):
The site configuration settings key.
default (object, optional):
The default value to return. If not provided, the registered
default will be returned.
Returns:
object:
The resulting value.
"""
if default is None:
try:
default = _DEFAULTS[self.pk][key]
except KeyError:
default = _GLOBAL_DEFAULTS.get(key)
return self.settings.get(key, default)
def set(self, key, value):
"""Set a value for a setting.
The setting will be stored locally until the model is saved, at which
point it will be synchronized with other processes/servers.
Args:
key (unicode):
The key for the setting.
value (object):
The JSON-serializable object to store.
"""
self.settings[key] = value
def add_defaults(self, defaults_dict):
"""Add a dictionary of defaults for settings.
These defaults will be used when calling :py:meth:`get` for any setting
not stored. These will only be registered for this site configuration,
and will not be registered for global defaults.
Args:
default_dict (dict):
A dictionary of defaults, mapping siteconfig settings keys to
JSON-serializable values.
"""
_DEFAULTS.setdefault(self.pk, {}).update(defaults_dict)
def add_default(self, key, default_value):
"""Add a default value for a settings key.
The default will be used when calling :py:meth:`get` for this key,
if a value is not stored. This will only be registered for this site
configuration, and will not be registered for global defaults.
Args:
key (unicode):
The settings key to set the default for.
default_value (object):
The value to set as the default.
"""
self.add_defaults({key: default_value})
def remove_default(self, key):
"""Remove a default value on this site configuration.
This will remove only defaults registered on this site configuration.
This does not affect global defaults.
Args:
key (unicode):
The settings key to remove the default for.
"""
try:
del _DEFAULTS[self.pk][key]
except KeyError:
pass
def clear_defaults(self):
"""Clear all default values for this site configuration.
This will clear only defaults registered on this site configuration.
This does not affect global defaults.
"""
_DEFAULTS[self.pk] = {}
def get_defaults(self):
"""Return all defaults for this site configuration.
This will return only defaults registered on this site configuration.
The result does not include global defaults.
Returns:
dict:
A dictionary of all registered defaults for settings.
"""
return _DEFAULTS.get(self.pk, {})
def is_expired(self):
"""Return whether or not this SiteConfiguration is expired.
If the configuration is expired, it will need to be reloaded before
accessing any settings.
Returns:
bool:
Whether or not the current state is expired.
"""
return self._gen_sync.is_expired()
def save(self, clear_caches=True, **kwargs):
"""Save the site configuration to the database.
By default, saving will clear the caches across all processes/servers
using this site configuration, causing them to be re-fetched on the
next request.
Args:
clear_caches (bool, optional):
Whether to clear the caches. This is ``True`` by default.
**kwargs (dict):
Additional keyword arguments to pass to the parent method.
"""
self._gen_sync.mark_updated()
if clear_caches:
# The cached siteconfig might be stale now. We'll want a refresh.
# Also refresh the Site cache, since callers may get this from
# Site.config.
SiteConfiguration.objects.clear_cache()
Site.objects.clear_cache()
super(SiteConfiguration, self).save(**kwargs)
def __str__(self):
"""Return a string version of the site configuration.
The returned string will list the associated site's domain and the
stored application version.
Returns:
unicode:
The string representation of the site configuration.
"""
return "%s (version %s)" % (six.text_type(self.site), self.version)
class Meta:
# Djblets 0.9+ sets an app label of "djblets_siteconfig" on
# Django 1.7+, which would affect the table name. We need to retain
# the old name for backwards-compatibility.
db_table = 'siteconfig_siteconfiguration'
class IntegrationManager(object):
"""Manages integrations with third-party services.
The manager keeps track of the integrations registered by extensions
or other components of an application, providing the ability to
register new ones, unregister existing ones, and list any that are
currently enabled.
It also manages the lookups of configurations for integrations, taking
care to cache the lookups for any integrations and invalidate them when
a configuration has been updated.
Attrs:
config_model (type):
The model used to store configuration data. This is a subclass of
:py:class:`djblets.integrations.models.BaseIntegrationConfig`.
"""
def __init__(self, config_model):
"""Initialize the integration manager.
Args:
config_model (type):
The model used to store configuration data. This must be a
subclass of
:py:class:`djblets.integrations.models.BaseIntegrationConfig`.
"""
# Check that the Django environment is set up for integrations to
# properly function.
if 'djblets.integrations' not in settings.INSTALLED_APPS:
raise ImproperlyConfigured(
'IntegrationManager requires djblets.integrations to be '
'listed in settings.INSTALLED_APPS.')
middleware = 'djblets.integrations.middleware.IntegrationsMiddleware'
if middleware not in settings.MIDDLEWARE_CLASSES:
raise ImproperlyConfigured(
'IntegrationManager requires %s to be listed in '
'settings.MIDDLEWARE_CLASSES' % middleware)
self.config_model = config_model
key = ('integrationmgr:%s.%s' %
(self.__class__.__module__, self.__class__.__name__))
self._integration_classes = {}
self._integration_configs = {}
self._integration_instances = {}
self._lock = threading.Lock()
self._needs_recalc = False
self._gen_sync = GenerationSynchronizer('%s:gen' % key)
instance_id = id(self)
_integration_managers[instance_id] = self
# Listen for any config model saves/deletes, so we can mark whether
# a reload of state is needed.
dispatch_uid = '%s:%s' % (key, id(self))
post_delete.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
post_save.connect(self._on_config_changes,
sender=config_model,
dispatch_uid=dispatch_uid)
def shutdown(self):
"""Shut down the integrations on this integration manager.
This should be called when the integration manager and integrations
will no longer be used. It will shut down every integration and
unregister all integrations.
"""
for integration in self.get_integrations():
integration.disable_integration()
self._integration_classes = {}
self._integration_configs = {}
self._integration_instances = {}
self._needs_recalc = False
try:
del _integration_managers[id(self)]
except KeyError:
pass
def get_integration_classes(self):
"""Return all the integration classes that have been registered.
This is not sorted in any particular order. It is up to the caller
to determine the correct sorting order.
Yields:
type:
The registered integration classes.
"""
return six.itervalues(self._integration_classes)
def get_integration(self, integration_id):
"""Return an integration instance for a given ID.
Args:
integration_id (unicode):
The integration ID that was registered.
Returns:
djblets.integrations.integration.Integration:
The integration instance.
Raises:
djblets.integrations.errors.IntegrationNotRegisteredError:
The integration class provided wasn't registered.
"""
try:
return self._integration_instances[integration_id]
except KeyError:
raise IntegrationNotRegisteredError(integration_id)
def get_integrations(self):
"""Return all the integration instances.
This is not sorted in any particular order. It is up to the caller
to determine the correct sorting order.
Yields:
djblets.integrations.integration.Integration:
The integration instances.
"""
return six.itervalues(self._integration_instances)
def get_integration_configs(self, integration_cls=None, **filter_kwargs):
"""Return a list of saved integration configurations.
By default, all configurations will be returned for all integrations,
including configurations that are disabled. This can be filtered
down by specifying an integration class and/or by filtering by fields
in the model through keyword arguments.
Each set of results for a unique combination of integration class and
filter arguments will be cached locally, to speed up further lookups.
This cache can be flushed using :py:meth:`clear_configs_cache` or
:py:meth:`clear_all_configs_cache`, and will be automatically cleared
when cnofigurations are added, updated, or removed.
Args:
integration_cls (type, optional):
The integration class to filter by.
**filter_kwargs (dict, optional):
Keyword arguments to filter by. Each must match a field and
value on the model.
Returns:
list of djblets.integrations.models.BaseIntegrationConfig:
A list of saved integration configurations matching the query.
"""
key = self._make_config_filter_cache_key(integration_cls,
**filter_kwargs)
try:
configs = self._integration_configs[key]
except KeyError:
queryset = self.config_model.objects.all()
if integration_cls:
queryset = queryset.filter(
integration_id=integration_cls.integration_id)
if filter_kwargs:
queryset = queryset.filter(**filter_kwargs)
configs = list(queryset)
self._integration_configs[key] = configs
return configs
def clear_configs_cache(self, integration_cls=None, **filter_kwargs):
"""Clear the configuration cache matching the given filters.
This is used to clear a subset of the configs cache, matching the exact
query arguments passed to a previous call to
:py:meth:`get_integration_configs`.
To clear the entire cache, use :py:meth:`clear_all_configs_cache`.
Args:
integration_cls (type, optional):
The integration class for the filter.
**filter_kwargs (dict, optional):
Keyword arguments for the filter.
"""
key = self._make_config_filter_cache_key(integration_cls,
**filter_kwargs)
try:
del self._integration_configs[key]
except KeyError:
pass
def clear_all_configs_cache(self):
"""Clear the entire configuration cache.
This will force all future lookups to re-query the database. To
clear only a subset of the cache, use :py:meth:`clear_configs_cache`.
"""
self._integration_configs = {}
def is_expired(self):
"""Return whether the integration manager has expired state.
Returns:
bool:
``True`` if there's either expired configuration state or
integrations that need their enabled state recalculated.
"""
return self._needs_recalc or self._gen_sync.is_expired()
def check_expired(self):
"""Check for and handle expired integration state.
If the configurations of one or more integrations have been updated
by another process, or there are new integrations registered that
may need to be enabled, this method will reset the cache state and
re-calculate the integrations to enable/disable.
"""
if self.is_expired():
# We're going to check the expiration, and then only lock if it's
# expired. Following that, we'll check again.
#
# We do this in order to prevent locking unnecessarily, which could
# impact performance or cause a problem if a thread is stuck.
#
# We're checking the expiration twice to prevent every blocked
# thread from making its own attempt to reload the integrations
# state the first thread holding the lock finishes.
with self._lock:
# Check again, since another thread may have already
# reloaded.
if self.is_expired():
self._gen_sync.refresh()
self.clear_all_configs_cache()
self._recalc_enabled_integrations()
def register_integration_class(self, integration_cls):
"""Register a class for an integration.
This will instantiate the integration and make it available for new
configurations.
Args:
integration_cls (type):
The integration class to register.
Returns:
djblets.integrations.integration.Integration:
The new instance of the registered integration class.
Raises:
djblets.integrations.errors.IntegrationAlreadyRegisteredError:
The integration class was already registered.
djblets.integrations.errors.IntegrationConstructionError:
Error initializing an instance of the integration. The
integration will not be registered.
"""
if not integration_cls.integration_id:
# A pre-existing integration ID doesn't exist, so create one
# based on the class path.
integration_cls.integration_id = '%s.%s' % (
integration_cls.__module__, integration_cls.__name__)
integration_id = integration_cls.integration_id
if integration_id in self._integration_classes:
raise IntegrationAlreadyRegisteredError(integration_id)
try:
# We're going to instantiate the integration, but we won't
# initialize it until later when we know that there are
# configurations available.
integration = integration_cls(self)
self._integration_instances[integration_id] = integration
self._integration_classes[integration_id] = integration_cls
except Exception as e:
# This should never happen, unless the subclass overrode
# __init__.
raise IntegrationRegistrationError(
'Unexpected error when constructing integration %s: %s' %
(integration_cls.__name__, e))
# Flag that we need to recalculate the list of enabled integrations.
# The next time a request is served, the middleware will perform the
# recalculation. We do it this way instead of during registration in
# order to cut back on the number of queries needed.
self._needs_recalc = True
return integration
def unregister_integration_class(self, integration_cls):
"""Unregister a class for an integration.
The integration instance will be shut down, and the integration will
no longer be made available for any further configuration.
If there is an error shutting down the integration, the output will
be logged, but no error will be returned.
Args:
integration_cls (type):
The integration class to unregister.
Raises:
djblets.integrations.errors.IntegrationNotRegisteredError:
The integration class was not previously registered.
"""
integration_id = integration_cls.integration_id
if (not integration_id
or integration_id not in self._integration_classes):
raise IntegrationNotRegisteredError(integration_id)
integration = self.get_integration(integration_id)
if integration.enabled:
try:
integration.disable_integration()
except Exception as e:
logging.exception(
'Unexpected error when shutting down integration %r: %s',
integration_cls, e)
del self._integration_classes[integration_id]
del self._integration_instances[integration_id]
def _recalc_enabled_integrations(self):
"""Recalculate the enabled states of all integrations.
The list of enabled configurations for integrations will be queried
from the database. Based on this, the desired enabled state of each
integration will be calculated. Those that are disabled but have
enabled configurations will be enabled, and those that are enabled
but no longer have enabled configurations will be disabled.
This allows us to keep memory requirements and event handling at a
minimum for any integrations not currently in use.
"""
enabled_integration_ids = set(
self.config_model.objects.filter(
enabled=True).distinct().values_list('integration_id',
flat=True))
for integration in self.get_integrations():
should_enable = (integration.integration_id
in enabled_integration_ids)
if should_enable != integration.enabled:
if should_enable:
integration.enable_integration()
else:
integration.disable_integration()
def _make_config_filter_cache_key(self, integration_cls, **filter_kwargs):
"""Return a cache key for a config query filter.
Args:
integration_cls (type):
The integration class used for the query.
**filter_kwargs (dict):
The filter keyword arguments used for the query.
Returns:
unicode:
The resulting cache key.
"""
if integration_cls:
return '%s:%s' % (integration_cls.integration_id, filter_kwargs)
else:
return '*:%s' % (filter_kwargs, )
def _on_config_changes(self, **kwargs):
"""Handler for when configuration state changes.
This will force the list of integrations to recalculate on this
process and others when a configuration is created, saved, or deleted.
Args:
**kwargs (dict):
Keyword arguments passed to the signal.
"""
self._needs_recalc = True
self._gen_sync.mark_updated()
class SiteConfiguration(models.Model):
"""Stored version and settings data for a Django site.
This stores dynamic settings for a site, along with version information,
allowing the application to alter and apply/synchronize settings across
threads, processes, and servers without restarting the server.
Consumers should not create or fetch their own instance of this class
through standard Django query functions. Instead, they should use
:py:meth:`SiteConfiguration.objects.get_current()
<djblets.siteconfig.managers.SiteConfigurationManager.get_current>`
instead. See the documentation for that method for details on how to safely
look up and use site configuration.
"""
site = models.ForeignKey(Site,
related_name='config',
on_delete=models.CASCADE)
version = models.CharField(max_length=20)
#: A JSON dictionary field of settings stored for a site.
settings = JSONField()
objects = SiteConfigurationManager()
@classmethod
def add_global_defaults(cls, defaults_dict):
"""Add a dictionary of global defaults for settings.
These defaults will be used when calling :py:meth:`get` for any setting
not stored. Defaults registered for a specific site configuration take
precedent over global defaults.
Args:
default_dict (dict):
A dictionary of defaults, mapping siteconfig settings keys to
JSON-serializable values.
"""
_GLOBAL_DEFAULTS.update(defaults_dict)
@classmethod
def add_global_default(cls, key, default_value):
"""Add a global default value for a settings key.
The default will be used when calling :py:meth:`get` for this key,
if a value is not stored. Defaults registered for a specific site
configuration take precedent over global defaults.
Args:
key (unicode):
The settings key to set the default for.
default_value (object):
The value to set as the default.
"""
cls.add_global_defaults({key: default_value})
@classmethod
def remove_global_default(self, key):
"""Remove a global default value for a settings key.
Args:
key (unicode):
The settings key to remove the default for.
"""
_GLOBAL_DEFAULTS.pop(key)
@classmethod
def clear_global_defaults(self):
"""Clear all default values for this site configuration.
This will clear only global defaults. This will not affect defaults
registered on specific site configurations.
"""
_GLOBAL_DEFAULTS.clear()
@classmethod
def get_global_defaults(cls):
"""Return all global defaults for settings.
Returns:
dict:
A dictionary of all registered global defaults for settings.
"""
return _GLOBAL_DEFAULTS
def __init__(self, *args, **kwargs):
"""Initialize the site configuration.
Args:
*args (tuple):
Positional arguments to pass to the parent constructor.
**kwargs (dict):
Keyword arguments to pass to the parent constructor.
"""
super(SiteConfiguration, self).__init__(*args, **kwargs)
# Optimistically try to set the Site to the current site instance,
# which either is cached now or soon will be. That way, we avoid
# a lookup on the relation later.
cur_site = Site.objects.get_current()
if cur_site.pk == self.site_id:
self.site = cur_site
# Begin managing the synchronization of settings between all
# SiteConfigurations.
self._gen_sync = GenerationSynchronizer('%s:siteconfig:%s:generation' %
(self.site.domain, self.pk))
self.settings_wrapper = SiteConfigSettingsWrapper(self)
def get(self, key, default=None):
"""Return the value for a setting.
If the setting is not found, the default value will be returned. This
is represented by the default parameter, if passed in, or a global
default (from :py:meth:`add_default`) if set.
If no default is available, ``None`` will be returned.
Args:
key (unicode):
The site configuration settings key.
default (object, optional):
The default value to return. If not provided, the registered
default will be returned.
Returns:
object:
The resulting value.
"""
if default is None:
try:
default = _DEFAULTS[self.pk][key]
except KeyError:
default = _GLOBAL_DEFAULTS.get(key)
return self.settings.get(key, default)
def set(self, key, value):
"""Set a value for a setting.
The setting will be stored locally until the model is saved, at which
point it will be synchronized with other processes/servers.
Args:
key (unicode):
The key for the setting.
value (object):
The JSON-serializable object to store.
"""
self.settings[key] = value
def add_defaults(self, defaults_dict):
"""Add a dictionary of defaults for settings.
These defaults will be used when calling :py:meth:`get` for any setting
not stored. These will only be registered for this site configuration,
and will not be registered for global defaults.
Args:
default_dict (dict):
A dictionary of defaults, mapping siteconfig settings keys to
JSON-serializable values.
"""
_DEFAULTS.setdefault(self.pk, {}).update(defaults_dict)
def add_default(self, key, default_value):
"""Add a default value for a settings key.
The default will be used when calling :py:meth:`get` for this key,
if a value is not stored. This will only be registered for this site
configuration, and will not be registered for global defaults.
Args:
key (unicode):
The settings key to set the default for.
default_value (object):
The value to set as the default.
"""
self.add_defaults({key: default_value})
def remove_default(self, key):
"""Remove a default value on this site configuration.
This will remove only defaults registered on this site configuration.
This does not affect global defaults.
Args:
key (unicode):
The settings key to remove the default for.
"""
try:
del _DEFAULTS[self.pk][key]
except KeyError:
pass
def clear_defaults(self):
"""Clear all default values for this site configuration.
This will clear only defaults registered on this site configuration.
This does not affect global defaults.
"""
_DEFAULTS[self.pk] = {}
def get_defaults(self):
"""Return all defaults for this site configuration.
This will return only defaults registered on this site configuration.
The result does not include global defaults.
Returns:
dict:
A dictionary of all registered defaults for settings.
"""
return _DEFAULTS.get(self.pk, {})
def is_expired(self):
"""Return whether or not this SiteConfiguration is expired.
If the configuration is expired, it will need to be reloaded before
accessing any settings.
Returns:
bool:
Whether or not the current state is expired.
"""
return self._gen_sync.is_expired()
def save(self, clear_caches=True, **kwargs):
"""Save the site configuration to the database.
By default, saving will clear the caches across all processes/servers
using this site configuration, causing them to be re-fetched on the
next request.
Args:
clear_caches (bool, optional):
Whether to clear the caches. This is ``True`` by default.
**kwargs (dict):
Additional keyword arguments to pass to the parent method.
"""
self._gen_sync.mark_updated()
if clear_caches:
# The cached siteconfig might be stale now. We'll want a refresh.
# Also refresh the Site cache, since callers may get this from
# Site.config.
SiteConfiguration.objects.clear_cache()
Site.objects.clear_cache()
super(SiteConfiguration, self).save(**kwargs)
def __str__(self):
"""Return a string version of the site configuration.
The returned string will list the associated site's domain and the
stored application version.
Returns:
unicode:
The string representation of the site configuration.
"""
return "%s (version %s)" % (six.text_type(self.site), self.version)
class Meta:
# Djblets 0.9+ sets an app label of "djblets_siteconfig" on
# Django 1.7+, which would affect the table name. We need to retain
# the old name for backwards-compatibility.
db_table = 'siteconfig_siteconfiguration'