def _cache_store_chunks(items, key, expiration):
"""Store a list of items as chunks in the cache.
The list of items will be combined into chunks and stored in the
cache as efficiently as possible. Each item in the list will be
yielded to the caller as it's fetched from the list or generator.
"""
chunks_data = StringIO()
chunks_data_len = 0
read_start = 0
item_count = 0
i = 0
for data, has_item, item in items:
if has_item:
yield item
item_count += 1
chunks_data.write(data)
chunks_data_len += len(data)
if chunks_data_len > CACHE_CHUNK_SIZE:
# We have enough data to fill a chunk now. Start processing
# what we've stored and create cache keys for each chunk.
# Anything remaining will be stored for the next round.
chunks_data.seek(read_start)
cached_data = {}
while chunks_data_len > CACHE_CHUNK_SIZE:
chunk = chunks_data.read(CACHE_CHUNK_SIZE)
chunk_len = len(chunk)
chunks_data_len -= chunk_len
read_start += chunk_len
# Note that we wrap the chunk in a list so that the cache
# backend won't try to perform any conversion on the string.
cached_data[make_cache_key('%s-%d' % (key, i))] = [chunk]
i += 1
# Store the keys in the cache in a single request.
cache.set_many(cached_data, expiration)
# Reposition back at the end of the stream.
chunks_data.seek(0, 2)
if chunks_data_len > 0:
# There's one last bit of data to store. Note that this should be
# less than the size of a chunk,
assert chunks_data_len <= CACHE_CHUNK_SIZE
chunks_data.seek(read_start)
chunk = chunks_data.read()
cache.set(make_cache_key('%s-%d' % (key, i)), [chunk], expiration)
i += 1
cache.set(make_cache_key(key), '%d' % i, expiration)
def _cache_store_chunks(items, key, expiration):
"""Store a list of items as chunks in the cache.
The list of items will be combined into chunks and stored in the
cache as efficiently as possible. Each item in the list will be
yielded to the caller as it's fetched from the list or generator.
"""
chunks_data = StringIO()
chunks_data_len = 0
read_start = 0
item_count = 0
i = 0
for data, has_item, item in items:
if has_item:
yield item
item_count += 1
chunks_data.write(data)
chunks_data_len += len(data)
if chunks_data_len > CACHE_CHUNK_SIZE:
# We have enough data to fill a chunk now. Start processing
# what we've stored and create cache keys for each chunk.
# Anything remaining will be stored for the next round.
chunks_data.seek(read_start)
cached_data = {}
while chunks_data_len > CACHE_CHUNK_SIZE:
chunk = chunks_data.read(CACHE_CHUNK_SIZE)
chunk_len = len(chunk)
chunks_data_len -= chunk_len
read_start += chunk_len
# Note that we wrap the chunk in a list so that the cache
# backend won't try to perform any conversion on the string.
cached_data[make_cache_key('%s-%d' % (key, i))] = [chunk]
i += 1
# Store the keys in the cache in a single request.
cache.set_many(cached_data, expiration)
# Reposition back at the end of the stream.
chunks_data.seek(0, 2)
if chunks_data_len > 0:
# There's one last bit of data to store. Note that this should be
# less than the size of a chunk,
assert chunks_data_len <= CACHE_CHUNK_SIZE
chunks_data.seek(read_start)
chunk = chunks_data.read()
cache.set(make_cache_key('%s-%d' % (key, i)), [chunk], expiration)
i += 1
cache.set(make_cache_key(key), '%d' % i, expiration)
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
try:
from django_evolution.management.commands.evolve import \
Command as Evolution
except ImportError:
raise InstallExtensionError(
"Unable to migrate the extension's database tables. Django "
"Evolution is not installed.")
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0,
interactive=False,
execute=True,
hint=False,
compile_sql=False,
purge=False,
database=False,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e,
output,
exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
try:
from django_evolution.management.commands.evolve import \
Command as Evolution
except ImportError:
raise InstallExtensionError(
"Unable to migrate the extension's database tables. Django "
"Evolution is not installed.")
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0, interactive=False,
execute=True, hint=False,
compile_sql=False, purge=False,
database=False,
stdout=stream, stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e, output, exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
if django_evolution is None:
# Django Evolution isn't installed. Extensions with evolutions
# are not supported.
return
try:
stream = StringIO()
call_command('evolve',
verbosity=0,
interactive=False,
execute=True,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e,
output,
exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _migrate_extension_models(self, ext_class):
"""Perform database migrations for an extension's models.
This will call out to Django Evolution to handle the migrations.
Args:
ext_class (djblets.extensions.extension.Extension):
The class for the extension to migrate.
"""
if django_evolution is None:
# Django Evolution isn't installed. Extensions with evolutions
# are not supported.
return
try:
stream = StringIO()
call_command('evolve',
verbosity=0,
interactive=False,
execute=True,
stdout=stream,
stderr=stream)
output = stream.getvalue()
if output:
logger.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logger.exception('Error evolving extension models: %s: %s',
e, output)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
def _install_extension_media_internal(self, ext_class):
"""Installs extension data.
Performs any installation necessary for an extension.
If the extension has a legacy htdocs/ directory for static media
files, they will be installed into MEDIA_ROOT/ext/, and a warning
will be logged.
If the extension has a modern static/ directory, they will be
installed into STATIC_ROOT/ext/.
"""
ext_htdocs_path = ext_class.info.installed_htdocs_path
ext_htdocs_path_exists = os.path.exists(ext_htdocs_path)
if ext_htdocs_path_exists:
# First, get rid of the old htdocs contents, so we can start
# fresh.
shutil.rmtree(ext_htdocs_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'htdocs'):
# This is an older extension that doesn't use the static file
# support. Log a deprecation notice and then install the files.
logging.warning('The %s extension uses the deprecated "htdocs" '
'directory for static files. It should be updated '
'to use a "static" directory instead.'
% ext_class.info.name)
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__, 'htdocs')
shutil.copytree(extracted_path, ext_htdocs_path, symlinks=True)
# We only want to install static media on a non-DEBUG install.
# Otherwise, we run the risk of creating a new 'static' directory and
# causing Django to look up all static files (not just from
# extensions) from there instead of from their source locations.
if not settings.DEBUG:
ext_static_path = ext_class.info.installed_static_path
ext_static_path_exists = os.path.exists(ext_static_path)
if ext_static_path_exists:
# Also get rid of the old static contents.
shutil.rmtree(ext_static_path, ignore_errors=True)
if pkg_resources.resource_exists(ext_class.__module__, 'static'):
extracted_path = \
pkg_resources.resource_filename(ext_class.__module__,
'static')
shutil.copytree(extracted_path, ext_static_path, symlinks=True)
# Mark the extension as installed
ext_class.registration.installed = True
ext_class.registration.save()
# Now let's build any tables that this extension might need
self._add_to_installed_apps(ext_class)
# Call syncdb to create the new tables
loading.cache.loaded = False
call_command('syncdb', verbosity=0, interactive=False)
# Run evolve to do any table modification
try:
stream = StringIO()
evolution = Evolution()
evolution.style = no_style()
evolution.execute(verbosity=0, interactive=False,
execute=True, hint=False,
compile_sql=False, purge=False,
database=False,
stdout=stream, stderr=stream)
output = stream.getvalue()
if output:
logging.info('Evolved extension models for %s: %s',
ext_class.id, stream.read())
stream.close()
except CommandError as e:
# Something went wrong while running django-evolution, so
# grab the output. We can't raise right away because we
# still need to put stdout back the way it was
output = stream.getvalue()
stream.close()
logging.error('Error evolving extension models: %s: %s',
e, output, exc_info=1)
load_error = self._store_load_error(ext_class.id, output)
raise InstallExtensionError(six.text_type(e), load_error)
# Remove this again, since we only needed it for syncdb and
# evolve. _init_extension will add it again later in
# the install.
self._remove_from_installed_apps(ext_class)
# Mark the extension as installed
ext_class.registration.installed = True
ext_class.registration.save()
