def _build(self, env, output_path, force, no_filters, parent_filters=[]):
"""Internal recursive build method.
"""
# TODO: We could support a nested bundle downgrading it's debug
# setting from "filters" to "merge only", i.e. enabling
# ``no_filters``. We cannot support downgrading to
# "full debug/no merge" (debug=True), of course.
#
# Right now we simply use the debug setting of the root bundle
# we build, und it overrides all the nested bundles. If we
# allow nested bundles to overwrite the debug value of parent
# bundles, as described above, then we should also deal with
# a child bundle enabling debug=True during a merge, i.e.
# raising an error rather than ignoring it as we do now.
resolved_contents = self.resolve_contents(env)
if not resolved_contents:
raise BuildError('empty bundle cannot be built')
# Ensure that the filters are ready
for filter in self.filters:
filter.set_environment(env)
# Apply input filters to all the contents. Note that we use
# both this bundle's filters as well as those given to us by
# the parent. We ONLY do those this for the input filters,
# because we need them to be applied before the apply our own
# output filters.
# TODO: Note that merge_filters() removes duplicates. Is this
# really the right thing to do, or does it just confuse things
# due to there now being different kinds of behavior...
combined_filters = merge_filters(self.filters, parent_filters)
cache = get_cache(env)
hunks = []
for c in resolved_contents:
if isinstance(c, Bundle):
hunk = c._build(env, output_path, force, no_filters,
combined_filters)
hunks.append(hunk)
else:
if is_url(c):
hunk = UrlHunk(c)
else:
hunk = FileHunk(env.abspath(c))
if no_filters:
hunks.append(hunk)
else:
hunks.append(
apply_filters(hunk,
combined_filters,
'input',
cache,
output_path=output_path))
# Return all source hunks as one, with output filters applied
final = merge(hunks)
if no_filters:
return final
else:
return apply_filters(final, self.filters, 'output', cache)
def build(self, env=None, force=False, no_filters=False):
"""Build this bundle, meaning create the file given by the
``output`` attribute, applying the configured filters etc.
A ``FileHunk`` will be returned.
TODO: Support locking. When called from inside a template tag,
this should lock, so that multiple requests don't all start
to build. When called from the command line, there is no need
to lock.
"""
if not self.output:
raise BuildError('No output target found for %s' % self)
env = self._get_env(env)
# Determine if we really need to build, or if the output file
# already exists and nothing has changed.
if force:
update_needed = True
elif not path.exists(env.abspath(self.output)):
if not env.updater:
raise BuildError(('\'%s\' needs to be created, but '
'automatic building is disabled ('
'configure an updater)') % self)
else:
update_needed = True
else:
source_paths = [p for p in self.get_files(env)]
update_needed = get_updater(env.updater)(
env.abspath(self.output), source_paths)
if not update_needed:
# We can simply return the existing output file
return FileHunk(env.abspath(self.output))
hunk = self._build(env, self.output, force, no_filters)
hunk.save(env.abspath(self.output))
return hunk
def pull_external(env, filename):
"""Helper which will pull ``filename`` into
:attr:`Environment.directory`, for the purposes of being able to
generate a url for it.
"""
# Generate the target filename. Use a hash to keep it unique and short,
# but attach the base filename for readability.
# The bit-shifting rids us of ugly leading - characters.
hashed_filename = hash(filename) & ((1 << 64) - 1)
rel_path = path.join('webassets-external',
"%s_%s" % (hashed_filename, path.basename(filename)))
full_path = path.join(env.directory, rel_path)
# Copy the file if necessary
if path.isfile(full_path):
gs = lambda p: os.stat(p).st_mtime
if gs(full_path) > gs(filename):
return rel_path
directory = path.dirname(full_path)
if not path.exists(directory):
os.makedirs(directory)
FileHunk(filename).save(full_path)
return full_path
def _build(self,
env,
extra_filters=[],
force=None,
output=None,
disable_cache=None):
"""Internal bundle build function.
This actually tries to build this very bundle instance, as opposed to
the public-facing ``build()``, which first deals with the possibility
that we are a container bundle, i.e. having no files of our own.
First checks whether an update for this bundle is required, via the
configured ``updater`` (which is almost always the timestamp-based one).
Unless ``force`` is given, in which case the bundle will always be
built, without considering timestamps.
Note: The default value of ``force`` is normally ``False``, unless
``auto_build`` is disabled, in which case ``True`` is assumed.
A ``FileHunk`` will be returned, or in a certain case, with no updater
defined and force=False, the return value may be ``False``.
TODO: Support locking. When called from inside a template tag, this
should lock, so that multiple requests don't all start to build. When
called from the command line, there is no need to lock.
"""
if not self.output:
raise BuildError('No output target found for %s' % self)
# Default force to True if auto_build is disabled, as otherwise
# no build would happen. This is only a question of API design.
# We want auto_build=False users to be able to call bundle.build()
# and have it have an effect.
if force is None:
force = not env.auto_build
# Determine if we really need to build, or if the output file
# already exists and nothing has changed.
if force:
update_needed = True
elif not env.auto_build:
# If the user disables the updater, he expects to be able
# to manage builds all on his one. Don't even bother wasting
# IO ops on an update check. It's also convenient for
# deployment scenarios where the media files are on a different
# server, and we can't even access the output file.
return False
elif not has_placeholder(self.output) and \
not path.exists(env.abspath(self.output)):
update_needed = True
else:
if env.auto_build:
update_needed = env.updater.needs_rebuild(self, env)
# _merge_and_apply() is now smart enough to do without
# this disable_cache hint, but for now, keep passing it
# along if we get the info from the updater.
if update_needed == SKIP_CACHE:
disable_cache = True
else:
update_needed = False
if not update_needed:
# We can simply return the existing output file
return FileHunk(env.abspath(self.output))
hunk = self._merge_and_apply(env,
self.output,
force,
disable_cache=disable_cache,
extra_filters=extra_filters)
if output:
# If we are given a stream, just write to it.
output.write(hunk.data())
else:
# If it doesn't exist yet, create the target directory.
output = env.abspath(self.output)
output_dir = path.dirname(output)
if not path.exists(output_dir):
os.makedirs(output_dir)
version = None
if env.versions:
version = env.versions.determine_version(self, env, hunk)
if not has_placeholder(self.output):
hunk.save(self.resolve_output(env))
else:
if not env.versions:
raise BuildError(
('You have not set the "versions" option, but %s '
'uses a version placeholder in the output target' %
self))
output = self.resolve_output(env, version=version)
hunk.save(output)
self.version = version
if env.manifest:
env.manifest.remember(self, env, version)
if env.versions and version:
# Hook for the versioner (for example set the timestamp of
# the file) to the actual version.
env.versions.set_version(self, env, output, version)
# The updater may need to know this bundle exists and how it
# has been last built, in order to detect changes in the
# bundle definition, like new source files.
env.updater.build_done(self, env)
return hunk
def _merge_and_apply(self,
env,
output,
force,
parent_debug=None,
parent_filters=[],
extra_filters=[],
disable_cache=None):
"""Internal recursive build method.
``parent_debug`` is the debug setting used by the parent bundle. This
is not necessarily ``bundle.debug``, but rather what the calling method
in the recursion tree is actually using.
``parent_filters`` are what the parent passes along, for us to be
applied as input filters. Like ``parent_debug``, it is a collection of
the filters of all parents in the hierarchy.
``extra_filters`` may exist if the parent is a container bundle passing
filters along to its children; these are applied as input and output
filters (since there is no parent who could do the latter), and they
are not passed further down the hierarchy (but instead they become part
of ``parent_filters``.
``disable_cache`` is necessary because in some cases, when an external
bundle dependency has changed, we must not rely on the cache, since the
cache key is not taking into account changes in those dependencies
(for now).
"""
assert not path.isabs(output)
# Determine the debug level to use. It determines if and which filters
# should be applied.
#
# The debug level is inherited (if the parent bundle is merging, a
# child bundle clearly cannot act in full debug=True mode). Bundles
# may define a custom ``debug`` attributes, but child bundles may only
# ever lower it, not increase it.
#
# If not parent_debug is given (top level), use the Environment value.
parent_debug = parent_debug if parent_debug is not None else env.debug
# Consider bundle's debug attribute and other things
current_debug_level = _effective_debug_level(env,
self,
extra_filters,
default=parent_debug)
# Special case: If we end up with ``True``, assume ``False`` instead.
# The alternative would be for the build() method to refuse to work at
# this point, which seems unnecessarily inconvenient (Instead how it
# works is that urls() simply doesn't call build() when debugging).
# Note: This can only happen if the Environment sets debug=True and
# nothing else overrides it.
if current_debug_level is True:
current_debug_level = False
# Put together a list of filters that we would want to run here.
# These will be the bundle's filters, and any extra filters given
# to use if the parent is a container bundle. Note we do not yet
# include input/open filters pushed down by a parent build iteration.
filters = merge_filters(self.filters, extra_filters)
# Given the debug level, determine which of the filters want to run
selected_filters = select_filters(filters, current_debug_level)
# We construct two lists of filters. The ones we want to use in this
# iteration, and the ones we want to pass down to child bundles.
# Why? Say we are in merge mode. Assume an "input()" filter which does
# not run in merge mode, and a child bundle that switches to
# debug=False. The child bundle then DOES want to run those input
# filters, so we do need to pass them.
filters_to_run = merge_filters(
selected_filters,
select_filters(parent_filters, current_debug_level))
filters_to_pass_down = merge_filters(filters, parent_filters)
# Initialize al the filters (those we use now, those we pass down).
for filter in filters:
filter.set_environment(env)
# Since we call this now every single time before the filter
# is used, we might pass the bundle instance it is going
# to be used with. For backwards-compatibility reasons, this
# is problematic. However, by inspecting the support arguments,
# we can deal with it. We probably then want to deprecate
# the old syntax before 1.0 (TODO).
filter.setup()
# Prepare contents
resolved_contents = self.resolve_contents(env, force=True)
if not resolved_contents:
raise BuildError('empty bundle cannot be built')
# Unless we have been told by our caller to use or not use the cache
# for this, try to decide for ourselves. The issue here is that when a
# bundle has dependencies, like a sass file with includes otherwise not
# listed in the bundle sources, a change in such an external include
# would not influence the cache key, those the use of the cache causing
# such a change to be ignored. For now, we simply do not use the cache
# for any bundle with dependencies. Another option would be to read
# the contents of all files declared via "depends", and use them as a
# cache key modifier. For now I am worried about the performance impact.
#
# Note: This decision only affects the current bundle instance. Even if
# dependencies cause us to ignore the cache for this bundle instance,
# child bundles may still use it!
if disable_cache is None:
actually_skip_cache_here = bool(self.resolve_depends(env))
else:
actually_skip_cache_here = disable_cache
filtertool = FilterTool(env.cache,
no_cache_read=actually_skip_cache_here,
kwargs={
'output': output,
'output_path': env.abspath(output)
})
# Apply input()/open() filters to all the contents.
hunks = []
for rel_name, item in resolved_contents:
if isinstance(item, Bundle):
hunk = item._merge_and_apply(env,
output,
force,
current_debug_level,
filters_to_pass_down,
disable_cache=disable_cache)
hunks.append(hunk)
else:
# Give a filter the chance to open his file.
try:
hunk = filtertool.apply_func(
filters_to_run,
'open',
[item],
# Also pass along the original relative path, as
# specified by the user, before resolving.
kwargs={'source': rel_name},
# We still need to open the file ourselves too and use
# it's content as part of the cache key, otherwise this
# filter application would only be cached by filename,
# and changes in the source not detected. The other
# option is to not use the cache at all here. Both have
# different performance implications, but I'm guessing
# that reading and hashing some files unnecessarily
# very often is better than running filters
# unnecessarily occasionally.
cache_key=[FileHunk(item)] if not is_url(item) else [])
except MoreThanOneFilterError, e:
raise BuildError(e)
if not hunk:
if is_url(item):
hunk = UrlHunk(item)
else:
hunk = FileHunk(item)
hunks.append(
filtertool.apply(
hunk,
filters_to_run,
'input',
# Pass along both the original relative path, as
# specified by the user, and the one that has been
# resolved to a filesystem location.
kwargs={
'source': rel_name,
'source_path': item
}))
def _build(self, env, extra_filters=[], force=None):
"""Internal bundle build function.
Check if an update for this bundle is required, and if so,
build it. If ``force`` is given, the bundle will always be
built, without checking for an update.
If no ``updater`` is configured, then ``force`` defaults to
``True``.
A ``FileHunk`` will be returned, or in a certain case, with
no updater defined and force=False, the return value may be
``False``.
TODO: Support locking. When called from inside a template tag,
this should lock, so that multiple requests don't all start
to build. When called from the command line, there is no need
to lock.
"""
if not self.output:
raise BuildError('No output target found for %s' % self)
# Default force to True if no updater is given, as otherwise
# no build would happen. This is only a question of API design.
# We want updater=False users to be able to call bundle.build()
# and have it have an effect.
if force is None:
force = not bool(env.updater)
# Determine if we really need to build, or if the output file
# already exists and nothing has changed.
if force:
update_needed = True
elif not env.updater:
# If the user disables the updater, he expects to be able
# to manage builds all on his one. Don't even bother wasting
# IO ops on an update check. It's also convenient for
# deployment scenarios where the media files are on a different
# server, and we can't even access the output file.
return False
elif not path.exists(env.abspath(self.output)):
update_needed = True
else:
if env.updater:
update_needed = env.updater.needs_rebuild(self, env)
else:
update_needed = False
if not update_needed:
# We can simply return the existing output file
return FileHunk(env.abspath(self.output))
hunk = self._merge_and_apply(env,
self.output,
force,
disable_cache=update_needed == SKIP_CACHE,
extra_filters=extra_filters)
# If it doesn't exist yet, create the target directory.
output = env.abspath(self.output)
output_dir = path.dirname(output)
if not path.exists(output_dir):
os.makedirs(output_dir)
hunk.save(output)
# The updater may need to know this bundle exists and how it
# has been last built, in order to detect changes in the
# bundle definition, like new source files.
if env.updater:
env.updater.build_done(self, env)
return hunk
def _merge_and_apply(self,
env,
output_path,
force,
parent_debug=None,
parent_filters=[],
extra_filters=[],
disable_cache=False):
"""Internal recursive build method.
``parent_debug`` is the debug setting used by the parent bundle.
This is not necessarily ``bundle.debug``, but rather what the
calling method in the recursion tree is actually using.
``parent_filters`` are what the parent passes along, for
us to be applied as input filters. Like ``parent_debug``, it is
a collection of the filters of all parents in the hierarchy.
``extra_filters`` may exist if the parent is a container bundle
passing filters along to it's children; these are applied as input
and output filters (since there is no parent who could do the
latter), and they are not passed further down the hierarchy
(but instead they become part of ``parent_filters``.
``disable_cache`` is necessary because in some cases, when an
external bundle dependency has changed, we must not rely on the
cache.
"""
# Determine the debug option to work, which will tell us what
# building the bundle entails. The reduce chooses the first
# non-None value.
debug = reduce(lambda x, y: x if not x is None else y,
[self.debug, parent_debug, env.debug])
if debug == 'merge':
no_filters = True
elif debug is True:
# This should be caught by urls().
if any([self.debug, parent_debug]):
raise BuildError("a bundle with debug=True cannot be built")
else:
raise BuildError("cannot build while in debug mode")
elif debug is False:
no_filters = False
else:
raise BundleError('Invalid debug value: %s' % debug)
# Prepare contents
resolved_contents = self.resolve_contents(env, force=True)
if not resolved_contents:
raise BuildError('empty bundle cannot be built')
# Prepare filters
filters = merge_filters(self.filters, extra_filters)
for filter in filters:
filter.set_environment(env)
# Apply input filters to all the contents. Note that we use
# both this bundle's filters as well as those given to us by
# the parent. We ONLY do those this for the input filters,
# because we need them to be applied before the apply our own
# output filters.
combined_filters = merge_filters(filters, parent_filters)
hunks = []
for _, c in resolved_contents:
if isinstance(c, Bundle):
hunk = c._merge_and_apply(env,
output_path,
force,
debug,
combined_filters,
disable_cache=disable_cache)
hunks.append(hunk)
else:
if is_url(c):
hunk = UrlHunk(c)
else:
hunk = FileHunk(c)
if no_filters:
hunks.append(hunk)
else:
hunks.append(
apply_filters(hunk,
combined_filters,
'input',
env.cache,
disable_cache,
output_path=output_path))
# Return all source hunks as one, with output filters applied
try:
final = merge(hunks)
except IOError, e:
raise BuildError(e)
def _build(self,
env,
extra_filters=[],
force=None,
output=None,
disable_cache=None):
"""Internal bundle build function.
This actually tries to build this very bundle instance, as opposed to
the public-facing ``build()``, which first deals with the possibility
that we are a container bundle, i.e. having no files of our own.
First checks whether an update for this bundle is required, via the
configured ``updater`` (which is almost always the timestamp-based one).
Unless ``force`` is given, in which case the bundle will always be
built, without considering timestamps.
A ``FileHunk`` will be returned, or in a certain case, with no updater
defined and force=False, the return value may be ``False``.
TODO: Support locking. When called from inside a template tag, this
should lock, so that multiple requests don't all start to build. When
called from the command line, there is no need to lock.
"""
if not self.output:
raise BuildError('No output target found for %s' % self)
# Determine if we really need to build, or if the output file
# already exists and nothing has changed.
if force:
update_needed = True
elif not has_placeholder(self.output) and \
not path.exists(self.resolve_output(env, self.output)):
update_needed = True
else:
update_needed = env.updater.needs_rebuild(self, env) \
if env.updater else True
if update_needed == SKIP_CACHE:
disable_cache = True
if not update_needed:
# We can simply return the existing output file
return FileHunk(self.resolve_output(env, self.output))
hunk = self._merge_and_apply(
env,
[self.output, self.resolve_output(env, version='?')],
force,
disable_cache=disable_cache,
extra_filters=extra_filters)
if output:
# If we are given a stream, just write to it.
output.write(hunk.data())
else:
# If it doesn't exist yet, create the target directory.
output = path.join(env.directory, self.output)
output_dir = path.dirname(output)
if not path.exists(output_dir):
os.makedirs(output_dir)
version = None
if env.versions:
version = env.versions.determine_version(self, env, hunk)
if not has_placeholder(self.output):
hunk.save(self.resolve_output(env))
else:
if not env.versions:
raise BuildError(
('You have not set the "versions" option, but %s '
'uses a version placeholder in the output target' %
self))
output = self.resolve_output(env, version=version)
hunk.save(output)
self.version = version
if env.manifest:
env.manifest.remember(self, env, version)
if env.versions and version:
# Hook for the versioner (for example set the timestamp of
# the file) to the actual version.
env.versions.set_version(self, env, output, version)
# The updater may need to know this bundle exists and how it
# has been last built, in order to detect changes in the
# bundle definition, like new source files.
if env.updater:
env.updater.build_done(self, env)
return hunk