def __init__(self, build_context, rule, *args, **kwargs):
"""Initializes a rule context.
Args:
build_context: BuildContext this rule is running in.
rule: Rule this context wraps.
"""
self.build_context = build_context
self.build_env = build_context.build_env
self.rule = rule
self.file_delta = None
self.deferred = Deferred()
self.status = Status.WAITING
self.start_time = None
self.end_time = None
self.exception = None
# TODO(benvanik): logger
self.logger = None
# Resolve all src paths
# If rules have their own attrs they'll have to do them themselves
self.src_paths = self._resolve_input_files(rule.srcs,
apply_src_filter=True)
# This list of all files this rule outputted, upon completion
self.all_output_files = []
def run_task_async(self, task):
if self.closed:
raise RuntimeError('Executor has been closed and cannot run new tasks')
deferred = Deferred()
try:
result = task.execute()
deferred.callback(result)
except Exception as e:
print 'exception in task:'
traceback.print_exc(e)
deferred.errback(exception=e)
return deferred
def __init__(self, build_context, rule, *args, **kwargs):
"""Initializes a rule context.
Args:
build_context: BuildContext this rule is running in.
rule: Rule this context wraps.
"""
self.build_context = build_context
self.build_env = build_context.build_env
self.rule = rule
self.file_delta = None
self.deferred = Deferred()
self.status = Status.WAITING
self.start_time = None
self.end_time = None
self.exception = None
# TODO(benvanik): logger
self.logger = None
# Resolve all src paths
# If rules have their own attrs they'll have to do them themselves
self.src_paths = self._resolve_input_files(rule.srcs, apply_src_filter=True)
# This list of all files this rule outputted, upon completion
self.all_output_files = []
def run_task_async(self, task):
if self.closed:
raise RuntimeError(
'Executor has been closed and cannot run new tasks')
deferred = Deferred()
try:
result = task.execute()
deferred.callback(result)
except Exception as e:
print 'exception in task:'
traceback.print_exc(e)
deferred.errback(exception=e)
return deferred
def run_task_async(self, task):
if self.closed:
raise RuntimeError(
'Executor has been closed and cannot run new tasks')
# Pass on results to the defered
deferred = Deferred()
def _thunk_callback(*args, **kwargs):
self._running_count = self._running_count - 1
del self._waiting_deferreds[deferred]
if len(args) and isinstance(args[0], Exception):
deferred.errback(exception=args[0])
else:
deferred.callback(*args)
# Queue
self._running_count = self._running_count + 1
async_result = self._pool.apply_async(_task_thunk, [task],
callback=_thunk_callback)
self._waiting_deferreds[deferred] = async_result
return deferred
class RuleContext(object):
"""A runtime context for an individual rule.
Must contain all of the state for a rule while it is being run, including
all resolved inputs and resulting outputs (once complete).
"""
def __init__(self, build_context, rule, *args, **kwargs):
"""Initializes a rule context.
Args:
build_context: BuildContext this rule is running in.
rule: Rule this context wraps.
"""
self.build_context = build_context
self.build_env = build_context.build_env
self.rule = rule
self.file_delta = None
self.deferred = Deferred()
self.status = Status.WAITING
self.start_time = None
self.end_time = None
self.exception = None
# TODO(benvanik): logger
self.logger = None
# Resolve all src paths
# If rules have their own attrs they'll have to do them themselves
self.src_paths = self._resolve_input_files(rule.srcs, apply_src_filter=True)
# This list of all files this rule outputted, upon completion
self.all_output_files = []
def _get_rule_path(self):
"""Gets the path to the directory containing the rule.
Returns:
Path to the directory that contains the module that this rule is in.
"""
return os.path.dirname(self.rule.parent_module.path)
def _resolve_input_files(self, paths, apply_src_filter=False):
"""Resolves the given paths into real file system paths, ready for use.
This adds direct file references, recursively enumerates paths, expands
globs, and grabs outputs from other rules.
Since this actually checks to see if specific files are present and raises
if not, this should be called in the initializer of all subclasses to
resolve all paths in a place where a good stack will occur.
Note that the resulting list is not deduplicated - certain rules may want
the exact list in the exact order defined. If you want a de-duped list,
simply use list(set(result)) to quickly de-dupe.
Args:
paths: Paths to resolve.
Returns:
A list of all file paths from the given paths.
Raises:
KeyError: A required rule was not found.
OSError: A source path was not found or could not be accessed.
RuntimeError: Internal runtime error (rule executed out of order/etc)
"""
include_filter_list = []
if apply_src_filter and self.rule.src_filter:
include_filter_list = self.rule.src_filter.split('|')
exclude_filter_list = []
if apply_src_filter and self.rule.src_exclude_filter:
exclude_filter_list = self.rule.src_exclude_filter.split('|')
base_path = os.path.dirname(self.rule.parent_module.path)
input_paths = []
for src in paths:
# Grab all items from the source
src_items = None
if util.is_rule_path(src):
# Reference to another rule
other_rule = self.build_context.project.resolve_rule(
src, requesting_module=self.rule.parent_module)
if not other_rule:
raise KeyError('Source rule "%s" not found' % (src))
if not self.build_context.rule_contexts.has_key(other_rule.path):
raise RuntimeError('Source rule "%s" not yet executed' % (src))
other_rule_ctx = self.build_context.rule_contexts[other_rule.path]
src_items = other_rule_ctx.all_output_files
else:
# File or folder path
src_path = os.path.join(base_path, src)
if not os.path.exists(src_path):
raise OSError('Source path "%s" not found' % (src_path))
elif os.path.isdir(src_path):
src_items = [os.path.join(src_path, child)
for child in os.listdir(src_path)]
else:
src_items = [src_path]
# Apply the src_filter, if any
if len(include_filter_list) or len(exclude_filter_list):
for file_path in src_items:
file_name = os.path.basename(file_path)
if any(fnmatch.fnmatch(file_name, f) for f in exclude_filter_list):
continue
if (not len(include_filter_list) or
any(fnmatch.fnmatch(file_name, f) for f in include_filter_list)):
input_paths.append(file_path)
else:
input_paths.extend(src_items)
return input_paths
def __get_target_path(self, base_path, name=None, suffix=None):
"""Handling of _get_*_path() methods.
Args:
base_path: Base path to the project root.
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file.
"""
if not name or not len(name):
name = self.rule.name
if suffix and len(suffix):
name += suffix
root_path = self.build_context.build_env.root_path
module_path = os.path.dirname(self.rule.parent_module.path)
rel_path = os.path.relpath(module_path, root_path)
return os.path.normpath(os.path.join(base_path, rel_path, name))
def _get_root_path(self, name=None, suffix=None):
"""Gets the path of the given output in the root path.
Always prefer _get_out_path and _get_gen_path.
Args:
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file to the proper root path.
"""
base_path = self.build_context.build_env.root_path
return self.__get_target_path(base_path, name=name, suffix=suffix)
def _get_out_path(self, name=None, suffix=None):
"""Gets the 'out' path for an output.
If no name is provided then the rule name will be used.
The 'out' path should be used for all content/binary results.
Args:
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file to the proper 'out' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-out')
return self.__get_target_path(base_path, name=name, suffix=suffix)
def _get_gen_path(self, name=None, suffix=None):
"""Gets the 'gen' path for an output.
If no name is provided then the rule name will be used.
The 'gen' path should be used for generated code only.
Args:
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file to the proper 'gen' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-gen')
return self.__get_target_path(base_path, name=name, suffix=suffix)
def __get_target_path_for_src(self, base_path, src, opt_path=None):
"""Handling of _get_*_path_for_src() methods.
Args:
base_path: Base path to the project root.
src: Absolute source path.
Returns:
A full path that can be used to write a file.
"""
root_path = self.build_context.build_env.root_path
rel_path = os.path.relpath(src, root_path)
rel_path = util.strip_build_paths(rel_path)
return os.path.normpath(os.path.join(base_path, rel_path))
def _get_out_path_for_src(self, src):
"""Gets the 'out' path for a source file.
The 'out' path should be used for all content/binary results.
Args:
src: Absolute source path.
Returns:
A full path that can be used to write a file to the proper 'out' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-out')
return self.__get_target_path_for_src(base_path, src)
def _get_gen_path_for_src(self, src):
"""Gets the 'gen' path for a source file.
The 'gen' path should be used for generated code only.
Args:
src: Absolute source path.
Returns:
A full path that can be used to write a file to the proper 'gen' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-gen')
return self.__get_target_path_for_src(base_path, src)
def _ensure_output_exists(self, path):
"""Makes the given path exist, if it doesn't.
Arg:
path: An absolute path to a folder that should exist.
"""
if not os.path.isdir(path):
os.makedirs(path)
def _append_output_paths(self, paths):
"""Appends the given paths to the output list.
Other rules that depend on this rule will receive these paths when it
is used as a source.
Args:
paths: A list of paths to add to the list.
"""
self.all_output_files.extend(paths)
def _run_task_async(self, task):
"""Runs a task asynchronously.
This is a utility method that makes it easier to execute tasks.
Args:
task: Task to execute.
Returns:
A deferred that signals when the task completes.
"""
return self.build_context.task_executor.run_task_async(task)
def check_predecessor_failures(self):
"""Checks all dependencies for failure.
Returns:
True if any dependency has failed.
"""
for dep in self.rule.get_dependent_paths():
if util.is_rule_path(dep):
other_rule = self.build_context.project.resolve_rule(
dep, requesting_module=self.rule.parent_module)
other_rule_ctx = self.build_context.rule_contexts.get(
other_rule.path, None)
if other_rule_ctx.status == Status.FAILED:
return True
return False
def begin(self):
"""Begins asynchronous rule execution.
Custom RuleContext implementations should override this method to perform
their behavior (spawning tasks/etc). When the returned Deferred is called
back the rule context should be completed, with all_output_files properly
set.
The default implementation ends immediately, passing all input files through
as output.
Returns:
A Deferred that can will be called back when the rule has completed.
"""
self.status = Status.RUNNING
self.start_time = util.timer()
# TODO(benvanik): real logging of rule processing
# This makes the path relative and strips BUILD: (which may be wrong, w/e)
rel_path = os.path.relpath(self.rule.path, self.build_env.root_path)
rel_path = util.strip_implicit_build_name(rel_path)
print '... %20s ~ %s' % (self.rule.rule_name, rel_path)
# Compute file delta
# Note that this could be done async (somehow)
self.file_delta = self.build_context.cache.compute_delta(
self.rule.path, 'src', self.src_paths)
return self.deferred
def _check_if_cached(self):
"""Checks if all inputs and outputs match their expected values.
Returns:
True if no inputs or outputs have changed.
"""
# If any input changed...
if self.file_delta.any_changes():
return False
# If any output was removed...
output_delta = self.build_context.cache.compute_delta(
self.rule.path, 'out', self.all_output_files)
if len(output_delta.removed_files):
return False
return True
def cascade_failure(self):
"""Instantly fails a rule, signaling that a rule prior to it has failed
and it should not be run.
Use this if a call to check_predecessor_failures returns True to properly
set a rule context up for cascading failures.
After calling this begin should not be called.
Returns:
A Deferred that has had its errback called.
"""
# TODO(benvanik): special CascadingError exception
self.start_time = util.timer()
self._fail()
return self.deferred
def _succeed(self):
"""Signals that rule execution has completed successfully.
This will set all state and issue the callback on the deferred.
"""
self.status = Status.SUCCEEDED
self.end_time = util.timer()
self.deferred.callback()
def _fail(self, exception=None, *args, **kwargs):
"""Signals that rule execution has completed in failure.
This will set all state and issue the errback on the deferred.
If an exception is provided it will be set on the context and passed as
the first argument to the deferred.
Args:
exception: The exception that resulted in the rule failure, if any.
"""
self.status = Status.FAILED
self.end_time = util.timer()
self.exception = exception
# TODO(benvanik): real logging of rule failure
print '!! failed %s' % (self.rule)
if exception:
self.deferred.errback(exception=exception)
else:
self.deferred.errback()
def _chain(self, deferreds):
"""Chains the completion of the rule on the given deferred.
Depending on the success or failure the deferred, the rule context will
succeeed or fail.
Args:
deferreds: A Deferred or list of deferreds that will be called back.
"""
deferred = async.gather_deferreds(deferreds, errback_if_any_fail=True)
def _callback(*args, **kwargs):
self._succeed()
deferred.add_callback_fn(_callback)
self._chain_errback(deferred)
def _chain_errback(self, deferred):
"""Chains an errback of a deferred to the failure of the rule.
Args:
deferred: A Deferred to listen for.
"""
def _errback(*args, **kwargs):
exception = None
if len(args):
for arg in args[0]:
if not arg[0]:
if len(arg[1]) and isinstance(arg[1][0], Exception):
exception = arg[1][0]
break
exception = arg[2].get('exception', None)
if exception:
break
self._fail(exception=exception)
deferred.add_errback_fn(_errback)
class RuleContext(object):
"""A runtime context for an individual rule.
Must contain all of the state for a rule while it is being run, including
all resolved inputs and resulting outputs (once complete).
"""
def __init__(self, build_context, rule, *args, **kwargs):
"""Initializes a rule context.
Args:
build_context: BuildContext this rule is running in.
rule: Rule this context wraps.
"""
self.build_context = build_context
self.build_env = build_context.build_env
self.rule = rule
self.file_delta = None
self.deferred = Deferred()
self.status = Status.WAITING
self.start_time = None
self.end_time = None
self.exception = None
# TODO(benvanik): logger
self.logger = None
# Resolve all src paths
# If rules have their own attrs they'll have to do them themselves
self.src_paths = self._resolve_input_files(rule.srcs,
apply_src_filter=True)
# This list of all files this rule outputted, upon completion
self.all_output_files = []
def _get_rule_path(self):
"""Gets the path to the directory containing the rule.
Returns:
Path to the directory that contains the module that this rule is in.
"""
return os.path.dirname(self.rule.parent_module.path)
def _resolve_input_files(self, paths, apply_src_filter=False):
"""Resolves the given paths into real file system paths, ready for use.
This adds direct file references, recursively enumerates paths, expands
globs, and grabs outputs from other rules.
Since this actually checks to see if specific files are present and raises
if not, this should be called in the initializer of all subclasses to
resolve all paths in a place where a good stack will occur.
Note that the resulting list is not deduplicated - certain rules may want
the exact list in the exact order defined. If you want a de-duped list,
simply use list(set(result)) to quickly de-dupe.
Args:
paths: Paths to resolve.
Returns:
A list of all file paths from the given paths.
Raises:
KeyError: A required rule was not found.
OSError: A source path was not found or could not be accessed.
RuntimeError: Internal runtime error (rule executed out of order/etc)
"""
include_filter_list = []
if apply_src_filter and self.rule.src_filter:
include_filter_list = self.rule.src_filter.split('|')
exclude_filter_list = []
if apply_src_filter and self.rule.src_exclude_filter:
exclude_filter_list = self.rule.src_exclude_filter.split('|')
base_path = os.path.dirname(self.rule.parent_module.path)
input_paths = []
for src in paths:
# Grab all items from the source
src_items = None
if util.is_rule_path(src):
# Reference to another rule
other_rule = self.build_context.project.resolve_rule(
src, requesting_module=self.rule.parent_module)
if not other_rule:
raise KeyError('Source rule "%s" not found' % (src))
if not self.build_context.rule_contexts.has_key(
other_rule.path):
raise RuntimeError('Source rule "%s" not yet executed' %
(src))
other_rule_ctx = self.build_context.rule_contexts[
other_rule.path]
src_items = other_rule_ctx.all_output_files
else:
# File or folder path
src_path = os.path.join(base_path, src)
if not os.path.exists(src_path):
raise OSError('Source path "%s" not found' % (src_path))
elif os.path.isdir(src_path):
src_items = [
os.path.join(src_path, child)
for child in os.listdir(src_path)
]
else:
src_items = [src_path]
# Apply the src_filter, if any
if len(include_filter_list) or len(exclude_filter_list):
for file_path in src_items:
file_name = os.path.basename(file_path)
if any(
fnmatch.fnmatch(file_name, f)
for f in exclude_filter_list):
continue
if (not len(include_filter_list) or any(
fnmatch.fnmatch(file_name, f)
for f in include_filter_list)):
input_paths.append(file_path)
else:
input_paths.extend(src_items)
return input_paths
def __get_target_path(self, base_path, name=None, suffix=None):
"""Handling of _get_*_path() methods.
Args:
base_path: Base path to the project root.
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file.
"""
if not name or not len(name):
name = self.rule.name
if suffix and len(suffix):
name += suffix
root_path = self.build_context.build_env.root_path
module_path = os.path.dirname(self.rule.parent_module.path)
rel_path = os.path.relpath(module_path, root_path)
return os.path.normpath(os.path.join(base_path, rel_path, name))
def _get_root_path(self, name=None, suffix=None):
"""Gets the path of the given output in the root path.
Always prefer _get_out_path and _get_gen_path.
Args:
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file to the proper root path.
"""
base_path = self.build_context.build_env.root_path
return self.__get_target_path(base_path, name=name, suffix=suffix)
def _get_out_path(self, name=None, suffix=None):
"""Gets the 'out' path for an output.
If no name is provided then the rule name will be used.
The 'out' path should be used for all content/binary results.
Args:
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file to the proper 'out' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-out')
return self.__get_target_path(base_path, name=name, suffix=suffix)
def _get_gen_path(self, name=None, suffix=None):
"""Gets the 'gen' path for an output.
If no name is provided then the rule name will be used.
The 'gen' path should be used for generated code only.
Args:
name: If a name is provided it will be used instead of the rule name.
suffix: Suffix to add to whatever path is built, such as '.txt' to add
an extension.
Returns:
A full path that can be used to write a file to the proper 'gen' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-gen')
return self.__get_target_path(base_path, name=name, suffix=suffix)
def __get_target_path_for_src(self, base_path, src, opt_path=None):
"""Handling of _get_*_path_for_src() methods.
Args:
base_path: Base path to the project root.
src: Absolute source path.
Returns:
A full path that can be used to write a file.
"""
root_path = self.build_context.build_env.root_path
rel_path = os.path.relpath(src, root_path)
rel_path = util.strip_build_paths(rel_path)
return os.path.normpath(os.path.join(base_path, rel_path))
def _get_out_path_for_src(self, src):
"""Gets the 'out' path for a source file.
The 'out' path should be used for all content/binary results.
Args:
src: Absolute source path.
Returns:
A full path that can be used to write a file to the proper 'out' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-out')
return self.__get_target_path_for_src(base_path, src)
def _get_gen_path_for_src(self, src):
"""Gets the 'gen' path for a source file.
The 'gen' path should be used for generated code only.
Args:
src: Absolute source path.
Returns:
A full path that can be used to write a file to the proper 'gen' path.
"""
base_path = os.path.join(self.build_context.build_env.root_path,
'build-gen')
return self.__get_target_path_for_src(base_path, src)
def _ensure_output_exists(self, path):
"""Makes the given path exist, if it doesn't.
Arg:
path: An absolute path to a folder that should exist.
"""
if not os.path.isdir(path):
os.makedirs(path)
def _append_output_paths(self, paths):
"""Appends the given paths to the output list.
Other rules that depend on this rule will receive these paths when it
is used as a source.
Args:
paths: A list of paths to add to the list.
"""
self.all_output_files.extend(paths)
def _run_task_async(self, task):
"""Runs a task asynchronously.
This is a utility method that makes it easier to execute tasks.
Args:
task: Task to execute.
Returns:
A deferred that signals when the task completes.
"""
return self.build_context.task_executor.run_task_async(task)
def check_predecessor_failures(self):
"""Checks all dependencies for failure.
Returns:
True if any dependency has failed or been interrupted.
"""
for dep in self.rule.get_dependent_paths():
if util.is_rule_path(dep):
other_rule = self.build_context.project.resolve_rule(
dep, requesting_module=self.rule.parent_module)
other_rule_ctx = self.build_context.rule_contexts.get(
other_rule.path, None)
if (other_rule_ctx.status == Status.FAILED):
return True
return False
def begin(self):
"""Begins asynchronous rule execution.
Custom RuleContext implementations should override this method to perform
their behavior (spawning tasks/etc). When the returned Deferred is called
back the rule context should be completed, with all_output_files properly
set.
The default implementation ends immediately, passing all input files through
as output.
Returns:
A Deferred that can will be called back when the rule has completed.
"""
self.status = Status.RUNNING
self.start_time = util.timer()
# TODO(benvanik): real logging of rule processing
# This makes the path relative and strips BUILD: (which may be wrong, w/e)
rel_path = os.path.relpath(self.rule.path, self.build_env.root_path)
rel_path = util.strip_implicit_build_name(rel_path)
print '... %20s ~ %s' % (self.rule.rule_name, rel_path)
# Compute file delta
# Note that this could be done async (somehow)
self.file_delta = self.build_context.cache.compute_delta(
self.rule.path, 'src', self.src_paths)
return self.deferred
def _check_if_cached(self):
"""Checks if all inputs and outputs match their expected values.
Returns:
True if no inputs or outputs have changed.
"""
# If -f (force) was passed to the BuildContext, return False.
if self.build_context.force:
return False
# If the rule does not have any source or output files, then nothing can
# be cached. Return False.
if not self.src_paths and not self.all_output_files:
return False
# If any input changed...
if self.file_delta.any_changes():
return False
# If any output was removed...
output_delta = self.build_context.cache.compute_delta(
self.rule.path, 'out', self.all_output_files)
if len(output_delta.removed_files):
return False
return True
def cascade_failure(self):
"""Instantly fails a rule, signaling that a rule prior to it has failed
and it should not be run.
Use this if a call to check_predecessor_failures returns True to properly
set a rule context up for cascading failures.
After calling this begin should not be called.
Returns:
A Deferred that has had its errback called.
"""
# TODO(benvanik): special CascadingError exception
self.start_time = util.timer()
self._fail()
return self.deferred
def _succeed(self):
"""Signals that rule execution has completed successfully.
This will set all state and issue the callback on the deferred.
"""
self.status = Status.SUCCEEDED
self.end_time = util.timer()
self.deferred.callback()
def _fail(self, exception=None, *args, **kwargs):
"""Signals that rule execution has completed in failure.
This will set all state and issue the errback on the deferred.
If an exception is provided it will be set on the context and passed as
the first argument to the deferred.
Args:
exception: The exception that resulted in the rule failure, if any.
"""
self.status = Status.FAILED
self.end_time = util.timer()
self.exception = exception
# TODO(benvanik): real logging of rule failure
print '!! failed %s' % (self.rule)
if exception:
self.deferred.errback(exception=exception)
else:
self.deferred.errback()
def _chain(self, deferreds):
"""Chains the completion of the rule on the given deferred.
Depending on the success or failure the deferred, the rule context will
succeeed or fail.
Args:
deferreds: A Deferred or list of deferreds that will be called back.
"""
deferred = async .gather_deferreds(deferreds, errback_if_any_fail=True)
def _callback(*args, **kwargs):
self._succeed()
deferred.add_callback_fn(_callback)
self._chain_errback(deferred)
def _chain_errback(self, deferred):
"""Chains an errback of a deferred to the failure of the rule.
Args:
deferred: A Deferred to listen for.
"""
def _errback(*args, **kwargs):
exception = None
if len(args):
for arg in args[0]:
if not arg[0]:
if len(arg[1]) and isinstance(arg[1][0], Exception):
exception = arg[1][0]
break
exception = arg[2].get('exception', None)
if exception:
break
self._fail(exception=exception)
deferred.add_errback_fn(_errback)
def testCallbackOrder(self):
cb = {}
def cb_thunk(*args, **kwargs):
cb['done'] = True
cb['args'] = args
cb['kwargs'] = kwargs
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback('a')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.callback('a')
d.add_callback_fn(cb_thunk)
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_errback_fn(cb_thunk)
d.errback('a')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.errback('a')
d.add_errback_fn(cb_thunk)
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_callback_fn(cb_thunk)
d.errback('a')
self.assertEqual(len(cb), 0)
cb.clear()
d = Deferred()
d.errback('a')
d.add_callback_fn(cb_thunk)
self.assertEqual(len(cb), 0)
cb.clear()
d = Deferred()
d.add_errback_fn(cb_thunk)
d.callback('a')
self.assertEqual(len(cb), 0)
cb.clear()
d = Deferred()
d.callback('a')
d.add_errback_fn(cb_thunk)
self.assertEqual(len(cb), 0)
cb.clear()
def testMultiCall(self):
d = Deferred()
d.callback()
with self.assertRaises(AssertionError):
d.callback()
d = Deferred()
d.errback()
with self.assertRaises(AssertionError):
d.errback()
d = Deferred()
d.callback()
with self.assertRaises(AssertionError):
d.errback()
d = Deferred()
d.errback()
with self.assertRaises(AssertionError):
d.callback()
def testErrback(self):
d = gather_deferreds([], errback_if_any_fail=True)
self.assertCallbackEqual(d, [])
da = Deferred()
db = Deferred()
dc = Deferred()
d = gather_deferreds([da, db, dc], errback_if_any_fail=True)
dc.callback('c')
db.callback('b')
da.callback('a')
self.assertCallbackEqual(d, [
(True, ('a',), {}),
(True, ('b',), {}),
(True, ('c',), {})])
da = Deferred()
db = Deferred()
dc = Deferred()
df = Deferred()
d = gather_deferreds([da, db, dc, df], errback_if_any_fail=True)
df.errback()
dc.callback('c')
db.callback('b')
da.callback('a')
self.assertErrbackEqual(d, [
(True, ('a',), {}),
(True, ('b',), {}),
(True, ('c',), {}),
(False, (), {})])
da = Deferred()
db = Deferred()
dc = Deferred()
df = Deferred()
df.errback('f')
dc.callback('c')
d = gather_deferreds([da, db, dc, df], errback_if_any_fail=True)
db.callback('b')
da.callback('a')
self.assertErrbackEqual(d, [
(True, ('a',), {}),
(True, ('b',), {}),
(True, ('c',), {}),
(False, ('f',), {})])
def testMultiCall(self):
d = Deferred()
d.callback()
with self.assertRaises(AssertionError):
d.callback()
d = Deferred()
d.errback()
with self.assertRaises(AssertionError):
d.errback()
d = Deferred()
d.callback()
with self.assertRaises(AssertionError):
d.errback()
d = Deferred()
d.errback()
with self.assertRaises(AssertionError):
d.callback()
def testMultiCallbacks(self):
cbs = []
def cb_multi_thunk(*args, **kwargs):
cbs.append({
'done': True,
'args': args,
'kwargs': kwargs
})
d = Deferred()
d.add_callback_fn(cb_multi_thunk)
d.callback('a')
self.assertEqual(len(cbs), 1)
self.assertNotEqual(len(cbs[0]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
cbs[:] = []
d = Deferred()
d.add_callback_fn(cb_multi_thunk)
d.add_callback_fn(cb_multi_thunk)
d.callback('a')
self.assertEqual(len(cbs), 2)
self.assertNotEqual(len(cbs[0]), 0)
self.assertNotEqual(len(cbs[1]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
self.assertEqual(cbs[1]['args'][0], 'a')
cbs[:] = []
d = Deferred()
d.add_callback_fn(cb_multi_thunk)
d.callback('a')
d.add_callback_fn(cb_multi_thunk)
self.assertEqual(len(cbs), 2)
self.assertNotEqual(len(cbs[0]), 0)
self.assertNotEqual(len(cbs[1]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
self.assertEqual(cbs[1]['args'][0], 'a')
cbs[:] = []
d = Deferred()
d.callback('a')
d.add_callback_fn(cb_multi_thunk)
d.add_callback_fn(cb_multi_thunk)
self.assertEqual(len(cbs), 2)
self.assertNotEqual(len(cbs[0]), 0)
self.assertNotEqual(len(cbs[1]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
self.assertEqual(cbs[1]['args'][0], 'a')
cbs[:] = []
def testCallbackOrder(self):
cb = {}
def cb_thunk(*args, **kwargs):
cb['done'] = True
cb['args'] = args
cb['kwargs'] = kwargs
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback('a')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.callback('a')
d.add_callback_fn(cb_thunk)
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_errback_fn(cb_thunk)
d.errback('a')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.errback('a')
d.add_errback_fn(cb_thunk)
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_callback_fn(cb_thunk)
d.errback('a')
self.assertEqual(len(cb), 0)
cb.clear()
d = Deferred()
d.errback('a')
d.add_callback_fn(cb_thunk)
self.assertEqual(len(cb), 0)
cb.clear()
d = Deferred()
d.add_errback_fn(cb_thunk)
d.callback('a')
self.assertEqual(len(cb), 0)
cb.clear()
d = Deferred()
d.callback('a')
d.add_errback_fn(cb_thunk)
self.assertEqual(len(cb), 0)
cb.clear()
def testCallbackArgs(self):
cb = {}
def cb_thunk(*args, **kwargs):
cb['done'] = True
cb['args'] = args
cb['kwargs'] = kwargs
d = Deferred()
self.assertFalse(d.is_done())
d.callback()
self.assertTrue(d.is_done())
d = Deferred()
self.assertFalse(d.is_done())
d.errback()
self.assertTrue(d.is_done())
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback()
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 0)
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback('a', 'b')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 2)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(cb['args'][1], 'b')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback('a', b='b')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 1)
self.assertEqual(cb['kwargs']['b'], 'b')
cb.clear()
def testErrback(self):
d = gather_deferreds([], errback_if_any_fail=True)
self.assertCallbackEqual(d, [])
da = Deferred()
db = Deferred()
dc = Deferred()
d = gather_deferreds([da, db, dc], errback_if_any_fail=True)
dc.callback('c')
db.callback('b')
da.callback('a')
self.assertCallbackEqual(d, [(True, ('a', ), {}), (True, ('b', ), {}),
(True, ('c', ), {})])
da = Deferred()
db = Deferred()
dc = Deferred()
df = Deferred()
d = gather_deferreds([da, db, dc, df], errback_if_any_fail=True)
df.errback()
dc.callback('c')
db.callback('b')
da.callback('a')
self.assertErrbackEqual(d, [(True, ('a', ), {}), (True, ('b', ), {}),
(True, ('c', ), {}), (False, (), {})])
da = Deferred()
db = Deferred()
dc = Deferred()
df = Deferred()
df.errback('f')
dc.callback('c')
d = gather_deferreds([da, db, dc, df], errback_if_any_fail=True)
db.callback('b')
da.callback('a')
self.assertErrbackEqual(d, [(True, ('a', ), {}), (True, ('b', ), {}),
(True, ('c', ), {}), (False, ('f', ), {})])
def testCallbackArgs(self):
cb = {}
def cb_thunk(*args, **kwargs):
cb['done'] = True
cb['args'] = args
cb['kwargs'] = kwargs
d = Deferred()
self.assertFalse(d.is_done())
d.callback()
self.assertTrue(d.is_done())
d = Deferred()
self.assertFalse(d.is_done())
d.errback()
self.assertTrue(d.is_done())
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback()
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 0)
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback('a', 'b')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 2)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(cb['args'][1], 'b')
self.assertEqual(len(cb['kwargs']), 0)
cb.clear()
d = Deferred()
d.add_callback_fn(cb_thunk)
d.callback('a', b='b')
self.assertNotEqual(len(cb), 0)
self.assertTrue(cb['done'])
self.assertEqual(len(cb['args']), 1)
self.assertEqual(cb['args'][0], 'a')
self.assertEqual(len(cb['kwargs']), 1)
self.assertEqual(cb['kwargs']['b'], 'b')
cb.clear()
def testMultiCallbacks(self):
cbs = []
def cb_multi_thunk(*args, **kwargs):
cbs.append({'done': True, 'args': args, 'kwargs': kwargs})
d = Deferred()
d.add_callback_fn(cb_multi_thunk)
d.callback('a')
self.assertEqual(len(cbs), 1)
self.assertNotEqual(len(cbs[0]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
cbs[:] = []
d = Deferred()
d.add_callback_fn(cb_multi_thunk)
d.add_callback_fn(cb_multi_thunk)
d.callback('a')
self.assertEqual(len(cbs), 2)
self.assertNotEqual(len(cbs[0]), 0)
self.assertNotEqual(len(cbs[1]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
self.assertEqual(cbs[1]['args'][0], 'a')
cbs[:] = []
d = Deferred()
d.add_callback_fn(cb_multi_thunk)
d.callback('a')
d.add_callback_fn(cb_multi_thunk)
self.assertEqual(len(cbs), 2)
self.assertNotEqual(len(cbs[0]), 0)
self.assertNotEqual(len(cbs[1]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
self.assertEqual(cbs[1]['args'][0], 'a')
cbs[:] = []
d = Deferred()
d.callback('a')
d.add_callback_fn(cb_multi_thunk)
d.add_callback_fn(cb_multi_thunk)
self.assertEqual(len(cbs), 2)
self.assertNotEqual(len(cbs[0]), 0)
self.assertNotEqual(len(cbs[1]), 0)
self.assertEqual(cbs[0]['args'][0], 'a')
self.assertEqual(cbs[1]['args'][0], 'a')
cbs[:] = []
