def _ensure_rules_present(self, rule_paths, requesting_module=None):
"""Ensures that the given list of rules are present in the graph, and if not
recursively loads them.
Args:
rule_paths: A list of target rule paths to add to the graph.
requesting_module: Module that is requesting the given rules or None if
all rule paths are absolute.
"""
# Add all of the rules listed
rules = []
for rule_path in rule_paths:
# Attempt to resolve the rule
rule = self.project.resolve_rule(
rule_path, requesting_module=requesting_module)
if not rule:
raise KeyError('Rule "%s" unable to be resolved' % (rule_path))
rules.append(rule)
# If already present, ignore (no need to recurse)
if rule.path in self.rule_nodes:
continue
# Add node to the graph
self.rule_nodes[rule.path] = rule
self.graph.add_node(rule.path)
# Recursively resolve all dependent rules
dependent_rule_paths = []
for dep in rule.get_dependent_paths():
if util.is_rule_path(dep):
dependent_rule_paths.append(dep)
if len(dependent_rule_paths):
self._ensure_rules_present(
dependent_rule_paths, requesting_module=rule.parent_module)
# Add edges for all of the requested rules (at this point, all rules should
# be added to the graph)
for rule in rules:
for dep in rule.get_dependent_paths():
if util.is_rule_path(dep):
dep_rule = self.project.resolve_rule(
dep, requesting_module=rule.parent_module)
# Node should exist due to recursive addition above
assert dep_rule.path in self.rule_nodes
self.graph.add_edge(dep_rule.path, rule.path)
# Ensure the graph is a DAG (no cycles)
if not nx.is_directed_acyclic_graph(self.graph):
# TODO(benvanik): use nx.simple_cycles() to print the cycles
raise ValueError('Cycle detected in the rule graph: %s' %
(nx.simple_cycles(self.graph)))
def _ensure_rules_present(self, rule_paths, requesting_module=None):
"""Ensures that the given list of rules are present in the graph, and if not
recursively loads them.
Args:
rule_paths: A list of target rule paths to add to the graph.
requesting_module: Module that is requesting the given rules or None if
all rule paths are absolute.
"""
# Add all of the rules listed
rules = []
for rule_path in rule_paths:
# Attempt to resolve the rule
rule = self.project.resolve_rule(rule_path,
requesting_module=requesting_module)
if not rule:
raise KeyError('Rule "%s" unable to be resolved' % (rule_path))
rules.append(rule)
# If already present, ignore (no need to recurse)
if rule.path in self.rule_nodes:
continue
# Add node to the graph
self.rule_nodes[rule.path] = rule
self.graph.add_node(rule.path)
# Recursively resolve all dependent rules
dependent_rule_paths = []
for dep in rule.get_dependent_paths():
if util.is_rule_path(dep):
dependent_rule_paths.append(dep)
if len(dependent_rule_paths):
self._ensure_rules_present(dependent_rule_paths,
requesting_module=rule.parent_module)
# Add edges for all of the requested rules (at this point, all rules should
# be added to the graph)
for rule in rules:
for dep in rule.get_dependent_paths():
if util.is_rule_path(dep):
dep_rule = self.project.resolve_rule(dep,
requesting_module=rule.parent_module)
# Node should exist due to recursive addition above
assert dep_rule.path in self.rule_nodes
self.graph.add_edge(dep_rule.path, rule.path)
# Ensure the graph is a DAG (no cycles)
if not nx.is_directed_acyclic_graph(self.graph):
# TODO(benvanik): use nx.simple_cycles() to print the cycles
raise ValueError('Cycle detected in the rule graph: %s' % (
nx.simple_cycles(self.graph)))
def testNames(self):
self.assertTrue(util.is_rule_path(':a'))
self.assertTrue(util.is_rule_path(':ab'))
self.assertTrue(util.is_rule_path('xx:ab'))
self.assertTrue(util.is_rule_path('/a/b:ab'))
self.assertFalse(util.is_rule_path('a'))
self.assertFalse(util.is_rule_path('/a/b.c'))
self.assertFalse(util.is_rule_path('a b c'))
def testNames(self):
self.assertTrue(util.is_rule_path(':a'))
self.assertTrue(util.is_rule_path(':ab'))
self.assertTrue(util.is_rule_path('xx:ab'))
self.assertTrue(util.is_rule_path('/a/b:ab'))
self.assertFalse(util.is_rule_path('a'))
self.assertFalse(util.is_rule_path('/a/b.c'))
self.assertFalse(util.is_rule_path('a b c'))
def check_predecessor_failures(self):
"""Checks all dependencies for failure.
Returns:
True if any dependency has failed or been interrupted.
"""
for dep in self.rule.get_dependent_paths():
if util.is_rule_path(dep):
other_rule = self.build_context.project.resolve_rule(dep, requesting_module=self.rule.parent_module)
other_rule_ctx = self.build_context.rule_contexts.get(other_rule.path, None)
if other_rule_ctx.status == Status.FAILED:
return True
return False
def check_predecessor_failures(self):
"""Checks all dependencies for failure.
Returns:
True if any dependency has failed or been interrupted.
"""
for dep in self.rule.get_dependent_paths():
if util.is_rule_path(dep):
other_rule = self.build_context.project.resolve_rule(
dep, requesting_module=self.rule.parent_module)
other_rule_ctx = self.build_context.rule_contexts.get(
other_rule.path, None)
if (other_rule_ctx.status == Status.FAILED):
return True
return False
def testTypes(self):
self.assertFalse(util.is_rule_path(4))
self.assertFalse(util.is_rule_path(['a']))
self.assertFalse(util.is_rule_path({'a': 1}))
def testEmpty(self):
self.assertFalse(util.is_rule_path(None))
self.assertFalse(util.is_rule_path(''))
def _resolve_input_files(self, paths, apply_src_filter=False):
"""Resolves the given paths into real file system paths, ready for use.
This adds direct file references, recursively enumerates paths, expands
globs, and grabs outputs from other rules.
Since this actually checks to see if specific files are present and raises
if not, this should be called in the initializer of all subclasses to
resolve all paths in a place where a good stack will occur.
Note that the resulting list is not deduplicated - certain rules may want
the exact list in the exact order defined. If you want a de-duped list,
simply use list(set(result)) to quickly de-dupe.
Args:
paths: Paths to resolve.
Returns:
A list of all file paths from the given paths.
Raises:
KeyError: A required rule was not found.
OSError: A source path was not found or could not be accessed.
RuntimeError: Internal runtime error (rule executed out of order/etc)
"""
include_filter_list = []
if apply_src_filter and self.rule.src_filter:
include_filter_list = self.rule.src_filter.split('|')
exclude_filter_list = []
if apply_src_filter and self.rule.src_exclude_filter:
exclude_filter_list = self.rule.src_exclude_filter.split('|')
base_path = os.path.dirname(self.rule.parent_module.path)
input_paths = []
for src in paths:
# Grab all items from the source
src_items = None
if util.is_rule_path(src):
# Reference to another rule
other_rule = self.build_context.project.resolve_rule(
src, requesting_module=self.rule.parent_module)
if not other_rule:
raise KeyError('Source rule "%s" not found' % (src))
if not self.build_context.rule_contexts.has_key(other_rule.path):
raise RuntimeError('Source rule "%s" not yet executed' % (src))
other_rule_ctx = self.build_context.rule_contexts[other_rule.path]
src_items = other_rule_ctx.all_output_files
else:
# File or folder path
src_path = os.path.join(base_path, src)
if not os.path.exists(src_path):
raise OSError('Source path "%s" not found' % (src_path))
elif os.path.isdir(src_path):
src_items = [os.path.join(src_path, child)
for child in os.listdir(src_path)]
else:
src_items = [src_path]
# Apply the src_filter, if any
if len(include_filter_list) or len(exclude_filter_list):
for file_path in src_items:
file_name = os.path.basename(file_path)
if any(fnmatch.fnmatch(file_name, f) for f in exclude_filter_list):
continue
if (not len(include_filter_list) or
any(fnmatch.fnmatch(file_name, f) for f in include_filter_list)):
input_paths.append(file_path)
else:
input_paths.extend(src_items)
return input_paths
def _resolve_input_files(self, paths, apply_src_filter=False):
"""Resolves the given paths into real file system paths, ready for use.
This adds direct file references, recursively enumerates paths, expands
globs, and grabs outputs from other rules.
Since this actually checks to see if specific files are present and raises
if not, this should be called in the initializer of all subclasses to
resolve all paths in a place where a good stack will occur.
Note that the resulting list is not deduplicated - certain rules may want
the exact list in the exact order defined. If you want a de-duped list,
simply use list(set(result)) to quickly de-dupe.
Args:
paths: Paths to resolve.
Returns:
A list of all file paths from the given paths.
Raises:
KeyError: A required rule was not found.
OSError: A source path was not found or could not be accessed.
RuntimeError: Internal runtime error (rule executed out of order/etc)
"""
include_filter_list = []
if apply_src_filter and self.rule.src_filter:
include_filter_list = self.rule.src_filter.split('|')
exclude_filter_list = []
if apply_src_filter and self.rule.src_exclude_filter:
exclude_filter_list = self.rule.src_exclude_filter.split('|')
base_path = os.path.dirname(self.rule.parent_module.path)
input_paths = []
for src in paths:
# Grab all items from the source
src_items = None
if util.is_rule_path(src):
# Reference to another rule
other_rule = self.build_context.project.resolve_rule(
src, requesting_module=self.rule.parent_module)
if not other_rule:
raise KeyError('Source rule "%s" not found' % (src))
if not self.build_context.rule_contexts.has_key(
other_rule.path):
raise RuntimeError('Source rule "%s" not yet executed' %
(src))
other_rule_ctx = self.build_context.rule_contexts[
other_rule.path]
src_items = other_rule_ctx.all_output_files
else:
# File or folder path
src_path = os.path.join(base_path, src)
if not os.path.exists(src_path):
raise OSError('Source path "%s" not found' % (src_path))
elif os.path.isdir(src_path):
src_items = [
os.path.join(src_path, child)
for child in os.listdir(src_path)
]
else:
src_items = [src_path]
# Apply the src_filter, if any
if len(include_filter_list) or len(exclude_filter_list):
for file_path in src_items:
file_name = os.path.basename(file_path)
if any(
fnmatch.fnmatch(file_name, f)
for f in exclude_filter_list):
continue
if (not len(include_filter_list) or any(
fnmatch.fnmatch(file_name, f)
for f in include_filter_list)):
input_paths.append(file_path)
else:
input_paths.extend(src_items)
return input_paths
def testTypes(self):
self.assertFalse(util.is_rule_path(4))
self.assertFalse(util.is_rule_path(['a']))
self.assertFalse(util.is_rule_path({'a': 1}))
def testEmpty(self):
self.assertFalse(util.is_rule_path(None))
self.assertFalse(util.is_rule_path(''))
