def _ensure_rules_present(self, rule_paths, requesting_module=None): """Ensures that the given list of rules are present in the graph, and if not recursively loads them. Args: rule_paths: A list of target rule paths to add to the graph. requesting_module: Module that is requesting the given rules or None if all rule paths are absolute. """ # Add all of the rules listed rules = [] for rule_path in rule_paths: # Attempt to resolve the rule rule = self.project.resolve_rule( rule_path, requesting_module=requesting_module) if not rule: raise KeyError('Rule "%s" unable to be resolved' % (rule_path)) rules.append(rule) # If already present, ignore (no need to recurse) if rule.path in self.rule_nodes: continue # Add node to the graph self.rule_nodes[rule.path] = rule self.graph.add_node(rule.path) # Recursively resolve all dependent rules dependent_rule_paths = [] for dep in rule.get_dependent_paths(): if util.is_rule_path(dep): dependent_rule_paths.append(dep) if len(dependent_rule_paths): self._ensure_rules_present( dependent_rule_paths, requesting_module=rule.parent_module) # Add edges for all of the requested rules (at this point, all rules should # be added to the graph) for rule in rules: for dep in rule.get_dependent_paths(): if util.is_rule_path(dep): dep_rule = self.project.resolve_rule( dep, requesting_module=rule.parent_module) # Node should exist due to recursive addition above assert dep_rule.path in self.rule_nodes self.graph.add_edge(dep_rule.path, rule.path) # Ensure the graph is a DAG (no cycles) if not nx.is_directed_acyclic_graph(self.graph): # TODO(benvanik): use nx.simple_cycles() to print the cycles raise ValueError('Cycle detected in the rule graph: %s' % (nx.simple_cycles(self.graph)))
def _ensure_rules_present(self, rule_paths, requesting_module=None): """Ensures that the given list of rules are present in the graph, and if not recursively loads them. Args: rule_paths: A list of target rule paths to add to the graph. requesting_module: Module that is requesting the given rules or None if all rule paths are absolute. """ # Add all of the rules listed rules = [] for rule_path in rule_paths: # Attempt to resolve the rule rule = self.project.resolve_rule(rule_path, requesting_module=requesting_module) if not rule: raise KeyError('Rule "%s" unable to be resolved' % (rule_path)) rules.append(rule) # If already present, ignore (no need to recurse) if rule.path in self.rule_nodes: continue # Add node to the graph self.rule_nodes[rule.path] = rule self.graph.add_node(rule.path) # Recursively resolve all dependent rules dependent_rule_paths = [] for dep in rule.get_dependent_paths(): if util.is_rule_path(dep): dependent_rule_paths.append(dep) if len(dependent_rule_paths): self._ensure_rules_present(dependent_rule_paths, requesting_module=rule.parent_module) # Add edges for all of the requested rules (at this point, all rules should # be added to the graph) for rule in rules: for dep in rule.get_dependent_paths(): if util.is_rule_path(dep): dep_rule = self.project.resolve_rule(dep, requesting_module=rule.parent_module) # Node should exist due to recursive addition above assert dep_rule.path in self.rule_nodes self.graph.add_edge(dep_rule.path, rule.path) # Ensure the graph is a DAG (no cycles) if not nx.is_directed_acyclic_graph(self.graph): # TODO(benvanik): use nx.simple_cycles() to print the cycles raise ValueError('Cycle detected in the rule graph: %s' % ( nx.simple_cycles(self.graph)))
def testNames(self): self.assertTrue(util.is_rule_path(':a')) self.assertTrue(util.is_rule_path(':ab')) self.assertTrue(util.is_rule_path('xx:ab')) self.assertTrue(util.is_rule_path('/a/b:ab')) self.assertFalse(util.is_rule_path('a')) self.assertFalse(util.is_rule_path('/a/b.c')) self.assertFalse(util.is_rule_path('a b c'))
def check_predecessor_failures(self): """Checks all dependencies for failure. Returns: True if any dependency has failed or been interrupted. """ for dep in self.rule.get_dependent_paths(): if util.is_rule_path(dep): other_rule = self.build_context.project.resolve_rule(dep, requesting_module=self.rule.parent_module) other_rule_ctx = self.build_context.rule_contexts.get(other_rule.path, None) if other_rule_ctx.status == Status.FAILED: return True return False
def check_predecessor_failures(self): """Checks all dependencies for failure. Returns: True if any dependency has failed or been interrupted. """ for dep in self.rule.get_dependent_paths(): if util.is_rule_path(dep): other_rule = self.build_context.project.resolve_rule( dep, requesting_module=self.rule.parent_module) other_rule_ctx = self.build_context.rule_contexts.get( other_rule.path, None) if (other_rule_ctx.status == Status.FAILED): return True return False
def testTypes(self): self.assertFalse(util.is_rule_path(4)) self.assertFalse(util.is_rule_path(['a'])) self.assertFalse(util.is_rule_path({'a': 1}))
def testEmpty(self): self.assertFalse(util.is_rule_path(None)) self.assertFalse(util.is_rule_path(''))
def _resolve_input_files(self, paths, apply_src_filter=False): """Resolves the given paths into real file system paths, ready for use. This adds direct file references, recursively enumerates paths, expands globs, and grabs outputs from other rules. Since this actually checks to see if specific files are present and raises if not, this should be called in the initializer of all subclasses to resolve all paths in a place where a good stack will occur. Note that the resulting list is not deduplicated - certain rules may want the exact list in the exact order defined. If you want a de-duped list, simply use list(set(result)) to quickly de-dupe. Args: paths: Paths to resolve. Returns: A list of all file paths from the given paths. Raises: KeyError: A required rule was not found. OSError: A source path was not found or could not be accessed. RuntimeError: Internal runtime error (rule executed out of order/etc) """ include_filter_list = [] if apply_src_filter and self.rule.src_filter: include_filter_list = self.rule.src_filter.split('|') exclude_filter_list = [] if apply_src_filter and self.rule.src_exclude_filter: exclude_filter_list = self.rule.src_exclude_filter.split('|') base_path = os.path.dirname(self.rule.parent_module.path) input_paths = [] for src in paths: # Grab all items from the source src_items = None if util.is_rule_path(src): # Reference to another rule other_rule = self.build_context.project.resolve_rule( src, requesting_module=self.rule.parent_module) if not other_rule: raise KeyError('Source rule "%s" not found' % (src)) if not self.build_context.rule_contexts.has_key(other_rule.path): raise RuntimeError('Source rule "%s" not yet executed' % (src)) other_rule_ctx = self.build_context.rule_contexts[other_rule.path] src_items = other_rule_ctx.all_output_files else: # File or folder path src_path = os.path.join(base_path, src) if not os.path.exists(src_path): raise OSError('Source path "%s" not found' % (src_path)) elif os.path.isdir(src_path): src_items = [os.path.join(src_path, child) for child in os.listdir(src_path)] else: src_items = [src_path] # Apply the src_filter, if any if len(include_filter_list) or len(exclude_filter_list): for file_path in src_items: file_name = os.path.basename(file_path) if any(fnmatch.fnmatch(file_name, f) for f in exclude_filter_list): continue if (not len(include_filter_list) or any(fnmatch.fnmatch(file_name, f) for f in include_filter_list)): input_paths.append(file_path) else: input_paths.extend(src_items) return input_paths
def _resolve_input_files(self, paths, apply_src_filter=False): """Resolves the given paths into real file system paths, ready for use. This adds direct file references, recursively enumerates paths, expands globs, and grabs outputs from other rules. Since this actually checks to see if specific files are present and raises if not, this should be called in the initializer of all subclasses to resolve all paths in a place where a good stack will occur. Note that the resulting list is not deduplicated - certain rules may want the exact list in the exact order defined. If you want a de-duped list, simply use list(set(result)) to quickly de-dupe. Args: paths: Paths to resolve. Returns: A list of all file paths from the given paths. Raises: KeyError: A required rule was not found. OSError: A source path was not found or could not be accessed. RuntimeError: Internal runtime error (rule executed out of order/etc) """ include_filter_list = [] if apply_src_filter and self.rule.src_filter: include_filter_list = self.rule.src_filter.split('|') exclude_filter_list = [] if apply_src_filter and self.rule.src_exclude_filter: exclude_filter_list = self.rule.src_exclude_filter.split('|') base_path = os.path.dirname(self.rule.parent_module.path) input_paths = [] for src in paths: # Grab all items from the source src_items = None if util.is_rule_path(src): # Reference to another rule other_rule = self.build_context.project.resolve_rule( src, requesting_module=self.rule.parent_module) if not other_rule: raise KeyError('Source rule "%s" not found' % (src)) if not self.build_context.rule_contexts.has_key( other_rule.path): raise RuntimeError('Source rule "%s" not yet executed' % (src)) other_rule_ctx = self.build_context.rule_contexts[ other_rule.path] src_items = other_rule_ctx.all_output_files else: # File or folder path src_path = os.path.join(base_path, src) if not os.path.exists(src_path): raise OSError('Source path "%s" not found' % (src_path)) elif os.path.isdir(src_path): src_items = [ os.path.join(src_path, child) for child in os.listdir(src_path) ] else: src_items = [src_path] # Apply the src_filter, if any if len(include_filter_list) or len(exclude_filter_list): for file_path in src_items: file_name = os.path.basename(file_path) if any( fnmatch.fnmatch(file_name, f) for f in exclude_filter_list): continue if (not len(include_filter_list) or any( fnmatch.fnmatch(file_name, f) for f in include_filter_list)): input_paths.append(file_path) else: input_paths.extend(src_items) return input_paths