def update(self, data): """ Update metadata, handle virtual hierarchy """ # Nothing to do if no data if data is None: return for key, value in sorted(data.items()): # Ensure there are no 'None' keys if key is None: raise utils.FormatError("Invalid key 'None'.") # Handle child attributes if key.startswith('/'): name = key.lstrip('/') # Handle deeper nesting (e.g. keys like /one/two/three) by # extracting only the first level of the hierarchy as name match = re.search("([^/]+)(/.*)", name) if match: name = match.groups()[0] value = {match.groups()[1]: value} # Update existing child or create a new one self.child(name, value) # Update regular attributes else: self.data[key] = value log.debug("Data for '{0}' updated.".format(self)) log.data(pretty(self.data))
def grow(self, path): """ Grow the metadata tree for the given directory path Note: For each path, grow() should be run only once. Growing the tree from the same path multiple times with attribute adding using the "+" sign leads to adding the value more than once! """ if path is None: return path = path.rstrip("/") log.info("Walking through directory {0}".format(os.path.abspath(path))) dirpath, dirnames, filenames = next(os.walk(path)) # Investigate main.fmf as the first file (for correct inheritance) filenames = sorted( [filename for filename in filenames if filename.endswith(SUFFIX)]) try: filenames.insert(0, filenames.pop(filenames.index(MAIN))) except ValueError: pass # Check every metadata file and load data (ignore hidden) for filename in filenames: if filename.startswith("."): continue fullpath = os.path.abspath(os.path.join(dirpath, filename)) log.info("Checking file {0}".format(fullpath)) try: with open(fullpath) as datafile: data = yaml.load(datafile) except yaml.scanner.ScannerError as error: raise (utils.FileError("Failed to parse '{0}'\n{1}".format( fullpath, error))) log.data(pretty(data)) # Handle main.fmf as data for self if filename == MAIN: self.sources.append(fullpath) self.update(data) # Handle other *.fmf files as children else: self.child(os.path.splitext(filename)[0], data, fullpath) # Explore every child directory (ignore hidden dirs and subtrees) for dirname in sorted(dirnames): if dirname.startswith("."): continue # Ignore metadata subtrees if os.path.isdir(os.path.join(path, dirname, SUFFIX)): log.debug("Ignoring metadata tree '{0}'.".format(dirname)) continue self.child(dirname, os.path.join(path, dirname)) # Remove empty children (ignore directories without metadata) for name in list(self.children.keys()): child = self.children[name] if not child.data and not child.children: del (self.children[name]) log.debug("Empty tree '{0}' removed.".format(child.name)) # Apply inheritance when all scattered data are gathered. # This is done only once, from the top parent object. if self.parent is None: self.inherit()
def inherit(self): """ Apply inheritance """ # Preserve original data and merge parent # (original data needed for custom inheritance extensions) self.original_data = self.data self.merge() log.debug("Data for '{0}' inherited.".format(self)) log.data(pretty(self.data)) # Apply inheritance to all children for child in self.children.values(): child.inherit()
def grow(self, path): """ Grow the metadata tree for the given directory path Note: For each path, grow() should be run only once. Growing the tree from the same path multiple times with attribute adding using the "+" sign leads to adding the value more than once! """ if path is None: return path = path.rstrip("/") log.info("Walking through directory {0}".format(path)) try: dirpath, dirnames, filenames = list(os.walk(path))[0] except IndexError: raise utils.FileError( "Unable to walk through the '{0}' directory.".format(path)) children = dict() # Investigate main.fmf as the first file (for correct inheritance) filenames = sorted( [filename for filename in filenames if filename.endswith(SUFFIX)]) try: filenames.insert(0, filenames.pop(filenames.index(MAIN))) except ValueError: pass # Check every metadata file and load data (ignore hidden) for filename in filenames: if filename.startswith("."): continue fullpath = os.path.join(dirpath, filename) log.info("Checking file {0}".format(fullpath)) with open(fullpath) as datafile: data = yaml.load(datafile) log.data(pretty(data)) if filename == MAIN: self.update(data) else: self.child(os.path.splitext(filename)[0], data) # Explore every child directory (ignore hidden) for dirname in sorted(dirnames): if dirname.startswith("."): continue self.child(dirname, os.path.join(path, dirname))
def grow(self, path): """ Grow the metadata tree for the given directory path Note: For each path, grow() should be run only once. Growing the tree from the same path multiple times with attribute adding using the "+" sign leads to adding the value more than once! """ if path != '/': path = path.rstrip("/") if path in IGNORED_DIRECTORIES: # pragma: no cover log.debug("Ignoring '{0}' (special directory).".format(path)) return log.info("Walking through directory {0}".format(os.path.abspath(path))) try: dirpath, dirnames, filenames = next(os.walk(path)) except StopIteration: log.debug("Skipping '{0}' (not accessible).".format(path)) return # Investigate main.fmf as the first file (for correct inheritance) filenames = sorted( [filename for filename in filenames if filename.endswith(SUFFIX)]) try: filenames.insert(0, filenames.pop(filenames.index(MAIN))) except ValueError: pass # Check every metadata file and load data (ignore hidden) for filename in filenames: if filename.startswith("."): continue fullpath = os.path.abspath(os.path.join(dirpath, filename)) log.info("Checking file {0}".format(fullpath)) try: with open(fullpath, encoding='utf-8') as datafile: data = YAML(typ="safe").load(datafile) except (YAMLError, DuplicateKeyError) as error: raise ( utils.FileError(f"Failed to parse '{fullpath}'.\n{error}")) log.data(pretty(data)) # Handle main.fmf as data for self if filename == MAIN: self.sources.append(fullpath) self._raw_data = copy.deepcopy(data) self.update(data) # Handle other *.fmf files as children else: self.child(os.path.splitext(filename)[0], data, fullpath) # Explore every child directory (ignore hidden dirs and subtrees) for dirname in sorted(dirnames): if dirname.startswith("."): continue fulldir = os.path.join(dirpath, dirname) if os.path.islink(fulldir): # According to the documentation, calling os.path.realpath # with strict = True will raise OSError if a symlink loop # is encountered. But it does not do that with a loop with # more than one node fullpath = os.path.realpath(fulldir) if fullpath in self._symlinkdirs: log.debug("Not entering symlink loop {}".format(fulldir)) continue else: self._symlinkdirs.append(fullpath) # Ignore metadata subtrees if os.path.isdir(os.path.join(path, dirname, SUFFIX)): log.debug("Ignoring metadata tree '{0}'.".format(dirname)) continue self.child(dirname, os.path.join(path, dirname)) # Ignore directories with no metadata (remove all child nodes which # do not have children and their data haven't been updated) for name in list(self.children.keys()): child = self.children[name] if not child.children and not child._updated: del (self.children[name]) log.debug("Empty tree '{0}' removed.".format(child.name))
def adjust(self, context, key='adjust', undecided='skip'): """ Adjust tree data based on provided context and rules The 'context' should be an instance of the fmf.context.Context class describing the environment context. By default, the key 'adjust' of each node is inspected for possible rules that should be applied. Provide 'key' to use a custom key instead. Optional 'undecided' parameter can be used to specify what should happen when a rule condition cannot be decided because context dimension is not defined. By default, such rules are skipped. In order to raise the fmf.context.CannotDecide exception in such cases use undecided='raise'. """ # Check context sanity if not isinstance(context, fmf.context.Context): raise utils.GeneralError("Invalid adjust context: '{}'.".format( type(context).__name__)) # Adjust rules should be a dictionary or a list of dictionaries try: rules = copy.deepcopy(self.data[key]) log.debug("Applying adjust rules for '{}'.".format(self)) log.data(rules) if isinstance(rules, dict): rules = [rules] if not isinstance(rules, list): raise utils.FormatError( "Invalid adjust rule format in '{}'. " "Should be a dictionary or a list of dictionaries, " "got '{}'.".format(self.name, type(rules).__name__)) except KeyError: rules = [] # Check and apply each rule for rule in rules: # Rule must be a dictionary if not isinstance(rule, dict): raise utils.FormatError("Adjust rule should be a dictionary.") # There must be a condition defined try: condition = rule.pop('when') except KeyError: raise utils.FormatError("No condition defined in adjust rule.") # The optional 'continue' key should be a bool continue_ = rule.pop('continue', True) if not isinstance(continue_, bool): raise utils.FormatError("The 'continue' value should be bool, " "got '{}'.".format(continue_)) # The 'because' key is reserved for optional comments (ignored) rule.pop('because', None) # Apply remaining rule attributes if context matches try: if context.matches(condition): self._merge_special(self.data, rule) # First matching rule wins, skip the rest unless continue if not continue_: break # Handle undecided rules as requested except fmf.context.CannotDecide: if undecided == 'skip': continue elif undecided == 'raise': raise else: raise utils.GeneralError( "Invalid value for the 'undecided' parameter. Should " "be 'skip' or 'raise', got '{}'.".format(undecided)) # Adjust all child nodes as well for child in self.children.values(): child.adjust(context, key, undecided)
def grow(self, path): """ Grow the metadata tree for the given directory path Note: For each path, grow() should be run only once. Growing the tree from the same path multiple times with attribute adding using the "+" sign leads to adding the value more than once! """ if path is None: return if path != '/': path = path.rstrip("/") if path in IGNORED_DIRECTORIES: # pragma: no cover log.debug("Ignoring '{0}' (special directory).".format(path)) return log.info("Walking through directory {0}".format( os.path.abspath(path))) try: dirpath, dirnames, filenames = next(os.walk(path)) except StopIteration: log.debug("Skipping '{0}' (not accessible).".format(path)) return # Investigate main.fmf as the first file (for correct inheritance) filenames = sorted( [filename for filename in filenames if filename.endswith(SUFFIX)]) try: filenames.insert(0, filenames.pop(filenames.index(MAIN))) except ValueError: pass # Check every metadata file and load data (ignore hidden) for filename in filenames: if filename.startswith("."): continue fullpath = os.path.abspath(os.path.join(dirpath, filename)) log.info("Checking file {0}".format(fullpath)) try: with open(fullpath, encoding='utf-8') as datafile: data = yaml.load(datafile, Loader=YamlLoader) except yaml.error.YAMLError as error: raise(utils.FileError("Failed to parse '{0}'\n{1}".format( fullpath, error))) log.data(pretty(data)) # Handle main.fmf as data for self if filename == MAIN: self.sources.append(fullpath) self._raw_data = data self.update(data) # Handle other *.fmf files as children else: self.child(os.path.splitext(filename)[0], data, fullpath) # Explore every child directory (ignore hidden dirs and subtrees) for dirname in sorted(dirnames): if dirname.startswith("."): continue # Ignore metadata subtrees if os.path.isdir(os.path.join(path, dirname, SUFFIX)): log.debug("Ignoring metadata tree '{0}'.".format(dirname)) continue self.child(dirname, os.path.join(path, dirname)) # Remove empty children (ignore directories without metadata) for name in list(self.children.keys()): child = self.children[name] if not child.data and not child.children: del(self.children[name]) log.debug("Empty tree '{0}' removed.".format(child.name))