def update(self, data):
""" Update metadata, handle virtual hierarchy """
# Nothing to do if no data
if data is None:
return
for key, value in sorted(data.items()):
# Ensure there are no 'None' keys
if key is None:
raise utils.FormatError("Invalid key 'None'.")
# Handle child attributes
if key.startswith('/'):
name = key.lstrip('/')
# Handle deeper nesting (e.g. keys like /one/two/three) by
# extracting only the first level of the hierarchy as name
match = re.search("([^/]+)(/.*)", name)
if match:
name = match.groups()[0]
value = {match.groups()[1]: value}
# Update existing child or create a new one
self.child(name, value)
# Update regular attributes
else:
self.data[key] = value
log.debug("Data for '{0}' updated.".format(self))
log.data(pretty(self.data))
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path is None:
return
path = path.rstrip("/")
log.info("Walking through directory {0}".format(os.path.abspath(path)))
dirpath, dirnames, filenames = next(os.walk(path))
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.abspath(os.path.join(dirpath, filename))
log.info("Checking file {0}".format(fullpath))
try:
with open(fullpath) as datafile:
data = yaml.load(datafile)
except yaml.scanner.ScannerError as error:
raise (utils.FileError("Failed to parse '{0}'\n{1}".format(
fullpath, error)))
log.data(pretty(data))
# Handle main.fmf as data for self
if filename == MAIN:
self.sources.append(fullpath)
self.update(data)
# Handle other *.fmf files as children
else:
self.child(os.path.splitext(filename)[0], data, fullpath)
# Explore every child directory (ignore hidden dirs and subtrees)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
# Ignore metadata subtrees
if os.path.isdir(os.path.join(path, dirname, SUFFIX)):
log.debug("Ignoring metadata tree '{0}'.".format(dirname))
continue
self.child(dirname, os.path.join(path, dirname))
# Remove empty children (ignore directories without metadata)
for name in list(self.children.keys()):
child = self.children[name]
if not child.data and not child.children:
del (self.children[name])
log.debug("Empty tree '{0}' removed.".format(child.name))
# Apply inheritance when all scattered data are gathered.
# This is done only once, from the top parent object.
if self.parent is None:
self.inherit()
def __init__(self, data, name=None, parent=None):
"""
Initialize metadata tree from directory path or data dictionary
Data parameter can be either a string with directory path to be
explored or a dictionary with the values already prepared.
"""
# Bail out if no data and no parent given
if not data and not parent:
raise utils.GeneralError(
"No data or parent provided to initialize the tree.")
# Initialize family relations, object data and source files
self.parent = parent
self.children = dict()
self.data = dict()
self.sources = list()
self.root = None
self.version = utils.VERSION
self.original_data = dict()
self._commit = None
self._raw_data = dict()
# Track whether the data dictionary has been updated
# (needed to prevent removing nodes with an empty dict).
self._updated = False
# Store symlinks in while walking tree in grow() to detect
# symlink loops
if parent is None:
self._symlinkdirs = []
else:
self._symlinkdirs = parent._symlinkdirs
# Special handling for top parent
if self.parent is None:
self.name = "/"
if not isinstance(data, dict):
self._initialize(path=data)
data = self.root
# Handle child node creation
else:
self.root = self.parent.root
self.name = os.path.join(self.parent.name, name)
# Update data from a dictionary (handle empty nodes)
if isinstance(data, dict) or data is None:
self.update(data)
# Grow the tree from a directory path
else:
self.grow(data)
# Apply inheritance when all scattered data are gathered.
# This is done only once, from the top parent object.
if self.parent is None:
self.inherit()
log.debug("New tree '{0}' created.".format(self))
def inherit(self):
""" Apply inheritance """
# Preserve original data and merge parent
# (original data needed for custom inheritance extensions)
self.original_data = self.data
self.merge()
log.debug("Data for '{0}' inherited.".format(self))
log.data(pretty(self.data))
# Apply inheritance to all children
for child in self.children.values():
child.inherit()
def __init__(self, data, name=None, parent=None):
"""
Initialize metadata tree from directory path or data dictionary
Data parameter can be either a string with directory path to be
explored or a dictionary with the values already prepared.
"""
# Bail out if no data and no parent given
if not data and not parent:
raise utils.GeneralError(
"No data or parent provided to initialize the tree.")
# Initialize family relations, object data and source files
self.parent = parent
self.children = dict()
self.data = dict()
self.sources = list()
self.root = None
self.version = utils.VERSION
self.original_data = dict()
# Special handling for top parent
if self.parent is None:
self.name = "/"
if not isinstance(data, dict):
self._initialize(path=data)
data = self.root
# Handle child node creation
else:
self.root = self.parent.root
self.name = os.path.join(self.parent.name, name)
# Initialize data
if isinstance(data, dict):
self.update(data)
else:
self.grow(data)
log.debug("New tree '{0}' created.".format(self))
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path != '/':
path = path.rstrip("/")
if path in IGNORED_DIRECTORIES: # pragma: no cover
log.debug("Ignoring '{0}' (special directory).".format(path))
return
log.info("Walking through directory {0}".format(os.path.abspath(path)))
try:
dirpath, dirnames, filenames = next(os.walk(path))
except StopIteration:
log.debug("Skipping '{0}' (not accessible).".format(path))
return
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.abspath(os.path.join(dirpath, filename))
log.info("Checking file {0}".format(fullpath))
try:
with open(fullpath, encoding='utf-8') as datafile:
data = YAML(typ="safe").load(datafile)
except (YAMLError, DuplicateKeyError) as error:
raise (
utils.FileError(f"Failed to parse '{fullpath}'.\n{error}"))
log.data(pretty(data))
# Handle main.fmf as data for self
if filename == MAIN:
self.sources.append(fullpath)
self._raw_data = copy.deepcopy(data)
self.update(data)
# Handle other *.fmf files as children
else:
self.child(os.path.splitext(filename)[0], data, fullpath)
# Explore every child directory (ignore hidden dirs and subtrees)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
fulldir = os.path.join(dirpath, dirname)
if os.path.islink(fulldir):
# According to the documentation, calling os.path.realpath
# with strict = True will raise OSError if a symlink loop
# is encountered. But it does not do that with a loop with
# more than one node
fullpath = os.path.realpath(fulldir)
if fullpath in self._symlinkdirs:
log.debug("Not entering symlink loop {}".format(fulldir))
continue
else:
self._symlinkdirs.append(fullpath)
# Ignore metadata subtrees
if os.path.isdir(os.path.join(path, dirname, SUFFIX)):
log.debug("Ignoring metadata tree '{0}'.".format(dirname))
continue
self.child(dirname, os.path.join(path, dirname))
# Ignore directories with no metadata (remove all child nodes which
# do not have children and their data haven't been updated)
for name in list(self.children.keys()):
child = self.children[name]
if not child.children and not child._updated:
del (self.children[name])
log.debug("Empty tree '{0}' removed.".format(child.name))
def adjust(self, context, key='adjust', undecided='skip'):
"""
Adjust tree data based on provided context and rules
The 'context' should be an instance of the fmf.context.Context
class describing the environment context. By default, the key
'adjust' of each node is inspected for possible rules that
should be applied. Provide 'key' to use a custom key instead.
Optional 'undecided' parameter can be used to specify what
should happen when a rule condition cannot be decided because
context dimension is not defined. By default, such rules are
skipped. In order to raise the fmf.context.CannotDecide
exception in such cases use undecided='raise'.
"""
# Check context sanity
if not isinstance(context, fmf.context.Context):
raise utils.GeneralError("Invalid adjust context: '{}'.".format(
type(context).__name__))
# Adjust rules should be a dictionary or a list of dictionaries
try:
rules = copy.deepcopy(self.data[key])
log.debug("Applying adjust rules for '{}'.".format(self))
log.data(rules)
if isinstance(rules, dict):
rules = [rules]
if not isinstance(rules, list):
raise utils.FormatError(
"Invalid adjust rule format in '{}'. "
"Should be a dictionary or a list of dictionaries, "
"got '{}'.".format(self.name,
type(rules).__name__))
except KeyError:
rules = []
# Check and apply each rule
for rule in rules:
# Rule must be a dictionary
if not isinstance(rule, dict):
raise utils.FormatError("Adjust rule should be a dictionary.")
# There must be a condition defined
try:
condition = rule.pop('when')
except KeyError:
raise utils.FormatError("No condition defined in adjust rule.")
# The optional 'continue' key should be a bool
continue_ = rule.pop('continue', True)
if not isinstance(continue_, bool):
raise utils.FormatError("The 'continue' value should be bool, "
"got '{}'.".format(continue_))
# The 'because' key is reserved for optional comments (ignored)
rule.pop('because', None)
# Apply remaining rule attributes if context matches
try:
if context.matches(condition):
self._merge_special(self.data, rule)
# First matching rule wins, skip the rest unless continue
if not continue_:
break
# Handle undecided rules as requested
except fmf.context.CannotDecide:
if undecided == 'skip':
continue
elif undecided == 'raise':
raise
else:
raise utils.GeneralError(
"Invalid value for the 'undecided' parameter. Should "
"be 'skip' or 'raise', got '{}'.".format(undecided))
# Adjust all child nodes as well
for child in self.children.values():
child.adjust(context, key, undecided)
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path is None:
return
if path != '/':
path = path.rstrip("/")
if path in IGNORED_DIRECTORIES: # pragma: no cover
log.debug("Ignoring '{0}' (special directory).".format(path))
return
log.info("Walking through directory {0}".format(
os.path.abspath(path)))
try:
dirpath, dirnames, filenames = next(os.walk(path))
except StopIteration:
log.debug("Skipping '{0}' (not accessible).".format(path))
return
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.abspath(os.path.join(dirpath, filename))
log.info("Checking file {0}".format(fullpath))
try:
with open(fullpath, encoding='utf-8') as datafile:
data = yaml.load(datafile, Loader=YamlLoader)
except yaml.error.YAMLError as error:
raise(utils.FileError("Failed to parse '{0}'\n{1}".format(
fullpath, error)))
log.data(pretty(data))
# Handle main.fmf as data for self
if filename == MAIN:
self.sources.append(fullpath)
self._raw_data = data
self.update(data)
# Handle other *.fmf files as children
else:
self.child(os.path.splitext(filename)[0], data, fullpath)
# Explore every child directory (ignore hidden dirs and subtrees)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
# Ignore metadata subtrees
if os.path.isdir(os.path.join(path, dirname, SUFFIX)):
log.debug("Ignoring metadata tree '{0}'.".format(dirname))
continue
self.child(dirname, os.path.join(path, dirname))
# Remove empty children (ignore directories without metadata)
for name in list(self.children.keys()):
child = self.children[name]
if not child.data and not child.children:
del(self.children[name])
log.debug("Empty tree '{0}' removed.".format(child.name))
