def __init__(self, arguments=None, path=None):
""" Prepare the parser. """
# Change current working directory (used for testing)
if path is not None:
os.chdir(path)
# Split command line if given as a string (used for testing)
if isinstance(arguments, type("")): # pragma: no cover
try:
# This is necessary for Python 2.6
self.arguments = [arg.decode('utf8')
for arg in shlex.split(arguments.encode('utf8'))]
except AttributeError:
self.arguments = shlex.split(arguments)
# Otherwise use sys.argv (plus decode unicode for Python 2)
if arguments is None: # pragma: no cover
try:
self.arguments = [arg.decode("utf-8") for arg in sys.argv]
except AttributeError:
self.arguments = sys.argv
# Enable debugging output if requested
if "--debug" in self.arguments:
utils.log.setLevel(utils.LOG_DEBUG)
# Handle subcommands (mapped to format_* methods)
self.parser = argparse.ArgumentParser(
usage="fmf command [options]\n" + __doc__)
self.parser.add_argument('command', help='Command to run')
self.command = self.parser.parse_args(self.arguments[1:2]).command
if not hasattr(self, "command_" + self.command):
self.parser.print_help()
raise utils.GeneralError(
"Unrecognized command: '{0}'".format(self.command))
# Initialize the rest and run the subcommand
self.output = ""
getattr(self, "command_" + self.command)()
def __init__(self, arguments=None, path=None):
""" Prepare the parser. """
# Change current working directory (used for testing)
if path is not None:
os.chdir(path)
# Split command line if given as a string (used for testing)
if isinstance(arguments, str):
self.arguments = shlex.split(arguments)
# Otherwise use sys.argv
if arguments is None:
self.arguments = sys.argv
# Enable debugging output if requested
if "--debug" in self.arguments:
utils.log.setLevel(utils.LOG_DEBUG)
# Handle subcommands (mapped to format_* methods)
self.parser = argparse.ArgumentParser(usage="fmf command [options]\n" +
__doc__)
self.parser.add_argument('command', help='Command to run')
self.command = self.parser.parse_args(self.arguments[1:2]).command
if not hasattr(self, "command_" + self.command):
self.parser.print_help()
raise utils.GeneralError("Unrecognized command: '{0}'".format(
self.command))
# Initialize the rest and run the subcommand
self.output = ""
getattr(self, "command_" + self.command)()
def __init__(self, data, name=None, parent=None):
"""
Initialize metadata tree from directory path or data dictionary
Data parameter can be either a string with directory path to be
explored or a dictionary with the values already prepared.
"""
# Bail out if no data and no parent given
if not data and not parent:
raise utils.GeneralError(
"No data or parent provided to initialize the tree.")
# Initialize family relations, object data and source files
self.parent = parent
self.children = dict()
self.data = dict()
self.sources = list()
self.root = None
self.version = utils.VERSION
self.original_data = dict()
self._commit = None
self._raw_data = dict()
# Track whether the data dictionary has been updated
# (needed to prevent removing nodes with an empty dict).
self._updated = False
# Store symlinks in while walking tree in grow() to detect
# symlink loops
if parent is None:
self._symlinkdirs = []
else:
self._symlinkdirs = parent._symlinkdirs
# Special handling for top parent
if self.parent is None:
self.name = "/"
if not isinstance(data, dict):
self._initialize(path=data)
data = self.root
# Handle child node creation
else:
self.root = self.parent.root
self.name = os.path.join(self.parent.name, name)
# Update data from a dictionary (handle empty nodes)
if isinstance(data, dict) or data is None:
self.update(data)
# Grow the tree from a directory path
else:
self.grow(data)
# Apply inheritance when all scattered data are gathered.
# This is done only once, from the top parent object.
if self.parent is None:
self.inherit()
log.debug("New tree '{0}' created.".format(self))
def node(reference):
"""
Return Tree node referenced by the fmf identifier
Keys supported in the reference:
url .... git repository url (optional)
ref .... branch, tag or commit (default branch if not provided)
path ... metadata tree root ('.' by default)
name ... tree node name ('/' by default)
See the documentation for the full fmf id specification:
https://fmf.readthedocs.io/en/latest/concept.html#identifiers
Raises ReferenceError if referenced node does not exist.
"""
# Fetch remote git repository
if 'url' in reference:
path = reference.get('path', '.').lstrip('/')
# Create lock path to fetch/read git from URL to the cache
cache_dir = utils.get_cache_directory()
# Use .read.lock suffix (different from the inner fetch lock)
lock_path = os.path.join(
cache_dir, reference["url"].replace('/', '_')) + '.read.lock'
try:
with FileLock(lock_path, timeout=NODE_LOCK_TIMEOUT) as lock:
# Write PID to lockfile so we know which process got it
with open(lock.lock_file, 'w') as lock_file:
lock_file.write(str(os.getpid()))
repository = utils.fetch(
reference.get('url'), reference.get('ref'))
root = os.path.join(repository, path)
tree = Tree(root)
except Timeout:
raise utils.GeneralError(
"Failed to acquire lock for {0} within {1} seconds".format(
lock_path, NODE_LOCK_TIMEOUT))
# Use local files
else:
root = reference.get('path', '.')
if not root.startswith('/') and root != '.':
raise utils.ReferenceError(
'Relative path "%s" specified.' % root)
tree = Tree(root)
found_node = tree.find(reference.get('name', '/'))
if found_node is None:
raise utils.ReferenceError(
"No tree node found for '{0}' reference".format(reference))
# FIXME Should be able to remove .cache if required
return found_node
def _locate_raw_data(self):
"""
Detect location of raw data from which the node has been created
Find the closest parent node which has raw data defined. In the
raw data identify the dictionary corresponding to the current
node, create if needed. Detect the raw data source filename.
Return tuple with the following three items:
node_data ... dictionary containing raw data for the current node
full_data ... full raw data from the closest parent node
source ... file system path where the full raw data are stored
"""
# List of node names in the virtual hierarchy
hierarchy = list()
# Find the closest parent with raw data defined
node = self
while True:
# Raw data found
full_data = node._raw_data
if full_data:
break
# No raw data, perhaps a Tree initialized from a dict?
if not node.parent:
raise utils.GeneralError(
"No raw data found, does the Tree grow on a filesystem?")
# Extend virtual hierarchy with the current node name, go up
hierarchy.insert(0, "/" + node.name.rsplit("/")[-1])
node = node.parent
# Localize node data dictionary in the virtual hierarchy
node_data = full_data
for key in hierarchy:
# Create a virtual hierarchy level if missing
if key not in node_data:
node_data[key] = dict()
# Initialize as an empty dict if leaf node is empty
if node_data[key] is None:
node_data[key] = dict()
node_data = node_data[key]
# The full raw data were read from the last source
return node_data, full_data, node.sources[-1]
def __init__(self, data, name=None, parent=None):
"""
Initialize metadata tree from directory path or data dictionary
Data parameter can be either a string with directory path to be
explored or a dictionary with the values already prepared.
"""
# Bail out if no data and no parent given
if not data and not parent:
raise utils.GeneralError(
"No data or parent provided to initialize the tree.")
# Initialize family relations, object data and source files
self.parent = parent
self.children = dict()
self.data = dict()
self.sources = list()
self.root = None
self.version = utils.VERSION
self.original_data = dict()
# Special handling for top parent
if self.parent is None:
self.name = "/"
if not isinstance(data, dict):
self._initialize(path=data)
data = self.root
# Handle child node creation
else:
self.root = self.parent.root
self.name = os.path.join(self.parent.name, name)
# Initialize data
if isinstance(data, dict):
self.update(data)
else:
self.grow(data)
log.debug("New tree '{0}' created.".format(self))
def adjust(self, context, key='adjust', undecided='skip'):
"""
Adjust tree data based on provided context and rules
The 'context' should be an instance of the fmf.context.Context
class describing the environment context. By default, the key
'adjust' of each node is inspected for possible rules that
should be applied. Provide 'key' to use a custom key instead.
Optional 'undecided' parameter can be used to specify what
should happen when a rule condition cannot be decided because
context dimension is not defined. By default, such rules are
skipped. In order to raise the fmf.context.CannotDecide
exception in such cases use undecided='raise'.
"""
# Check context sanity
if not isinstance(context, fmf.context.Context):
raise utils.GeneralError("Invalid adjust context: '{}'.".format(
type(context).__name__))
# Adjust rules should be a dictionary or a list of dictionaries
try:
rules = copy.deepcopy(self.data[key])
log.debug("Applying adjust rules for '{}'.".format(self))
log.data(rules)
if isinstance(rules, dict):
rules = [rules]
if not isinstance(rules, list):
raise utils.FormatError(
"Invalid adjust rule format in '{}'. "
"Should be a dictionary or a list of dictionaries, "
"got '{}'.".format(self.name,
type(rules).__name__))
except KeyError:
rules = []
# Check and apply each rule
for rule in rules:
# Rule must be a dictionary
if not isinstance(rule, dict):
raise utils.FormatError("Adjust rule should be a dictionary.")
# There must be a condition defined
try:
condition = rule.pop('when')
except KeyError:
raise utils.FormatError("No condition defined in adjust rule.")
# The optional 'continue' key should be a bool
continue_ = rule.pop('continue', True)
if not isinstance(continue_, bool):
raise utils.FormatError("The 'continue' value should be bool, "
"got '{}'.".format(continue_))
# The 'because' key is reserved for optional comments (ignored)
rule.pop('because', None)
# Apply remaining rule attributes if context matches
try:
if context.matches(condition):
self._merge_special(self.data, rule)
# First matching rule wins, skip the rest unless continue
if not continue_:
break
# Handle undecided rules as requested
except fmf.context.CannotDecide:
if undecided == 'skip':
continue
elif undecided == 'raise':
raise
else:
raise utils.GeneralError(
"Invalid value for the 'undecided' parameter. Should "
"be 'skip' or 'raise', got '{}'.".format(undecided))
# Adjust all child nodes as well
for child in self.children.values():
child.adjust(context, key, undecided)