def init(path):
""" Create metadata tree root under given path """
root = os.path.abspath(os.path.join(path, ".fmf"))
if os.path.exists(root):
raise utils.FileError("{0} '{1}' already exists.".format(
"Directory" if os.path.isdir(root) else "File", root))
try:
os.makedirs(root)
with open(os.path.join(root, "version"), "w") as version:
version.write("{0}\n".format(utils.VERSION))
except OSError as error:
raise utils.FileError("Failed to create '{}': {}.".format(
root, error))
return root
def _initialize(self, path):
""" Find metadata tree root, detect format version """
# Find the tree root
root = os.path.abspath(path)
try:
while ".fmf" not in next(os.walk(root))[1]:
if root == "/":
raise utils.RootError(
"Unable to find tree root for '{0}'.".format(
os.path.abspath(path)))
root = os.path.abspath(os.path.join(root, os.pardir))
except StopIteration:
raise utils.FileError("Invalid directory path: {0}".format(root))
log.info("Root directory found: {0}".format(root))
self.root = root
# Detect format version
try:
with open(os.path.join(self.root, ".fmf", "version")) as version:
self.version = int(version.read())
log.info("Format version detected: {0}".format(self.version))
except IOError as error:
raise utils.FormatError(
"Unable to detect format version: {0}".format(error))
except ValueError:
raise utils.FormatError("Invalid version format")
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path is None:
return
path = path.rstrip("/")
log.info("Walking through directory {0}".format(os.path.abspath(path)))
dirpath, dirnames, filenames = next(os.walk(path))
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.abspath(os.path.join(dirpath, filename))
log.info("Checking file {0}".format(fullpath))
try:
with open(fullpath) as datafile:
data = yaml.load(datafile)
except yaml.scanner.ScannerError as error:
raise (utils.FileError("Failed to parse '{0}'\n{1}".format(
fullpath, error)))
log.data(pretty(data))
# Handle main.fmf as data for self
if filename == MAIN:
self.sources.append(fullpath)
self.update(data)
# Handle other *.fmf files as children
else:
self.child(os.path.splitext(filename)[0], data, fullpath)
# Explore every child directory (ignore hidden dirs and subtrees)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
# Ignore metadata subtrees
if os.path.isdir(os.path.join(path, dirname, SUFFIX)):
log.debug("Ignoring metadata tree '{0}'.".format(dirname))
continue
self.child(dirname, os.path.join(path, dirname))
# Remove empty children (ignore directories without metadata)
for name in list(self.children.keys()):
child = self.children[name]
if not child.data and not child.children:
del (self.children[name])
log.debug("Empty tree '{0}' removed.".format(child.name))
# Apply inheritance when all scattered data are gathered.
# This is done only once, from the top parent object.
if self.parent is None:
self.inherit()
def command_init(self):
""" Initialize tree """
self.parser = argparse.ArgumentParser(
description="Initialize a new metadata tree")
self.options_utils()
self.options = self.parser.parse_args(self.arguments[2:])
# For each path create an .fmf directory and version file
for path in self.options.paths or ["."]:
root = os.path.abspath(os.path.join(path, ".fmf"))
if os.path.exists(root):
raise utils.FileError("{0} '{1}' already exists.".format(
"Directory" if os.path.isdir(root) else "File", root))
os.makedirs(root)
with open(os.path.join(root, "version"), "w") as version:
version.write("{0}\n".format(utils.VERSION))
print("Metadata tree '{0}' successfully initialized.".format(root))
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path is None:
return
path = path.rstrip("/")
log.info("Walking through directory {0}".format(path))
try:
dirpath, dirnames, filenames = list(os.walk(path))[0]
except IndexError:
raise utils.FileError(
"Unable to walk through the '{0}' directory.".format(path))
children = dict()
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.join(dirpath, filename)
log.info("Checking file {0}".format(fullpath))
with open(fullpath) as datafile:
data = yaml.load(datafile)
log.data(pretty(data))
if filename == MAIN:
self.update(data)
else:
self.child(os.path.splitext(filename)[0], data)
# Explore every child directory (ignore hidden)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
self.child(dirname, os.path.join(path, dirname))
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path != '/':
path = path.rstrip("/")
if path in IGNORED_DIRECTORIES: # pragma: no cover
log.debug("Ignoring '{0}' (special directory).".format(path))
return
log.info("Walking through directory {0}".format(os.path.abspath(path)))
try:
dirpath, dirnames, filenames = next(os.walk(path))
except StopIteration:
log.debug("Skipping '{0}' (not accessible).".format(path))
return
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.abspath(os.path.join(dirpath, filename))
log.info("Checking file {0}".format(fullpath))
try:
with open(fullpath, encoding='utf-8') as datafile:
data = YAML(typ="safe").load(datafile)
except (YAMLError, DuplicateKeyError) as error:
raise (
utils.FileError(f"Failed to parse '{fullpath}'.\n{error}"))
log.data(pretty(data))
# Handle main.fmf as data for self
if filename == MAIN:
self.sources.append(fullpath)
self._raw_data = copy.deepcopy(data)
self.update(data)
# Handle other *.fmf files as children
else:
self.child(os.path.splitext(filename)[0], data, fullpath)
# Explore every child directory (ignore hidden dirs and subtrees)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
fulldir = os.path.join(dirpath, dirname)
if os.path.islink(fulldir):
# According to the documentation, calling os.path.realpath
# with strict = True will raise OSError if a symlink loop
# is encountered. But it does not do that with a loop with
# more than one node
fullpath = os.path.realpath(fulldir)
if fullpath in self._symlinkdirs:
log.debug("Not entering symlink loop {}".format(fulldir))
continue
else:
self._symlinkdirs.append(fullpath)
# Ignore metadata subtrees
if os.path.isdir(os.path.join(path, dirname, SUFFIX)):
log.debug("Ignoring metadata tree '{0}'.".format(dirname))
continue
self.child(dirname, os.path.join(path, dirname))
# Ignore directories with no metadata (remove all child nodes which
# do not have children and their data haven't been updated)
for name in list(self.children.keys()):
child = self.children[name]
if not child.children and not child._updated:
del (self.children[name])
log.debug("Empty tree '{0}' removed.".format(child.name))
def grow(self, path):
"""
Grow the metadata tree for the given directory path
Note: For each path, grow() should be run only once. Growing the tree
from the same path multiple times with attribute adding using the "+"
sign leads to adding the value more than once!
"""
if path is None:
return
if path != '/':
path = path.rstrip("/")
if path in IGNORED_DIRECTORIES: # pragma: no cover
log.debug("Ignoring '{0}' (special directory).".format(path))
return
log.info("Walking through directory {0}".format(
os.path.abspath(path)))
try:
dirpath, dirnames, filenames = next(os.walk(path))
except StopIteration:
log.debug("Skipping '{0}' (not accessible).".format(path))
return
# Investigate main.fmf as the first file (for correct inheritance)
filenames = sorted(
[filename for filename in filenames if filename.endswith(SUFFIX)])
try:
filenames.insert(0, filenames.pop(filenames.index(MAIN)))
except ValueError:
pass
# Check every metadata file and load data (ignore hidden)
for filename in filenames:
if filename.startswith("."):
continue
fullpath = os.path.abspath(os.path.join(dirpath, filename))
log.info("Checking file {0}".format(fullpath))
try:
with open(fullpath, encoding='utf-8') as datafile:
data = yaml.load(datafile, Loader=YamlLoader)
except yaml.error.YAMLError as error:
raise(utils.FileError("Failed to parse '{0}'\n{1}".format(
fullpath, error)))
log.data(pretty(data))
# Handle main.fmf as data for self
if filename == MAIN:
self.sources.append(fullpath)
self._raw_data = data
self.update(data)
# Handle other *.fmf files as children
else:
self.child(os.path.splitext(filename)[0], data, fullpath)
# Explore every child directory (ignore hidden dirs and subtrees)
for dirname in sorted(dirnames):
if dirname.startswith("."):
continue
# Ignore metadata subtrees
if os.path.isdir(os.path.join(path, dirname, SUFFIX)):
log.debug("Ignoring metadata tree '{0}'.".format(dirname))
continue
self.child(dirname, os.path.join(path, dirname))
# Remove empty children (ignore directories without metadata)
for name in list(self.children.keys()):
child = self.children[name]
if not child.data and not child.children:
del(self.children[name])
log.debug("Empty tree '{0}' removed.".format(child.name))
