def parse_encoded_styles(text, normalize_key=None):
"""
Parse text styles encoded in a string into a nested data structure.
:param text: The encoded styles (a string).
:returns: A dictionary in the structure of the :data:`DEFAULT_FIELD_STYLES`
and :data:`DEFAULT_LEVEL_STYLES` dictionaries.
Here's an example of how this function works:
>>> from coloredlogs import parse_encoded_styles
>>> from pprint import pprint
>>> encoded_styles = 'debug=green;warning=yellow;error=red;critical=red,bold'
>>> pprint(parse_encoded_styles(encoded_styles))
{'debug': {'color': 'green'},
'warning': {'color': 'yellow'},
'error': {'color': 'red'},
'critical': {'bold': True, 'color': 'red'}}
"""
parsed_styles = {}
for token in split(text, ';'):
name, _, styles = token.partition('=')
parsed_styles[name] = dict(
('color', word) if word in ANSI_COLOR_CODES else (word, True)
for word in split(styles, ','))
return parsed_styles
def test_split(self):
from humanfriendly.text import split
self.assertEqual(split(''), [])
self.assertEqual(split('foo'), ['foo'])
self.assertEqual(split('foo, bar'), ['foo', 'bar'])
self.assertEqual(split('foo, bar, baz'), ['foo', 'bar', 'baz'])
self.assertEqual(split('foo,bar,baz'), ['foo', 'bar', 'baz'])
def parse_encoded_styles(text, normalize_key=None):
"""
Parse text styles encoded in a string into a nested data structure.
:param text: The encoded styles (a string).
:returns: A dictionary in the structure of the :data:`DEFAULT_FIELD_STYLES`
and :data:`DEFAULT_LEVEL_STYLES` dictionaries.
Here's an example of how this function works:
>>> from coloredlogs import parse_encoded_styles
>>> from pprint import pprint
>>> encoded_styles = 'debug=green;warning=yellow;error=red;critical=red,bold'
>>> pprint(parse_encoded_styles(encoded_styles))
{'debug': {'color': 'green'},
'warning': {'color': 'yellow'},
'error': {'color': 'red'},
'critical': {'bold': True, 'color': 'red'}}
"""
parsed_styles = {}
for token in split(text, ';'):
name, _, styles = token.partition('=')
parsed_styles[name] = dict(('color', word) if word in ANSI_COLOR_CODES else
(word, True) for word in split(styles, ','))
return parsed_styles
def test_split(self):
"""Test :func:`humanfriendly.text.split()`."""
from humanfriendly.text import split
self.assertEqual(split(''), [])
self.assertEqual(split('foo'), ['foo'])
self.assertEqual(split('foo, bar'), ['foo', 'bar'])
self.assertEqual(split('foo, bar, baz'), ['foo', 'bar', 'baz'])
self.assertEqual(split('foo,bar,baz'), ['foo', 'bar', 'baz'])
def test_split(self):
"""Test :func:`humanfriendly.text.split()`."""
from humanfriendly.text import split
self.assertEqual(split(''), [])
self.assertEqual(split('foo'), ['foo'])
self.assertEqual(split('foo, bar'), ['foo', 'bar'])
self.assertEqual(split('foo, bar, baz'), ['foo', 'bar', 'baz'])
self.assertEqual(split('foo,bar,baz'), ['foo', 'bar', 'baz'])
def load_config_file(configuration_file=None):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
When `configuration_file` isn't given :data:`LOCAL_CONFIG_FILE` and
:data:`GLOBAL_CONFIG_FILE` are checked and the first configuration file
that exists is loaded. This function is used by :class:`RotateBackups` to
discover user defined rotation schemes and by :mod:`rotate_backups.cli` to
discover directories for which backup rotation is configured.
"""
parser = configparser.RawConfigParser()
if configuration_file:
logger.verbose("Reading configuration file %s ..",
format_path(configuration_file))
loaded_files = parser.read(configuration_file)
if len(loaded_files) == 0:
msg = "Failed to read configuration file! (%s)"
raise ValueError(msg % configuration_file)
else:
for config_file in LOCAL_CONFIG_FILE, GLOBAL_CONFIG_FILE:
pathname = parse_path(config_file)
if parser.read(pathname):
logger.verbose("Reading configuration file %s ..",
format_path(pathname))
break
for section in parser.sections():
items = dict(parser.items(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(
items.get('prefer-recent', 'no')))
yield location, rotation_scheme, options
def load_config_file(configuration_file=None):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
When `configuration_file` isn't given :data:`LOCAL_CONFIG_FILE` and
:data:`GLOBAL_CONFIG_FILE` are checked and the first configuration file
that exists is loaded. This function is used by :class:`RotateBackups` to
discover user defined rotation schemes and by :mod:`rotate_backups.cli` to
discover directories for which backup rotation is configured.
"""
parser = configparser.RawConfigParser()
if configuration_file:
logger.verbose("Reading configuration file %s ..", format_path(configuration_file))
loaded_files = parser.read(configuration_file)
if len(loaded_files) == 0:
msg = "Failed to read configuration file! (%s)"
raise ValueError(msg % configuration_file)
else:
for config_file in LOCAL_CONFIG_FILE, GLOBAL_CONFIG_FILE:
pathname = parse_path(config_file)
if parser.read(pathname):
logger.verbose("Reading configuration file %s ..", format_path(pathname))
break
for section in parser.sections():
items = dict(parser.items(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
yield location, rotation_scheme, options
def parse_alternatives(expression):
"""
Parse an expression containing one or more alternative relationships.
:param expression: A relationship expression (a string).
:returns: A :class:`Relationship` object.
:raises: :exc:`~exceptions.ValueError` when parsing fails.
This function parses an expression containing one or more alternative
relationships of the form ``python2.6 | python2.7.``, i.e. a list of
relationship expressions separated by ``|`` tokens. Uses
:func:`parse_relationship()` to parse each ``|`` separated expression.
An example:
>>> from deb_pkg_tools.deps import parse_alternatives
>>> parse_alternatives('python2.6')
Relationship(name='python2.6')
>>> parse_alternatives('python2.6 | python2.7')
AlternativeRelationship(Relationship(name='python2.6'),
Relationship(name='python2.7'))
"""
if '|' in expression:
logger.debug("Parsing relationship with alternatives: %r", expression)
return AlternativeRelationship(
*map(parse_relationship, split(expression, '|')))
else:
return parse_relationship(expression)
def parse_alternatives(expression):
"""
Parse an expression containing one or more alternative relationships.
:param expression: A relationship expression (a string).
:returns: A :class:`Relationship` object.
:raises: :exc:`~exceptions.ValueError` when parsing fails.
This function parses an expression containing one or more alternative
relationships of the form ``python2.6 | python2.7.``, i.e. a list of
relationship expressions separated by ``|`` tokens. Uses
:func:`parse_relationship()` to parse each ``|`` separated expression.
An example:
>>> from deb_pkg_tools.deps import parse_alternatives
>>> parse_alternatives('python2.6')
Relationship(name='python2.6')
>>> parse_alternatives('python2.6 | python2.7')
AlternativeRelationship(Relationship(name='python2.6'),
Relationship(name='python2.7'))
"""
if '|' in expression:
logger.debug("Parsing relationship with alternatives: %r", expression)
return AlternativeRelationship(*map(parse_relationship, split(expression, '|')))
else:
return parse_relationship(expression)
def parse_depends(relationships):
"""
Parse a Debian package relationship declaration line.
:param relationships: A string containing one or more comma separated
package relationships or a list of strings with
package relationships.
:returns: A :class:`RelationshipSet` object.
:raises: :exc:`~exceptions.ValueError` when parsing fails.
This function parses a list of package relationships of the form ``python
(>= 2.6), python (<< 3)``, i.e. a comma separated list of relationship
expressions. Uses :func:`parse_alternatives()` to parse each comma
separated expression.
Here's an example:
>>> from deb_pkg_tools.deps import parse_depends
>>> dependencies = parse_depends('python (>= 2.6), python (<< 3)')
>>> print(repr(dependencies))
RelationshipSet(VersionedRelationship(name='python', operator='>=', version='2.6'),
VersionedRelationship(name='python', operator='<<', version='3'))
>>> dependencies.matches('python', '2.5')
False
>>> dependencies.matches('python', '2.6')
True
>>> dependencies.matches('python', '2.7')
True
>>> dependencies.matches('python', '3.0')
False
"""
logger.debug("Parsing relationships: %r", relationships)
if isinstance(relationships, string_types):
relationships = split(relationships, ',')
return RelationshipSet(*map(parse_alternatives, relationships))
def parse_depends(relationships):
"""
Parse a Debian package relationship declaration line.
:param relationships: A string containing one or more comma separated
package relationships or a list of strings with
package relationships.
:returns: A :class:`RelationshipSet` object.
:raises: :exc:`~exceptions.ValueError` when parsing fails.
This function parses a list of package relationships of the form ``python
(>= 2.6), python (<< 3)``, i.e. a comma separated list of relationship
expressions. Uses :func:`parse_alternatives()` to parse each comma
separated expression.
Here's an example:
>>> from deb_pkg_tools.deps import parse_depends
>>> dependencies = parse_depends('python (>= 2.6), python (<< 3)')
>>> print(repr(dependencies))
RelationshipSet(VersionedRelationship(name='python', operator='>=', version='2.6'),
VersionedRelationship(name='python', operator='<<', version='3'))
>>> dependencies.matches('python', '2.5')
False
>>> dependencies.matches('python', '2.6')
True
>>> dependencies.matches('python', '2.7')
True
>>> dependencies.matches('python', '3.0')
False
"""
logger.debug("Parsing relationships: %r", relationships)
if isinstance(relationships, string_types):
relationships = split(relationships, ',')
return RelationshipSet(*map(parse_alternatives, relationships))
def parse_encoded_styles(text, normalize_key=None):
"""
Parse text styles encoded in a string into a nested data structure.
:param text: The encoded styles (a string).
:returns: A dictionary in the structure of the :data:`DEFAULT_FIELD_STYLES`
and :data:`DEFAULT_LEVEL_STYLES` dictionaries.
Here's an example of how this function works:
>>> from coloredlogs import parse_encoded_styles
>>> from pprint import pprint
>>> encoded_styles = 'debug=green;warning=yellow;error=red;critical=red,bold'
>>> pprint(parse_encoded_styles(encoded_styles))
{'debug': {'color': 'green'},
'warning': {'color': 'yellow'},
'error': {'color': 'red'},
'critical': {'bold': True, 'color': 'red'}}
"""
parsed_styles = {}
for assignment in split(text, ';'):
name, _, styles = assignment.partition('=')
target = parsed_styles.setdefault(name, {})
for token in split(styles, ','):
# When this code was originally written, setting background colors
# wasn't supported yet, so there was no need to disambiguate
# between the text color and background color. This explains why
# a color name or number implies setting the text color (for
# backwards compatibility).
if token.isdigit():
target['color'] = int(token)
elif token in ANSI_COLOR_CODES:
target['color'] = token
elif '=' in token:
name, _, value = token.partition('=')
if name in ('color', 'background'):
if value.isdigit():
target[name] = int(value)
elif value in ANSI_COLOR_CODES:
target[name] = value
else:
target[token] = True
return parsed_styles
def merge_conflicts(self):
"""The filenames of any files with merge conflicts (a list of strings)."""
filenames = set()
listing = self.context.capture('git', 'ls-files', '--unmerged', '-z')
for entry in split(listing, '\0'):
# The output of `git ls-files --unmerged -z' consists of two
# tab-delimited fields per zero-byte terminated record, where the
# first field contains metadata and the second field contains the
# filename. A single filename can be output more than once.
metadata, _, name = entry.partition('\t')
if metadata and name:
filenames.add(name)
return sorted(filenames)
def entries(self):
"""A list of :class:`PasswordEntry` objects."""
timer = Timer()
passwords = []
logger.info("Scanning %s ..", format_path(self.directory))
listing = self.context.capture("find", "-type", "f", "-name", "*.gpg", "-print0")
for filename in split(listing, "\0"):
basename, extension = os.path.splitext(filename)
if extension == ".gpg":
# We use os.path.normpath() to remove the leading `./' prefixes
# that `find' adds because it searches the working directory.
passwords.append(PasswordEntry(name=os.path.normpath(basename), store=self))
logger.verbose("Found %s in %s.", pluralize(len(passwords), "password"), timer)
return natsort(passwords, key=lambda e: e.name)
def glob(self, pattern):
"""
Find matches for a given filename pattern.
:param pattern: A filename pattern (a string).
:returns: A list of strings with matches.
Some implementation notes:
- This method *emulates* filename globbing as supported by system
shells like Bash and ZSH. It works by forking a Python interpreter
and using that to call the :func:`glob.glob()` function. This
approach is of course rather heavyweight.
- Initially this method used Bash for filename matching (similar to
`this StackOverflow answer <https://unix.stackexchange.com/a/34012/44309>`_)
but I found it impossible to make this work well for patterns
containing whitespace.
- I took the whitespace issue as a sign that I was heading down the
wrong path (trying to add robustness to a fragile solution) and so
the new implementation was born (which prioritizes robustness over
performance).
"""
listing = self.capture(
'python',
input=dedent(
r'''
import glob
matches = glob.glob({pattern})
print('\x00'.join(matches))
''',
pattern=repr(pattern),
),
)
return split(listing, '\x00')
def glob(self, pattern):
"""
Find matches for a given filename pattern.
:param pattern: A filename pattern (a string).
:returns: A list of strings with matches.
Some implementation notes:
- This method *emulates* filename globbing as supported by system
shells like Bash and ZSH. It works by forking a Python interpreter
and using that to call the :func:`glob.glob()` function. This
approach is of course rather heavyweight.
- Initially this method used Bash for filename matching (similar to
`this StackOverflow answer <https://unix.stackexchange.com/a/34012/44309>`_)
but I found it impossible to make this work well for patterns
containing whitespace.
- I took the whitespace issue as a sign that I was heading down the
wrong path (trying to add robustness to a fragile solution) and so
the new implementation was born (which prioritizes robustness over
performance).
"""
listing = self.capture(
'python',
input=dedent(
r'''
import glob
matches = glob.glob({pattern})
print('\x00'.join(matches))
''',
pattern=repr(pattern),
),
)
return split(listing, '\x00')
def format_text(self,
include_password=True,
use_colors=None,
padding=True):
"""
Format :attr:`text` for viewing on a terminal.
:param include_password: :data:`True` to include the password in the
formatted text, :data:`False` to exclude the
password from the formatted text.
:param use_colors: :data:`True` to use ANSI escape sequences,
:data:`False` otherwise. When this is :data:`None`
:func:`~humanfriendly.terminal.terminal_supports_colors()`
will be used to detect whether ANSI escape sequences
are supported.
:param padding: :data:`True` to add empty lines before and after the
entry and indent the entry's text with two spaces,
:data:`False` to skip the padding.
:returns: The formatted entry (a string).
"""
# Determine whether we can use ANSI escape sequences.
if use_colors is None:
use_colors = terminal_supports_colors()
# Extract the password (first line) from the entry.
lines = self.text.splitlines()
password = lines.pop(0).strip()
text = trim_empty_lines('\n'.join(lines))
# Include the password in the formatted text?
if include_password:
text = "Password: %s\n%s" % (password, text)
# Add the name to the entry (only when there's something to show).
if text and not text.isspace():
title = ' / '.join(split(self.name, '/'))
if use_colors:
title = ansi_wrap(title, bold=True)
text = "%s\n\n%s" % (title, text)
# Highlight the entry's text using ANSI escape sequences.
lines = []
for line in text.splitlines():
# Check for a "Key: Value" line.
match = KEY_VALUE_PATTERN.match(line)
if match:
key = "%s:" % match.group(1).strip()
value = match.group(2).strip()
if use_colors:
# Highlight the key.
key = ansi_wrap(key, color=HIGHLIGHT_COLOR)
# Underline hyperlinks in the value.
tokens = value.split()
for i in range(len(tokens)):
if '://' in tokens[i]:
tokens[i] = ansi_wrap(tokens[i], underline=True)
# Replace the line with a highlighted version.
line = key + ' ' + ' '.join(tokens)
if padding:
line = ' ' + line
lines.append(line)
text = '\n'.join(lines)
if text and padding:
text = '\n%s\n' % text
return text
def options(self):
"""The encryption options for the filesystem (a list of strings)."""
return split(self.tokens[3])
def load_config_file(configuration_file=None, expand=True):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:param expand: :data:`True` to expand filename patterns to their matches,
:data:`False` otherwise.
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
This function is used by :class:`RotateBackups` to discover user defined
rotation schemes and by :mod:`rotate_backups.cli` to discover directories
for which backup rotation is configured. When `configuration_file` isn't
given :class:`~update_dotdee.ConfigLoader` is used to search for
configuration files in the following locations:
- ``/etc/rotate-backups.ini`` and ``/etc/rotate-backups.d/*.ini``
- ``~/.rotate-backups.ini`` and ``~/.rotate-backups.d/*.ini``
- ``~/.config/rotate-backups.ini`` and ``~/.config/rotate-backups.d/*.ini``
All of the available configuration files are loaded in the order given
above, so that sections in user-specific configuration files override
sections by the same name in system-wide configuration files.
"""
expand_notice_given = False
if configuration_file:
loader = ConfigLoader(available_files=[configuration_file], strict=True)
else:
loader = ConfigLoader(program_name='rotate-backups', strict=False)
for section in loader.section_names:
items = dict(loader.get_options(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
# Don't override the value of the 'removal_command' property unless the
# 'removal-command' configuration file option has a value set.
if items.get('removal-command'):
options['removal_command'] = shlex.split(items['removal-command'])
# Expand filename patterns?
if expand and location.have_wildcards:
logger.verbose("Expanding filename pattern %s on %s ..", location.directory, location.context)
if location.is_remote and not expand_notice_given:
logger.notice("Expanding remote filename patterns (may be slow) ..")
expand_notice_given = True
for match in sorted(location.context.glob(location.directory)):
if location.context.is_directory(match):
logger.verbose("Matched directory: %s", match)
expanded = Location(context=location.context, directory=match)
yield expanded, rotation_scheme, options
else:
logger.verbose("Ignoring match (not a directory): %s", match)
else:
yield location, rotation_scheme, options
def format_text(self, include_password=True, use_colors=None, padding=True, filters=()):
"""
Format :attr:`text` for viewing on a terminal.
:param include_password: :data:`True` to include the password in the
formatted text, :data:`False` to exclude the
password from the formatted text.
:param use_colors: :data:`True` to use ANSI escape sequences,
:data:`False` otherwise. When this is :data:`None`
:func:`~humanfriendly.terminal.terminal_supports_colors()`
will be used to detect whether ANSI escape sequences
are supported.
:param padding: :data:`True` to add empty lines before and after the
entry and indent the entry's text with two spaces,
:data:`False` to skip the padding.
:param filters: An iterable of regular expression patterns (defaults to
an empty tuple). If a line in the entry's text matches
one of these patterns it won't be shown on the
terminal.
:returns: The formatted entry (a string).
"""
# Determine whether we can use ANSI escape sequences.
if use_colors is None:
use_colors = terminal_supports_colors()
# Extract the password (first line) from the entry.
lines = self.text.splitlines()
password = lines.pop(0).strip()
# Compile the given patterns to case insensitive regular expressions
# and use them to ignore lines that match any of the given filters.
patterns = [coerce_pattern(f, re.IGNORECASE) for f in filters]
lines = [l for l in lines if not any(p.search(l) for p in patterns)]
text = trim_empty_lines("\n".join(lines))
# Include the password in the formatted text?
if include_password:
text = "Password: %s\n%s" % (password, text)
# Add the name to the entry (only when there's something to show).
if text and not text.isspace():
title = " / ".join(split(self.name, "/"))
if use_colors:
title = ansi_wrap(title, bold=True)
text = "%s\n\n%s" % (title, text)
# Highlight the entry's text using ANSI escape sequences.
lines = []
for line in text.splitlines():
# Check for a "Key: Value" line.
match = KEY_VALUE_PATTERN.match(line)
if match:
key = "%s:" % match.group(1).strip()
value = match.group(2).strip()
if use_colors:
# Highlight the key.
key = ansi_wrap(key, color=HIGHLIGHT_COLOR)
# Underline hyperlinks in the value.
tokens = value.split()
for i in range(len(tokens)):
if "://" in tokens[i]:
tokens[i] = ansi_wrap(tokens[i], underline=True)
# Replace the line with a highlighted version.
line = key + " " + " ".join(tokens)
if padding:
line = " " + line
lines.append(line)
text = "\n".join(lines)
text = trim_empty_lines(text)
if text and padding:
text = "\n%s\n" % text
return text
def load_config_file(configuration_file=None, expand=True):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:param expand: :data:`True` to expand filename patterns to their matches,
:data:`False` otherwise.
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
This function is used by :class:`RotateBackups` to discover user defined
rotation schemes and by :mod:`rotate_backups.cli` to discover directories
for which backup rotation is configured. When `configuration_file` isn't
given :class:`~update_dotdee.ConfigLoader` is used to search for
configuration files in the following locations:
- ``/etc/rotate-backups.ini`` and ``/etc/rotate-backups.d/*.ini``
- ``~/.rotate-backups.ini`` and ``~/.rotate-backups.d/*.ini``
- ``~/.config/rotate-backups.ini`` and ``~/.config/rotate-backups.d/*.ini``
All of the available configuration files are loaded in the order given
above, so that sections in user-specific configuration files override
sections by the same name in system-wide configuration files.
"""
expand_notice_given = False
if configuration_file:
loader = ConfigLoader(available_files=[configuration_file], strict=True)
else:
loader = ConfigLoader(program_name='rotate-backups', strict=False)
for section in loader.section_names:
items = dict(loader.get_options(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
timestamp=items.get('timestamp'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
# Don't override the value of the 'removal_command' property unless the
# 'removal-command' configuration file option has a value set.
if items.get('removal-command'):
options['removal_command'] = shlex.split(items['removal-command'])
# Expand filename patterns?
if expand and location.have_wildcards:
logger.verbose("Expanding filename pattern %s on %s ..", location.directory, location.context)
if location.is_remote and not expand_notice_given:
logger.notice("Expanding remote filename patterns (may be slow) ..")
expand_notice_given = True
for match in sorted(location.context.glob(location.directory)):
if location.context.is_directory(match):
logger.verbose("Matched directory: %s", match)
expanded = Location(context=location.context, directory=match)
yield expanded, rotation_scheme, options
else:
logger.verbose("Ignoring match (not a directory): %s", match)
else:
yield location, rotation_scheme, options
def parse_usage(text):
"""
Parse a usage message by inferring its structure (and making some assumptions :-).
:param text: The usage message to parse (a string).
:returns: A tuple of two lists:
1. A list of strings with the paragraphs of the usage message's
"introduction" (the paragraphs before the documentation of the
supported command line options).
2. A list of strings with pairs of command line options and their
descriptions: Item zero is a line listing a supported command
line option, item one is the description of that command line
option, item two is a line listing another supported command
line option, etc.
Usage messages in general are free format of course, however
:func:`parse_usage()` assume a certain structure from usage messages in
order to successfully parse them:
- The usage message starts with a line ``Usage: ...`` that shows a symbolic
representation of the way the program is to be invoked.
- After some free form text a line ``Supported options:`` (surrounded by
empty lines) precedes the documentation of the supported command line
options.
- The command line options are documented as follows::
-v, --verbose
Make more noise.
So all of the variants of the command line option are shown together on a
separate line, followed by one or more paragraphs describing the option.
- There are several other minor assumptions, but to be honest I'm not sure if
anyone other than me is ever going to use this functionality, so for now I
won't list every intricate detail :-).
If you're curious anyway, refer to the usage message of the `humanfriendly`
package (defined in the :mod:`humanfriendly.cli` module) and compare it with
the usage message you see when you run ``humanfriendly --help`` and the
generated usage message embedded in the readme.
Feel free to request more detailed documentation if you're interested in
using the :mod:`humanfriendly.usage` module outside of the little ecosystem
of Python packages that I have been building over the past years.
"""
introduction = []
documented_options = []
# Split the raw usage message into paragraphs.
paragraphs = split_paragraphs(text)
# Get the paragraphs that are part of the introduction.
while paragraphs:
# Check whether we've found the end of the introduction.
end_of_intro = (paragraphs[0] == START_OF_OPTIONS_MARKER)
# Append the current paragraph to the introduction.
introduction.append(join_lines(paragraphs.pop(0)))
# Stop after we've processed the complete introduction.
if end_of_intro:
break
logger.debug("Parsed introduction: %s", introduction)
# Parse the paragraphs that document command line options.
while paragraphs:
documented_options.append(dedent(paragraphs.pop(0)))
description = []
while paragraphs:
# Check if the next paragraph starts the documentation of another
# command line option.
if all(OPTION_PATTERN.match(t) for t in split(paragraphs[0])):
break
else:
description.append(paragraphs.pop(0))
# Join the description's paragraphs back together so we can remove
# common leading indentation.
documented_options.append(dedent('\n\n'.join(description)))
logger.debug("Parsed options: %s", documented_options)
return introduction, documented_options
def parse_usage(text):
"""
Parse a usage message by inferring its structure (and making some assumptions :-).
:param text: The usage message to parse (a string).
:returns: A tuple of two lists:
1. A list of strings with the paragraphs of the usage message's
"introduction" (the paragraphs before the documentation of the
supported command line options).
2. A list of strings with pairs of command line options and their
descriptions: Item zero is a line listing a supported command
line option, item one is the description of that command line
option, item two is a line listing another supported command
line option, etc.
Usage messages in general are free format of course, however
:func:`parse_usage()` assume a certain structure from usage messages in
order to successfully parse them:
- The usage message starts with a line ``Usage: ...`` that shows a symbolic
representation of the way the program is to be invoked.
- After some free form text a line ``Supported options:`` (surrounded by
empty lines) precedes the documentation of the supported command line
options.
- The command line options are documented as follows::
-v, --verbose
Make more noise.
So all of the variants of the command line option are shown together on a
separate line, followed by one or more paragraphs describing the option.
- There are several other minor assumptions, but to be honest I'm not sure if
anyone other than me is ever going to use this functionality, so for now I
won't list every intricate detail :-).
If you're curious anyway, refer to the usage message of the `humanfriendly`
package (defined in the :mod:`humanfriendly.cli` module) and compare it with
the usage message you see when you run ``humanfriendly --help`` and the
generated usage message embedded in the readme.
Feel free to request more detailed documentation if you're interested in
using the :mod:`humanfriendly.usage` module outside of the little ecosystem
of Python packages that I have been building over the past years.
"""
introduction = []
documented_options = []
# Split the raw usage message into paragraphs.
paragraphs = split_paragraphs(text)
# Get the paragraphs that are part of the introduction.
while paragraphs:
# Check whether we've found the end of the introduction.
end_of_intro = (paragraphs[0] == START_OF_OPTIONS_MARKER)
# Append the current paragraph to the introduction.
introduction.append(join_lines(paragraphs.pop(0)))
# Stop after we've processed the complete introduction.
if end_of_intro:
break
logger.debug("Parsed introduction: %s", introduction)
# Parse the paragraphs that document command line options.
while paragraphs:
documented_options.append(dedent(paragraphs.pop(0)))
description = []
while paragraphs:
# Check if the next paragraph starts the documentation of another
# command line option.
if all(OPTION_PATTERN.match(t) for t in split(paragraphs[0])):
break
else:
description.append(paragraphs.pop(0))
# Join the description's paragraphs back together so we can remove
# common leading indentation.
documented_options.append(dedent('\n\n'.join(description)))
logger.debug("Parsed options: %s", documented_options)
return introduction, documented_options
def is_debian_mirror(url):
"""Check whether the given URL looks like a Debian mirror URL."""
url = normalize_mirror_url(url)
if has_compatible_scheme(url):
components = split(url, '/')
return components[-1] == 'debian'
