def test_config_loader(self):
"""Tests for the :class:`ConfigLoader` class."""
# Test support for custom filename extensions.
loader = ConfigLoader(program_name='update-dotdee', filename_extension='conf')
assert '/etc/update-dotdee.conf' in loader.filename_patterns
# Test loading of multiple configuration files.
with MockedHomeDirectory() as directory:
# Create the main user configuration file at ~/.update-dotdee.ini.
main_file = os.path.join(directory, '.update-dotdee.ini')
write_file(main_file, dedent('''
[main-section]
main-option = value
'''))
# Create some modular ~/.config/update-dotdee.d/*.ini configuration files.
config_directory = os.path.join(directory, '.config', 'update-dotdee.d')
os.makedirs(config_directory)
modular_file_1 = os.path.join(config_directory, '1.ini')
modular_file_2 = os.path.join(config_directory, '2.ini')
modular_file_11 = os.path.join(config_directory, '11.ini')
write_file(modular_file_1, dedent('''
[modular-section-1]
my-option-name = value
'''))
write_file(modular_file_2, dedent('''
[modular-section-2]
my-option-name = value
[main-section]
modular-option = value
'''))
write_file(modular_file_11, dedent('''
[modular-section-11]
my-option-name = value
'''))
# Use ConfigLoader to load the configuration files.
loader = ConfigLoader(program_name='update-dotdee')
# Make sure all configuration files were found.
assert len(loader.available_files) == 4
assert loader.available_files[0] == main_file
assert loader.available_files[1] == modular_file_1
assert loader.available_files[2] == modular_file_2
assert loader.available_files[3] == modular_file_11
# Make sure all configuration file sections are loaded.
assert set(loader.section_names) == set([
'main-section',
'modular-section-1',
'modular-section-2',
'modular-section-11',
])
assert loader.get_options('main-section') == {
'main-option': 'value',
'modular-option': 'value',
}
def config_loader(self):
"""
A :class:`~update_dotdee.ConfigLoader` object.
See also :attr:`config` and :attr:`config_section`.
"""
return ConfigLoader(program_name='unlock-remote-system')
def config_loader(self):
r"""
A :class:`~update_dotdee.ConfigLoader` object that provides access to the configuration.
.. [[[cog
.. from update_dotdee import inject_documentation
.. inject_documentation(program_name='chat-archive')
.. ]]]
Configuration files are text files in the subset of `ini syntax`_ supported by
Python's configparser_ module. They can be located in the following places:
========= ========================== ===============================
Directory Main configuration file Modular configuration files
========= ========================== ===============================
/etc /etc/chat-archive.ini /etc/chat-archive.d/\*.ini
~ ~/.chat-archive.ini ~/.chat-archive.d/\*.ini
~/.config ~/.config/chat-archive.ini ~/.config/chat-archive.d/\*.ini
========= ========================== ===============================
The available configuration files are loaded in the order given above, so that
user specific configuration files override system wide configuration files.
.. _configparser: https://docs.python.org/3/library/configparser.html
.. _ini syntax: https://en.wikipedia.org/wiki/INI_file
.. [[[end]]]
"""
return ConfigLoader(program_name="chat-archive")
def get_post_context(name):
"""
Get an execution context for the post-boot environment of a remote host.
:param name: The configuration section name or SSH alias of the remote host (a string).
:returns: A :class:`~executor.contexts.RemoteContext` object.
"""
account = RemoteAccount(name)
loader = ConfigLoader(program_name='unlock-remote-system')
if account.ssh_alias in loader.section_names:
options = loader.get_options(account.ssh_alias)
post_boot = options.get('post-boot')
if post_boot:
profile = ConnectionProfile(expression=post_boot)
return RemoteContext(
identity_file=profile.identity_file,
port=profile.port_number,
ssh_alias=profile.hostname,
ssh_user=account.ssh_user or profile.username,
)
return RemoteContext(ssh_alias=name)
def main():
"""Command line interface for ``unlock-remote-system``."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
program_opts = {}
identity_file = None
do_shell = False
do_watch = False
watch_all = False
try:
options, arguments = getopt.gnu_getopt(sys.argv[1:], 'i:k:p:r:swavqh',
[
'identity-file=',
'known-hosts=',
'password='******'remote-host=',
'shell',
'watch',
'all',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-i', '--identity-file'):
identity_file = parse_path(value)
elif option in ('-k', '--known-hosts'):
program_opts['known_hosts_file'] = parse_path(value)
elif option in ('-p', '--password'):
program_opts['password'] = get_password_from_store(value)
elif option in ('-r', '--remote-host'):
program_opts['ssh_proxy'] = value
elif option in ('-s', '--shell'):
do_shell = True
elif option in ('-w', '--watch'):
do_watch = True
elif option in ('-a', '--all'):
watch_all = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
else:
raise Exception("Unhandled option!")
if not arguments:
usage(__doc__)
sys.exit(0)
elif len(arguments) > 2:
raise Exception("only two positional arguments allowed")
# Create a ConfigLoader object and prepare to pass it to the program to
# avoid scanning for configuration files more than once (which isn't a
# real problem but does generate somewhat confusing log output).
loader = ConfigLoader(program_name='unlock-remote-system')
program_opts['config_loader'] = loader
# Check if a single positional argument was given that matches the name
# of a user defined configuration section.
if len(arguments) == 1 and arguments[0] in loader.section_names:
logger.info("Loading configuration section '%s' ..", arguments[0])
program_opts['config_section'] = arguments[0]
else:
# The SSH connection profile of the pre-boot environment
# is given as the first positional argument.
program_opts['pre_boot'] = ConnectionProfile(
expression=arguments[0], identity_file=identity_file)
# The SSH connection profile of the post-boot environment
# can be given as the second positional argument, otherwise
# it will be inferred from the connection profile of
# the pre-boot environment.
if len(arguments) == 2:
program_opts['post_boot'] = ConnectionProfile(
expression=arguments[1])
else:
# By default we don't use root to login to the post-boot environment.
program_opts['post_boot'] = ConnectionProfile(
expression=arguments[0])
program_opts['post_boot'].username = find_local_username()
# Prompt the operator to enter the disk encryption password for the remote host?
if not program_opts.get('password'):
program_opts['password'] = prompt_for_password(
program_opts['pre_boot'].hostname)
except Exception as e:
warning("Failed to parse command line arguments! (%s)", e)
sys.exit(1)
# Try to unlock the remote system.
try:
if do_watch and watch_all:
watch_all_systems(loader)
else:
with EncryptedSystem(**program_opts) as program:
if do_watch:
program.watch_system()
else:
program.unlock_system()
if do_shell:
start_interactive_shell(program.post_context)
except EncryptedSystemError as e:
logger.error("Aborting due to error: %s", e)
sys.exit(2)
except Exception:
logger.exception("Aborting due to unexpected exception!")
sys.exit(3)
def reboot_remote_system(context=None, name=None):
"""
Reboot a remote Linux system (unattended).
:param context: A :class:`~executor.contexts.RemoteContext`
object (or :data:`None`).
:param name: The name of the ``unlock-remote-system`` configuration
section for the remote host (a string) or :data:`None`.
:raises: :exc:`~exceptions.ValueError` when the remote system appears to be
using root disk encryption but there's no ``unlock-remote-system``
configuration section available. The reasoning behind this is to
err on the side of caution when we suspect we won't be able to get
the remote system back online.
This function reboots a remote Linux system, waits for the system to go
down and then waits for it to come back up.
If the :attr:`~executor.ssh.client.RemoteAccount.ssh_alias` of the context
matches a section in the `unlock-remote-system` configuration, the root disk
encryption of the remote system will be unlocked after it is rebooted.
"""
timer = Timer()
# Get the execution context from a configuration section.
if name and not context:
context = get_post_context(name)
# Sanity check the provided execution context.
if not isinstance(context, RemoteContext):
msg = "Expected a RemoteContext object, got %s instead!"
raise TypeError(msg % type(context))
# Default the remote host name to the SSH alias.
if not name:
name = context.ssh_alias
logger.info("Preparing to reboot %s ..", context)
# Check if the name matches a configuration section.
loader = ConfigLoader(program_name='unlock-remote-system')
have_config = (name in loader.section_names)
# Check if the remote system is using root disk encryption.
needs_unlock = is_encrypted(context)
# Refuse to reboot if we can't get the system back online.
if needs_unlock and not have_config:
raise ValueError(
compact("""
It looks like the {context} is using root disk encryption but
there's no configuration defined for this system! Refusing to
reboot the system because we won't be able to unlock it.
""",
context=context))
# Get the current uptime of the remote system.
old_uptime = get_uptime(context)
logger.info("Rebooting after %s of uptime ..", format_timespan(old_uptime))
# Issue the `reboot' command.
try:
context.execute('reboot', shell=False, silent=True, sudo=True)
# TODO Investigate how to pro-actively close the master
# connection for multiplexed OpenSSH connections.
except RemoteConnectFailed:
logger.notice(
compact("""
While issuing the `reboot' command the SSH client reported dropping
the connection. We will proceed under the assumption that this was
caused by the remote SSH server being shut down as a result of the
`reboot' command.
"""))
# Unlock the root disk encryption.
if have_config:
options = dict(config_loader=loader, config_section=name)
with EncryptedSystem(**options) as program:
program.unlock_system()
# Wait for a successful SSH connection to report a lower uptime. We do this
# even when we just unlocked remote disk encryption, because in that case
# the SSH server in the post-boot environment hasn't been confirmed to
# successfully accept connections (we just observed the host keys
# changing). This works to counteract SSH accepting the connection attempt
# but reporting "System is booting up. See pam_nologin(8)" on the standard
# error stream followed by exit(255) which is fairly useless to callers of
# reboot_remote_system() that expect the system to be available...
logger.info("Waiting for %s to come back online ..", context)
while True:
try:
new_uptime = get_uptime(context)
if old_uptime > new_uptime:
break
else:
time.sleep(1)
except RemoteConnectFailed:
time.sleep(0.1)
logger.success("Took %s to reboot %s.", timer, context)
def load_config_file(configuration_file=None, expand=True):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:param expand: :data:`True` to expand filename patterns to their matches,
:data:`False` otherwise.
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
This function is used by :class:`RotateBackups` to discover user defined
rotation schemes and by :mod:`rotate_backups.cli` to discover directories
for which backup rotation is configured. When `configuration_file` isn't
given :class:`~update_dotdee.ConfigLoader` is used to search for
configuration files in the following locations:
- ``/etc/rotate-backups.ini`` and ``/etc/rotate-backups.d/*.ini``
- ``~/.rotate-backups.ini`` and ``~/.rotate-backups.d/*.ini``
- ``~/.config/rotate-backups.ini`` and ``~/.config/rotate-backups.d/*.ini``
All of the available configuration files are loaded in the order given
above, so that sections in user-specific configuration files override
sections by the same name in system-wide configuration files.
"""
expand_notice_given = False
if configuration_file:
loader = ConfigLoader(available_files=[configuration_file], strict=True)
else:
loader = ConfigLoader(program_name='rotate-backups', strict=False)
for section in loader.section_names:
items = dict(loader.get_options(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
# Don't override the value of the 'removal_command' property unless the
# 'removal-command' configuration file option has a value set.
if items.get('removal-command'):
options['removal_command'] = shlex.split(items['removal-command'])
# Expand filename patterns?
if expand and location.have_wildcards:
logger.verbose("Expanding filename pattern %s on %s ..", location.directory, location.context)
if location.is_remote and not expand_notice_given:
logger.notice("Expanding remote filename patterns (may be slow) ..")
expand_notice_given = True
for match in sorted(location.context.glob(location.directory)):
if location.context.is_directory(match):
logger.verbose("Matched directory: %s", match)
expanded = Location(context=location.context, directory=match)
yield expanded, rotation_scheme, options
else:
logger.verbose("Ignoring match (not a directory): %s", match)
else:
yield location, rotation_scheme, options
def load_config_file(configuration_file=None, expand=True):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:param expand: :data:`True` to expand filename patterns to their matches,
:data:`False` otherwise.
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
This function is used by :class:`RotateBackups` to discover user defined
rotation schemes and by :mod:`rotate_backups.cli` to discover directories
for which backup rotation is configured. When `configuration_file` isn't
given :class:`~update_dotdee.ConfigLoader` is used to search for
configuration files in the following locations:
- ``/etc/rotate-backups.ini`` and ``/etc/rotate-backups.d/*.ini``
- ``~/.rotate-backups.ini`` and ``~/.rotate-backups.d/*.ini``
- ``~/.config/rotate-backups.ini`` and ``~/.config/rotate-backups.d/*.ini``
All of the available configuration files are loaded in the order given
above, so that sections in user-specific configuration files override
sections by the same name in system-wide configuration files.
"""
expand_notice_given = False
if configuration_file:
loader = ConfigLoader(available_files=[configuration_file], strict=True)
else:
loader = ConfigLoader(program_name='rotate-backups', strict=False)
for section in loader.section_names:
items = dict(loader.get_options(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
timestamp=items.get('timestamp'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
# Don't override the value of the 'removal_command' property unless the
# 'removal-command' configuration file option has a value set.
if items.get('removal-command'):
options['removal_command'] = shlex.split(items['removal-command'])
# Expand filename patterns?
if expand and location.have_wildcards:
logger.verbose("Expanding filename pattern %s on %s ..", location.directory, location.context)
if location.is_remote and not expand_notice_given:
logger.notice("Expanding remote filename patterns (may be slow) ..")
expand_notice_given = True
for match in sorted(location.context.glob(location.directory)):
if location.context.is_directory(match):
logger.verbose("Matched directory: %s", match)
expanded = Location(context=location.context, directory=match)
yield expanded, rotation_scheme, options
else:
logger.verbose("Ignoring match (not a directory): %s", match)
else:
yield location, rotation_scheme, options