def coerce_location(value, **options):
"""
Coerce a string to a :class:`Location` object.
:param value: The value to coerce (a string or :class:`Location` object).
:param options: Any keyword arguments are passed on to
:func:`~executor.contexts.create_context()`.
:returns: A :class:`Location` object.
"""
# Location objects pass through untouched.
if not isinstance(value, Location):
# Other values are expected to be strings.
if not isinstance(value, string_types):
msg = "Expected Location object or string, got %s instead!"
raise ValueError(msg % type(value))
# Try to parse a remote location.
ssh_alias, _, directory = value.partition(':')
if ssh_alias and directory and '/' not in ssh_alias:
options['ssh_alias'] = ssh_alias
else:
directory = value
# Create the location object.
value = Location(
context=create_context(**options),
directory=parse_path(directory),
)
return value
def coerce_location(value, **options):
"""
Coerce a string to a :class:`Location` object.
:param value: The value to coerce (a string or :class:`Location` object).
:param options: Any keyword arguments are passed on to
:func:`~executor.contexts.create_context()`.
:returns: A :class:`Location` object.
"""
# Location objects pass through untouched.
if not isinstance(value, Location):
# Other values are expected to be strings.
if not isinstance(value, string_types):
msg = "Expected Location object or string, got %s instead!"
raise ValueError(msg % type(value))
# Try to parse a remote location.
ssh_alias, _, directory = value.partition(':')
if ssh_alias and directory and '/' not in ssh_alias:
options['ssh_alias'] = ssh_alias
else:
directory = value
# Create the location object.
value = Location(
context=create_context(**options),
directory=parse_path(directory),
)
return value
def store_host_keys(self, pre_server, post_server):
"""
Store the SSH host keys in the configuration.
:param pre_server: The :class:`ServerDetails` object from :func:`wait_for_pre_boot()`.
:param post_server: The :class:`ServerDetails` object from :func:`wait_for_post_boot()`.
"""
if self.config_section:
items = (('pre-boot-host-keys', pre_server),
('post-boot-host-keys', post_server))
options = dict((name, '\n'.join(sorted(server.host_keys)))
for name, server in items)
if not all(
self.config.get(name) == value
for name, value in options.items()):
logger.info("Storing SSH host keys in %s ..", HOST_KEYS_FILE)
filename = parse_path(HOST_KEYS_FILE)
directory = os.path.dirname(filename)
if not os.path.isdir(directory):
os.makedirs(directory)
parser = configparser.RawConfigParser()
parser.read(filename)
if not parser.has_section(self.config_section):
parser.add_section(self.config_section)
for name, value in options.items():
parser.set(self.config_section, name, value)
with open(filename, 'w') as handle:
parser.write(handle)
else:
logger.verbose(
"Not storing SSH host keys (no configuration available).")
def prepare_config(config):
"""Prepare the ``~/.vcs-repo-mgr.ini`` configuration file."""
with open(parse_path(USER_CONFIG_FILE), 'w') as handle:
for name, options in config.items():
handle.write('[%s]\n' % name)
for key, value in options.items():
handle.write('%s = %s\n' % (key, value))
def directory(self, value):
"""Normalize the value of :attr:`directory` when it's set."""
# Normalize the value of `directory'.
set_property(self, "directory", parse_path(value))
# Clear the computed values of `context' and `entries'.
clear_property(self, "context")
clear_property(self, "entries")
def pre_boot(self):
"""A connection profile for the pre-boot environment (a :class:`ConnectionProfile` object)."""
if self.config_section or 'pre-boot' in self.config:
return ConnectionProfile(
expression=self.config.get('pre-boot', self.config_section),
identity_file=(parse_path(self.config['identity-file'])
if 'identity-file' in self.config else None),
username='******',
)
def data_directory(self):
"""
The pathname of the directory where data files are stored (a string).
The environment variable ``$CHAT_ARCHIVE_DIRECTORY`` can be used to set
the value of this property. When the environment variable isn't set the
default value ``~/.local/share/chat-archive`` is used (where ``~`` is
expanded to the profile directory of the current user).
"""
return parse_path(os.environ.get("CHAT_ARCHIVE_DIRECTORY", "~/.local/share/chat-archive"))
def initialize_gnupg():
"""
Make sure the ``~/.gnupg`` directory exists.
Older versions of GPG can/will fail when the ``~/.gnupg`` directory doesn't
exist (e.g. in a newly created chroot). GPG itself creates the directory
after noticing that it's missing, but then still fails! Later runs work
fine however. To avoid this problem we make sure ``~/.gnupg`` exists before
we run GPG.
"""
makedirs(parse_path('~/.gnupg'))
def load_config_file(configuration_file=None):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
When `configuration_file` isn't given :data:`LOCAL_CONFIG_FILE` and
:data:`GLOBAL_CONFIG_FILE` are checked and the first configuration file
that exists is loaded. This function is used by :class:`RotateBackups` to
discover user defined rotation schemes and by :mod:`rotate_backups.cli` to
discover directories for which backup rotation is configured.
"""
parser = configparser.RawConfigParser()
if configuration_file:
logger.verbose("Reading configuration file %s ..",
format_path(configuration_file))
loaded_files = parser.read(configuration_file)
if len(loaded_files) == 0:
msg = "Failed to read configuration file! (%s)"
raise ValueError(msg % configuration_file)
else:
for config_file in LOCAL_CONFIG_FILE, GLOBAL_CONFIG_FILE:
pathname = parse_path(config_file)
if parser.read(pathname):
logger.verbose("Reading configuration file %s ..",
format_path(pathname))
break
for section in parser.sections():
items = dict(parser.items(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(
items.get('prefer-recent', 'no')))
yield location, rotation_scheme, options
def initialize_gnupg():
"""
Make sure the ``~/.gnupg`` directory exists.
Older versions of GPG can/will fail when the ``~/.gnupg`` directory doesn't
exist (e.g. in a newly created chroot). GPG itself creates the directory
after noticing that it's missing, but then still fails! Later runs work
fine however. To avoid this problem we make sure ``~/.gnupg`` exists before
we run GPG.
"""
makedirs(parse_path('~/.gnupg'))
def download_cache(self):
"""
The absolute pathname of pip's download cache directory (a string).
- Environment variable: ``$PIP_DOWNLOAD_CACHE``
- Configuration option: ``download-cache``
- Default: ``~/.pip/download-cache``
"""
return parse_path(self.get(property_name='download_cache',
environment_variable='PIP_DOWNLOAD_CACHE',
configuration_option='download-cache',
default='~/.pip/download-cache'))
def check_directory(argument):
"""
Make sure a command line argument points to an existing directory.
:param argument: The original command line argument.
:returns: The absolute pathname of an existing directory.
"""
directory = parse_path(argument)
if not os.path.isdir(directory):
msg = "Directory doesn't exist! (%s)"
raise Exception(msg % directory)
return directory
def check_directory(argument):
"""
Make sure a command line argument points to an existing directory.
:param argument: The original command line argument.
:returns: The absolute pathname of an existing directory.
"""
directory = parse_path(argument)
if not os.path.isdir(directory):
msg = "Directory doesn't exist! (%s)"
raise Exception(msg % directory)
return directory
def data_directory(self):
"""
The absolute pathname of the directory where pip-accel's data files are stored (a string).
- Environment variable: ``$PIP_ACCEL_CACHE``
- Configuration option: ``data-directory``
- Default: ``/var/cache/pip-accel`` if running as ``root``, ``~/.pip-accel`` otherwise
"""
return parse_path(self.get(property_name='data_directory',
environment_variable='PIP_ACCEL_CACHE',
configuration_option='data-directory',
default='/var/cache/pip-accel' if os.getuid() == 0 else '~/.pip-accel'))
def available_configuration_files(self):
"""A list of strings with the absolute pathnames of the available configuration files."""
known_files = [
GLOBAL_CONFIG, LOCAL_CONFIG,
self.environment.get('PIP_ACCEL_CONFIG')
]
absolute_paths = [
parse_path(pathname) for pathname in known_files if pathname
]
return [
pathname for pathname in absolute_paths if os.path.isfile(pathname)
]
def get_password_from_store(name, store=None):
"""Get the disk encryption password from the 'pass' program."""
options = dict(capture=True, shell=False, tty=True)
options['environment'] = dict(GPG_TTY=execute('tty', **options))
if store:
options['environment']['PASSWORD_STORE_DIR'] = parse_path(store)
output = execute('pass', 'show', name, **options)
lines = output.splitlines()
if lines and lines[0]:
return lines[0]
else:
logger.warning(
"Failed to get disk encryption password using 'pass' program!")
def download_cache(self):
"""
The absolute pathname of pip's download cache directory (a string).
- Environment variable: ``$PIP_DOWNLOAD_CACHE``
- Configuration option: ``download-cache``
- Default: ``~/.pip/download-cache``
"""
return parse_path(
self.get(property_name='download_cache',
environment_variable='PIP_DOWNLOAD_CACHE',
configuration_option='download-cache',
default='~/.pip/download-cache'))
def load_config_file(configuration_file=None):
"""
Load a configuration file with backup directories and rotation schemes.
:param configuration_file: Override the pathname of the configuration file
to load (a string or :data:`None`).
:returns: A generator of tuples with four values each:
1. An execution context created using :mod:`executor.contexts`.
2. The pathname of a directory with backups (a string).
3. A dictionary with the rotation scheme.
4. A dictionary with additional options.
:raises: :exc:`~exceptions.ValueError` when `configuration_file` is given
but doesn't exist or can't be loaded.
When `configuration_file` isn't given :data:`LOCAL_CONFIG_FILE` and
:data:`GLOBAL_CONFIG_FILE` are checked and the first configuration file
that exists is loaded. This function is used by :class:`RotateBackups` to
discover user defined rotation schemes and by :mod:`rotate_backups.cli` to
discover directories for which backup rotation is configured.
"""
parser = configparser.RawConfigParser()
if configuration_file:
logger.verbose("Reading configuration file %s ..", format_path(configuration_file))
loaded_files = parser.read(configuration_file)
if len(loaded_files) == 0:
msg = "Failed to read configuration file! (%s)"
raise ValueError(msg % configuration_file)
else:
for config_file in LOCAL_CONFIG_FILE, GLOBAL_CONFIG_FILE:
pathname = parse_path(config_file)
if parser.read(pathname):
logger.verbose("Reading configuration file %s ..", format_path(pathname))
break
for section in parser.sections():
items = dict(parser.items(section))
context_options = {}
if coerce_boolean(items.get('use-sudo')):
context_options['sudo'] = True
if items.get('ssh-user'):
context_options['ssh_user'] = items['ssh-user']
location = coerce_location(section, **context_options)
rotation_scheme = dict((name, coerce_retention_period(items[name]))
for name in SUPPORTED_FREQUENCIES
if name in items)
options = dict(include_list=split(items.get('include-list', '')),
exclude_list=split(items.get('exclude-list', '')),
io_scheduling_class=items.get('ionice'),
strict=coerce_boolean(items.get('strict', 'yes')),
prefer_recent=coerce_boolean(items.get('prefer-recent', 'no')))
yield location, rotation_scheme, options
def data_directory(self):
"""
The absolute pathname of the directory where pip-accel's data files are stored (a string).
- Environment variable: ``$PIP_ACCEL_CACHE``
- Configuration option: ``data-directory``
- Default: ``/var/cache/pip-accel`` if running as ``root``, ``~/.pip-accel`` otherwise
"""
return parse_path(
self.get(property_name='data_directory',
environment_variable='PIP_ACCEL_CACHE',
configuration_option='data-directory',
default='/var/cache/pip-accel'
if os.getuid() == 0 else '~/.pip-accel'))
def directory(self):
"""
The pathname of the password storage directory (a string).
When the environment variable given by :data:`DIRECTORY_VARIABLE` is
set the value of that environment variable is used, otherwise
:data:`DEFAULT_DIRECTORY` is used. In either case the resulting
directory pathname is normalized using
:func:`~humanfriendly.parse_path()`.
When you set the :attr:`directory` property, the value you set will be
normalized using :func:`~humanfriendly.parse_path()` and the computed
value of the :attr:`context` property is cleared.
"""
return parse_path(os.environ.get(DIRECTORY_VARIABLE, DEFAULT_DIRECTORY))
def available_files(self):
"""
The filenames of the available configuration files (a list of strings).
The value of :attr:`available_files` is computed the first time its
needed by searching for available configuration files that match
:attr:`filename_patterns` using :func:`~glob.glob()`. If you set
:attr:`available_files` this effectively disables searching for
configuration files.
"""
matches = []
for pattern in self.filename_patterns:
logger.debug("Matching filename pattern: %s", pattern)
matches.extend(natsort(glob.glob(parse_path(pattern))))
return matches
def find_fixed_agent_socket():
"""
Search for a GPG agent UNIX socket in one of the "fixed locations".
:returns: The pathname of the found socket file (a string) or :data:`None`.
Two locations are searched, in the given order (the first that is found is
returned):
- Starting from GnuPG 2.1.13 the location ``/run/user/$UID/gnupg/S.gpg-agent``
is used (only when the directory ``/run/user/$UID`` exists).
- GnuPG 2.1 removed the ``$GPG_AGENT_INFO`` related code and switched to
the fixed location ``~/.gnupg/S.gpg-agent``.
"""
socket_on_tmpfs = '/run/user/%i/gnupg/S.gpg-agent' % os.getuid()
for socket in (socket_on_tmpfs, parse_path(NEW_STYLE_SOCKET)):
if os.path.exists(socket):
return socket
def load_configuration_file(self, configuration_file):
"""
Load configuration defaults from a configuration file.
:param configuration_file: The pathname of a configuration file (a
string).
:raises: :py:exc:`Exception` when the configuration file cannot be
loaded.
"""
configuration_file = parse_path(configuration_file)
logger.debug("Loading configuration file: %s", configuration_file)
parser = configparser.RawConfigParser()
files_loaded = parser.read(configuration_file)
if len(files_loaded) != 1:
msg = "Failed to load configuration file! (%s)"
raise Exception(msg % configuration_file)
elif not parser.has_section('pip-accel'):
msg = "Missing 'pip-accel' section in configuration file! (%s)"
raise Exception(msg % configuration_file)
else:
self.configuration.update(parser.items('pip-accel'))
def load_configuration_file(self, configuration_file):
"""
Load configuration defaults from a configuration file.
:param configuration_file: The pathname of a configuration file (a
string).
:raises: :py:exc:`Exception` when the configuration file cannot be
loaded.
"""
configuration_file = parse_path(configuration_file)
logger.debug("Loading configuration file: %s", configuration_file)
parser = configparser.RawConfigParser()
files_loaded = parser.read(configuration_file)
if len(files_loaded) != 1:
msg = "Failed to load configuration file! (%s)"
raise Exception(msg % configuration_file)
elif not parser.has_section('pip-accel'):
msg = "Missing 'pip-accel' section in configuration file! (%s)"
raise Exception(msg % configuration_file)
else:
self.configuration.update(parser.items('pip-accel'))
def expand_path(pathname):
"""
Expand the home directory in a pathname based on the effective user id.
:param pathname: A pathname that may start with ``~/``, indicating the path
should be interpreted as being relative to the home
directory of the current (effective) user.
:returns: The (modified) pathname.
This function is a variant of :func:`os.path.expanduser()` that doesn't use
``$HOME`` but instead uses the home directory of the effective user id.
This is basically a workaround for ``sudo -s`` not resetting ``$HOME``.
"""
# The following logic previously used regular expressions but that approach
# turned out to be very error prone, hence the current contraption based on
# direct string manipulation :-).
home_directory = find_home_directory()
separators = set([os.sep])
if os.altsep is not None:
separators.add(os.altsep)
if len(pathname) >= 2 and pathname[0] == '~' and pathname[1] in separators:
pathname = os.path.join(home_directory, pathname[2:])
# Also expand environment variables.
return parse_path(pathname)
def expand_path(pathname):
"""
Expand the home directory in a pathname based on the effective user id.
:param pathname: A pathname that may start with ``~/``, indicating the path
should be interpreted as being relative to the home
directory of the current (effective) user.
:returns: The (modified) pathname.
This function is a variant of :func:`os.path.expanduser()` that doesn't use
``$HOME`` but instead uses the home directory of the effective user id.
This is basically a workaround for ``sudo -s`` not resetting ``$HOME``.
"""
# The following logic previously used regular expressions but that approach
# turned out to be very error prone, hence the current contraption based on
# direct string manipulation :-).
home_directory = find_home_directory()
separators = set([os.sep])
if os.altsep is not None:
separators.add(os.altsep)
if len(pathname) >= 2 and pathname[0] == '~' and pathname[1] in separators:
pathname = os.path.join(home_directory, pathname[2:])
# Also expand environment variables.
return parse_path(pathname)
def __init__(self,
name=None,
description=None,
secret_key_file=None,
public_key_file=None,
key_id=None):
"""
Initialize a GPG key object.
:param name: The name of the GPG key pair (a string). Used only when
the key pair is generated because it doesn't exist yet.
:param description: The description of the GPG key pair (a string).
Used only when the key pair is generated because it
doesn't exist yet.
:param secret_key_file: The absolute pathname of the secret key file (a
string). Defaults to ``~/.gnupg/secring.gpg``.
:param public_key_file: The absolute pathname of the public key file (a
string). Defaults to ``~/.gnupg/pubring.gpg``.
:param key_id: The key ID of an existing key pair to use (a string). If
this argument is provided then the key pair's secret and
public key files must already exist.
This method initializes a GPG key object in one of several ways:
1. If `key_id` is specified then the GPG key must have been created
previously. If `secret_key_file` and `public_key_file` are not
specified they default to ``~/.gnupg/secring.gpg`` and
``~/.gnupg/pubring.gpg``. In this case `key_id` is the only
required argument.
The following example assumes that the provided GPG key ID is
defined in the default keyring of the current user:
>>> from deb_pkg_tools.gpg import GPGKey
>>> key = GPGKey(key_id='58B6B02B')
>>> key.gpg_command
'gpg --no-default-keyring --secret-keyring /home/peter/.gnupg/secring.gpg --keyring /home/peter/.gnupg/pubring.gpg --recipient 58B6B02B'
2. If `secret_key_file` and `public_key_file` are specified but the
files don't exist yet, a GPG key will be generated for you. In this
case `name` and `description` are required arguments and `key_id`
must be :data:`None` (the default). An example:
>>> name = 'deb-pkg-tools'
>>> description = 'Automatic signing key for deb-pkg-tools'
>>> secret_key_file = '/home/peter/.deb-pkg-tools/automatic-signing-key.sec'
>>> public_key_file = '/home/peter/.deb-pkg-tools/automatic-signing-key.pub'
>>> key = GPGKey(name, description, secret_key_file, public_key_file)
>>> key.gpg_command
'gpg --no-default-keyring --secret-keyring /home/peter/.deb-pkg-tools/automatic-signing-key.sec --keyring /home/peter/.deb-pkg-tools/automatic-signing-key.pub'
"""
# If the secret or public key file is provided, the other key file must
# be provided as well.
if secret_key_file and not public_key_file:
raise Exception(
"You provided a GPG secret key file without a public key file; please provide both!"
)
elif public_key_file and not secret_key_file:
raise Exception(
"You provided a GPG public key file without a secret key file; please provide both!"
)
# If neither of the key files is provided we'll default to the
# locations that GnuPG uses by default.
if not secret_key_file and not public_key_file:
gnupg_directory = parse_path('~/.gnupg')
secret_key_file = os.path.join(gnupg_directory, 'secring.gpg')
public_key_file = os.path.join(gnupg_directory, 'pubring.gpg')
# If a key ID was specified then the two key files must already exist;
# we won't generate them because that makes no sense :-)
if key_id and not os.path.isfile(secret_key_file):
text = "The provided GPG secret key file (%s) doesn't exist but a key ID was specified!"
raise Exception(text % secret_key_file)
if key_id and not os.path.isfile(public_key_file):
text = "The provided GPG public key file (%s) doesn't exist but a key ID was specified!"
raise Exception(text % public_key_file)
# If we're going to generate a GPG key for the caller we don't want to
# overwrite a secret or public key file without its counterpart. We'll
# also need a name and description for the generated key.
existing_files = list(
filter(os.path.isfile, [secret_key_file, public_key_file]))
if len(existing_files) not in (0, 2):
text = "Refusing to overwrite existing key file! (%s)"
raise Exception(text % existing_files[0])
elif len(existing_files) == 0 and not (name and description):
logger.error("GPG key pair doesn't exist! (%s and %s)",
format_path(secret_key_file),
format_path(public_key_file))
raise Exception(
"To generate a GPG key you must provide a name and description!"
)
# Store the arguments.
self.name = name
self.description = description
self.secret_key_file = secret_key_file
self.public_key_file = public_key_file
self.key_id = key_id
# Generate the GPG key pair if required.
if not existing_files:
# Make sure the directories of the secret/public key files exist.
for filename in [secret_key_file, public_key_file]:
makedirs(os.path.dirname(filename))
# Generate a file with batch instructions
# suitable for `gpg --batch --gen-key'.
fd, gpg_script = tempfile.mkstemp()
with open(gpg_script, 'w') as handle:
handle.write(
textwrap.dedent('''
Key-Type: DSA
Key-Length: 1024
Subkey-Type: ELG-E
Subkey-Length: 1024
Name-Real: {name}
Name-Comment: {description}
Name-Email: none
Expire-Date: 0
%pubring {public_key_file}
%secring {secret_key_file}
%commit
''').format(name=self.name,
description=self.description,
secret_key_file=self.secret_key_file,
public_key_file=self.public_key_file))
# Generate the GPG key pair.
logger.info("Generating GPG key pair %s (%s) ..", self.name,
self.description)
logger.debug("Private key: %s", format_path(self.secret_key_file))
logger.debug("Public key: %s", format_path(self.public_key_file))
logger.info(
"Please note: Generating a GPG key pair can take a long time. "
"If you are logged into a virtual machine or a remote server "
"over SSH, now is a good time to familiarize yourself with "
"the concept of entropy and how to make more of it :-)")
start_time = time.time()
initialize_gnupg()
with EntropyGenerator():
execute('gpg',
'--batch',
'--gen-key',
gpg_script,
logger=logger)
logger.info("Finished generating GPG key pair in %s.",
format_timespan(time.time() - start_time))
os.unlink(gpg_script)
def main():
"""Command line interface for ``unlock-remote-system``."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
program_opts = {}
identity_file = None
do_shell = False
do_watch = False
watch_all = False
try:
options, arguments = getopt.gnu_getopt(sys.argv[1:], 'i:k:p:r:swavqh',
[
'identity-file=',
'known-hosts=',
'password='******'remote-host=',
'shell',
'watch',
'all',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-i', '--identity-file'):
identity_file = parse_path(value)
elif option in ('-k', '--known-hosts'):
program_opts['known_hosts_file'] = parse_path(value)
elif option in ('-p', '--password'):
program_opts['password'] = get_password_from_store(value)
elif option in ('-r', '--remote-host'):
program_opts['ssh_proxy'] = value
elif option in ('-s', '--shell'):
do_shell = True
elif option in ('-w', '--watch'):
do_watch = True
elif option in ('-a', '--all'):
watch_all = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
else:
raise Exception("Unhandled option!")
if not arguments:
usage(__doc__)
sys.exit(0)
elif len(arguments) > 2:
raise Exception("only two positional arguments allowed")
# Create a ConfigLoader object and prepare to pass it to the program to
# avoid scanning for configuration files more than once (which isn't a
# real problem but does generate somewhat confusing log output).
loader = ConfigLoader(program_name='unlock-remote-system')
program_opts['config_loader'] = loader
# Check if a single positional argument was given that matches the name
# of a user defined configuration section.
if len(arguments) == 1 and arguments[0] in loader.section_names:
logger.info("Loading configuration section '%s' ..", arguments[0])
program_opts['config_section'] = arguments[0]
else:
# The SSH connection profile of the pre-boot environment
# is given as the first positional argument.
program_opts['pre_boot'] = ConnectionProfile(
expression=arguments[0], identity_file=identity_file)
# The SSH connection profile of the post-boot environment
# can be given as the second positional argument, otherwise
# it will be inferred from the connection profile of
# the pre-boot environment.
if len(arguments) == 2:
program_opts['post_boot'] = ConnectionProfile(
expression=arguments[1])
else:
# By default we don't use root to login to the post-boot environment.
program_opts['post_boot'] = ConnectionProfile(
expression=arguments[0])
program_opts['post_boot'].username = find_local_username()
# Prompt the operator to enter the disk encryption password for the remote host?
if not program_opts.get('password'):
program_opts['password'] = prompt_for_password(
program_opts['pre_boot'].hostname)
except Exception as e:
warning("Failed to parse command line arguments! (%s)", e)
sys.exit(1)
# Try to unlock the remote system.
try:
if do_watch and watch_all:
watch_all_systems(loader)
else:
with EncryptedSystem(**program_opts) as program:
if do_watch:
program.watch_system()
else:
program.unlock_system()
if do_shell:
start_interactive_shell(program.post_context)
except EncryptedSystemError as e:
logger.error("Aborting due to error: %s", e)
sys.exit(2)
except Exception:
logger.exception("Aborting due to unexpected exception!")
sys.exit(3)
def main():
"""Command line interface for the ``chat-archive`` program."""
# Enable logging to the terminal.
coloredlogs.install()
# Parse the command line options.
program_opts = dict()
command_name = None
try:
options, arguments = getopt.gnu_getopt(
sys.argv[1:],
"C:fl:c:p:vqh",
[
"context=",
"force",
"log-file=",
"color=",
"colour=",
"profile=",
"verbose",
"quiet",
"help",
],
)
for option, value in options:
if option in ("-C", "--context"):
program_opts["context"] = int(value)
elif option in ("-f", "--force"):
program_opts["force"] = True
elif option in ("-l", "--log-file"):
handler = logging.FileHandler(parse_path(value))
handler.setFormatter(
logging.Formatter(
fmt=
"%(asctime)s %(name)s[%(process)d] %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"))
handler.setLevel(logging.DEBUG)
logging.root.addHandler(handler)
logging.root.setLevel(logging.NOTSET)
elif option in ("-c", "--color", "--colour"):
mapping = dict(always=True, never=False)
program_opts["use_colors"] = mapping[
value] if value in mapping else coerce_boolean(value)
elif option in ("-p", "--profile"):
program_opts["profile_file"] = parse_path(value)
elif option in ("-v", "--verbose"):
coloredlogs.increase_verbosity()
elif option in ("-q", "--quiet"):
coloredlogs.decrease_verbosity()
elif option in ("-h", "--help"):
usage(__doc__)
sys.exit(0)
else:
assert False, "Unhandled option!"
# Make sure the operator provided a command.
if not arguments:
usage(__doc__)
sys.exit(0)
except Exception as e:
warning("Failed to parse command line arguments: %s", e)
sys.exit(1)
try:
# We extract any search keywords from the command line arguments before
# initializing an instance of the UserInterface class, to enable
# initialization of the KeywordHighlighter class.
if arguments[0] == "search":
program_opts["keywords"] = arguments[1:]
# Initialize the chat archive.
with UserInterface(**program_opts) as program:
# Validate the requested command.
command_name = arguments.pop(0)
method_name = "%s_cmd" % command_name
if not hasattr(program, method_name):
warning("Error: Invalid command name '%s'!", command_name)
sys.exit(1)
# Execute the requested command.
command_fn = getattr(program, method_name)
command_fn(arguments)
except KeyboardInterrupt:
logger.notice("Interrupted by Control-C ..")
sys.exit(1)
except Exception:
logger.exception("Aborting due to unexpected exception!")
sys.exit(1)
def available_configuration_files(self):
"""A list of strings with the absolute pathnames of the available configuration files."""
known_files = [GLOBAL_CONFIG, LOCAL_CONFIG, self.environment.get('PIP_ACCEL_CONFIG')]
absolute_paths = [parse_path(pathname) for pathname in known_files if pathname]
return [pathname for pathname in absolute_paths if os.path.isfile(pathname)]
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
config_file = None
dry_run = False
exclude_list = []
include_list = []
io_scheduling_class = None
rotation_scheme = {}
use_sudo = False
strict = True
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'H:d:w:m:y:I:x:ri:c:r:unvqh', [
'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=', 'include=',
'exclude=', 'relaxed', 'ionice=', 'config=', 'use-sudo', 'dry-run',
'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-r', '--relaxed'):
strict = False
elif option in ('-i', '--ionice'):
value = value.lower().strip()
expected = ('idle', 'best-effort', 'realtime')
if value not in expected:
msg = "Invalid I/O scheduling class! (got %r while valid options are %s)"
raise Exception(msg % (value, concatenate(expected)))
io_scheduling_class = value
elif option in ('-c', '--config'):
config_file = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..", option)
dry_run = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.debug("Parsed rotation scheme: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
selected_locations.extend(location for location, rotation_scheme, options in load_config_file(config_file))
# Show the usage message when no directories are given nor configured.
if not selected_locations:
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
for location in selected_locations:
RotateBackups(
rotation_scheme=rotation_scheme,
include_list=include_list,
exclude_list=exclude_list,
io_scheduling_class=io_scheduling_class,
dry_run=dry_run,
config_file=config_file,
strict=strict,
).rotate_backups(location)
"""Configuration defaults for the `deb-pkg-tools` package."""
# Standard library modules.
import os
# External dependencies.
from humanfriendly import parse_path
system_config_directory = '/etc/deb-pkg-tools'
"""The pathname of the global (system wide) configuration directory used by `deb-pkg-tools` (a string)."""
system_cache_directory = '/var/cache/deb-pkg-tools'
"""The pathname of the global (system wide) package cache directory (a string)."""
user_config_directory = parse_path('~/.deb-pkg-tools')
"""
The pathname of the current user's configuration directory used by `deb-pkg-tools` (a string).
:default: The expanded value of ``~/.deb-pkg-tools``.
"""
user_cache_directory = parse_path('~/.cache/deb-pkg-tools')
"""
The pathname of the current user's package cache directory (a string).
:default: The expanded value of ``~/.cache/deb-pkg-tools``.
"""
# The location of the package cache. If we're running as root we have write
# access to the system wide package cache so we'll pick that; the more users
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
config_file = None
dry_run = False
exclude_list = []
include_list = []
io_scheduling_class = None
rotation_scheme = {}
use_sudo = False
strict = True
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'H:d:w:m:y:I:x:ri:c:r:unvqh', [
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'relaxed',
'ionice=',
'config=',
'use-sudo',
'dry-run',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-r', '--relaxed'):
strict = False
elif option in ('-i', '--ionice'):
value = value.lower().strip()
expected = ('idle', 'best-effort', 'realtime')
if value not in expected:
msg = "Invalid I/O scheduling class! (got %r while valid options are %s)"
raise Exception(msg % (value, concatenate(expected)))
io_scheduling_class = value
elif option in ('-c', '--config'):
config_file = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
dry_run = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.debug("Parsed rotation scheme: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
selected_locations.extend(
coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
selected_locations.extend(location for location, rotation_scheme,
options in load_config_file(config_file))
# Show the usage message when no directories are given nor configured.
if not selected_locations:
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
for location in selected_locations:
RotateBackups(
rotation_scheme=rotation_scheme,
include_list=include_list,
exclude_list=exclude_list,
io_scheduling_class=io_scheduling_class,
dry_run=dry_run,
config_file=config_file,
strict=strict,
).rotate_backups(location)
def __init__(self, name=None, description=None, secret_key_file=None, public_key_file=None, key_id=None):
"""
Initialize a GPG key object.
:param name: The name of the GPG key pair (a string). Used only when
the key pair is generated because it doesn't exist yet.
:param description: The description of the GPG key pair (a string).
Used only when the key pair is generated because it
doesn't exist yet.
:param secret_key_file: The absolute pathname of the secret key file (a
string). Defaults to ``~/.gnupg/secring.gpg``.
:param public_key_file: The absolute pathname of the public key file (a
string). Defaults to ``~/.gnupg/pubring.gpg``.
:param key_id: The key ID of an existing key pair to use (a string). If
this argument is provided then the key pair's secret and
public key files must already exist.
This method initializes a GPG key object in one of several ways:
1. If `key_id` is specified then the GPG key must have been created
previously. If `secret_key_file` and `public_key_file` are not
specified they default to ``~/.gnupg/secring.gpg`` and
``~/.gnupg/pubring.gpg``. In this case `key_id` is the only
required argument.
The following example assumes that the provided GPG key ID is
defined in the default keyring of the current user:
>>> from deb_pkg_tools.gpg import GPGKey
>>> key = GPGKey(key_id='58B6B02B')
>>> key.gpg_command
'gpg --no-default-keyring --secret-keyring /home/peter/.gnupg/secring.gpg --keyring /home/peter/.gnupg/pubring.gpg --recipient 58B6B02B'
2. If `secret_key_file` and `public_key_file` are specified but the
files don't exist yet, a GPG key will be generated for you. In this
case `name` and `description` are required arguments and `key_id`
must be :data:`None` (the default). An example:
>>> name = 'deb-pkg-tools'
>>> description = 'Automatic signing key for deb-pkg-tools'
>>> secret_key_file = '/home/peter/.deb-pkg-tools/automatic-signing-key.sec'
>>> public_key_file = '/home/peter/.deb-pkg-tools/automatic-signing-key.pub'
>>> key = GPGKey(name, description, secret_key_file, public_key_file)
>>> key.gpg_command
'gpg --no-default-keyring --secret-keyring /home/peter/.deb-pkg-tools/automatic-signing-key.sec --keyring /home/peter/.deb-pkg-tools/automatic-signing-key.pub'
"""
# If the secret or public key file is provided, the other key file must
# be provided as well.
if secret_key_file and not public_key_file:
raise Exception("You provided a GPG secret key file without a public key file; please provide both!")
elif public_key_file and not secret_key_file:
raise Exception("You provided a GPG public key file without a secret key file; please provide both!")
# If neither of the key files is provided we'll default to the
# locations that GnuPG uses by default.
if not secret_key_file and not public_key_file:
gnupg_directory = parse_path('~/.gnupg')
secret_key_file = os.path.join(gnupg_directory, 'secring.gpg')
public_key_file = os.path.join(gnupg_directory, 'pubring.gpg')
# If a key ID was specified then the two key files must already exist;
# we won't generate them because that makes no sense :-)
if key_id and not os.path.isfile(secret_key_file):
text = "The provided GPG secret key file (%s) doesn't exist but a key ID was specified!"
raise Exception(text % secret_key_file)
if key_id and not os.path.isfile(public_key_file):
text = "The provided GPG public key file (%s) doesn't exist but a key ID was specified!"
raise Exception(text % public_key_file)
# If we're going to generate a GPG key for the caller we don't want to
# overwrite a secret or public key file without its counterpart. We'll
# also need a name and description for the generated key.
existing_files = list(filter(os.path.isfile, [secret_key_file, public_key_file]))
if len(existing_files) not in (0, 2):
text = "Refusing to overwrite existing key file! (%s)"
raise Exception(text % existing_files[0])
elif len(existing_files) == 0 and not (name and description):
logger.error("GPG key pair doesn't exist! (%s and %s)",
format_path(secret_key_file),
format_path(public_key_file))
raise Exception("To generate a GPG key you must provide a name and description!")
# Store the arguments.
self.name = name
self.description = description
self.secret_key_file = secret_key_file
self.public_key_file = public_key_file
self.key_id = key_id
# Generate the GPG key pair if required.
if not existing_files:
# Make sure the directories of the secret/public key files exist.
for filename in [secret_key_file, public_key_file]:
makedirs(os.path.dirname(filename))
# Generate a file with batch instructions
# suitable for `gpg --batch --gen-key'.
fd, gpg_script = tempfile.mkstemp()
with open(gpg_script, 'w') as handle:
handle.write(textwrap.dedent('''
Key-Type: DSA
Key-Length: 1024
Subkey-Type: ELG-E
Subkey-Length: 1024
Name-Real: {name}
Name-Comment: {description}
Name-Email: none
Expire-Date: 0
%pubring {public_key_file}
%secring {secret_key_file}
%commit
''').format(name=self.name,
description=self.description,
secret_key_file=self.secret_key_file,
public_key_file=self.public_key_file))
# Generate the GPG key pair.
logger.info("Generating GPG key pair %s (%s) ..", self.name, self.description)
logger.debug("Private key: %s", format_path(self.secret_key_file))
logger.debug("Public key: %s", format_path(self.public_key_file))
logger.info("Please note: Generating a GPG key pair can take a long time. "
"If you are logged into a virtual machine or a remote server "
"over SSH, now is a good time to familiarize yourself with "
"the concept of entropy and how to make more of it :-)")
start_time = time.time()
initialize_gnupg()
with EntropyGenerator():
execute('gpg', '--batch', '--gen-key', gpg_script, logger=logger)
logger.info("Finished generating GPG key pair in %s.",
format_timespan(time.time() - start_time))
os.unlink(gpg_script)
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
rotation_scheme = {}
kw = dict(include_list=[], exclude_list=[])
parallel = False
use_sudo = False
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh', [
'minutely=', 'hourly=', 'daily=', 'weekly=', 'monthly=', 'yearly=',
'include=', 'exclude=', 'parallel', 'prefer-recent', 'relaxed',
'ionice=', 'config=', 'use-sudo', 'dry-run', 'removal-command=',
'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-M', '--minutely'):
rotation_scheme['minutely'] = coerce_retention_period(value)
elif option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
kw['include_list'].append(value)
elif option in ('-x', '--exclude'):
kw['exclude_list'].append(value)
elif option in ('-j', '--parallel'):
parallel = True
elif option in ('-p', '--prefer-recent'):
kw['prefer_recent'] = True
elif option in ('-r', '--relaxed'):
kw['strict'] = False
elif option in ('-i', '--ionice'):
value = validate_ionice_class(value.lower().strip())
kw['io_scheduling_class'] = value
elif option in ('-c', '--config'):
kw['config_file'] = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..", option)
kw['dry_run'] = True
elif option in ('-C', '--removal-command'):
removal_command = shlex.split(value)
logger.info("Using custom removal command: %s", removal_command)
kw['removal_command'] = removal_command
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.verbose("Rotation scheme defined on command line: %s", rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
location_source = 'command line arguments'
selected_locations.extend(coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
location_source = 'configuration file'
selected_locations.extend(
location for location, rotation_scheme, options
in load_config_file(configuration_file=kw.get('config_file'), expand=True)
)
# Inform the user which location(s) will be rotated.
if selected_locations:
logger.verbose("Selected %s based on %s:",
pluralize(len(selected_locations), "location"),
location_source)
for number, location in enumerate(selected_locations, start=1):
logger.verbose(" %i. %s", number, location)
else:
# Show the usage message when no directories are given nor configured.
logger.verbose("No location(s) to rotate selected.")
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
program = RotateBackups(rotation_scheme, **kw)
if parallel:
program.rotate_concurrent(*selected_locations)
else:
for location in selected_locations:
program.rotate_backups(location)
def test_parse_path(self):
"""Test :func:`humanfriendly.parse_path()`."""
friendly_path = os.path.join('~', '.vimrc')
absolute_path = os.path.join(os.environ['HOME'], '.vimrc')
self.assertEqual(absolute_path, humanfriendly.parse_path(friendly_path))
def directory_default(self):
"""The pathname of the default GnuPG home directory (a string)."""
return parse_path('~/.gnupg')
def cache_directory(self):
"""The absolute pathname of the directory where ``node_modules`` directories are cached (a string)."""
return ('/var/cache/npm-accel'
if os.getuid() == 0 and os.access('/var/cache', os.W_OK) else
parse_path('~/.cache/npm-accel'))
def main():
"""Command line interface for the ``npm-accel`` program."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Command line option defaults.
program_opts = {}
context_opts = {}
directory = None
action = 'install'
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'pi:c:l:nbr:vqh', [
'production',
'installer=',
'cache-directory=',
'cache-limit=',
'no-cache',
'benchmark',
'remote-host=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('p', '--production'):
program_opts['production'] = True
elif option in ('i', '--installer'):
program_opts['installer_name'] = value
elif option in ('-c', '--cache-directory'):
program_opts['cache_directory'] = parse_path(value)
elif option in ('-l', '--cache-limit'):
program_opts['cache_limit'] = int(value)
elif option in ('-n', '--no-cache'):
program_opts['write_to_cache'] = False
elif option in ('-b', '--benchmark'):
action = 'benchmark'
elif option in ('-r', '--remote-host'):
context_opts['ssh_alias'] = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option!"
if arguments:
directory = arguments.pop(0)
if arguments:
raise Exception("Got more positional arguments than expected!")
if not directory:
if context_opts.get('ssh_alias'):
raise Exception(
"When operating on a remote system the directory needs to be specified explicitly!"
)
directory = os.getcwd()
except Exception as e:
warning("Error: Failed to parse command line arguments! (%s)" % e)
sys.exit(1)
# Perform the requested action(s).
try:
context = create_context(**context_opts)
program_opts['context'] = context
accelerator = NpmAccel(**program_opts)
method = getattr(accelerator, action)
method(directory)
except NpmAccelError as e:
warning("Error: %s", e)
sys.exit(1)
except Exception:
logger.exception("Encountered unexpected exception! Aborting ..")
sys.exit(1)
def main():
"""Command line interface for the ``rotate-backups-s3`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
aws_access_key_id = None
aws_secret_access_key = None
aws_host = 's3.amazonaws.com'
config_file = None
dry_run = False
exclude_list = []
include_list = []
rotation_scheme = {}
prefer_recent = False
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'U:P:H:d:w:m:y:I:x:c:nvhp', [
'aws-access-key-id=',
'aws-secret-access-key=',
'aws-host=',
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'config=',
'dry-run',
'verbose',
'help',
'prefer-recent',
])
for option, value in options:
if option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
include_list.append(value)
elif option in ('-x', '--exclude'):
exclude_list.append(value)
elif option in ('-c', '--config'):
config_file = parse_path(value)
elif option in ('-U', '--aws-access-key-id'):
aws_access_key_id = value
elif option in ('-P', '--aws-secret-access-key'):
aws_secret_access_key = value
elif option in ('--aws-host'):
aws_host = value
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
dry_run = True
elif option in ('-p', '--prefer-recent'):
prefer_recent = True
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.debug("Parsed rotation scheme: %s", rotation_scheme)
# If no arguments are given but the system has a configuration file
# then the backups in the configured directories are rotated.
if not arguments:
arguments.extend(s3path
for s3path, _, _ in load_config_file(config_file))
# Show the usage message when no directories are given nor configured.
if not arguments:
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the given or configured directories.
for s3path in arguments:
prefix = ''
bucket = s3path[5:] if s3path.startswith('s3://') else s3path
pos = bucket.find('/')
if pos != -1:
prefix = bucket[pos:].strip('/')
bucket = bucket[:pos]
S3RotateBackups(
rotation_scheme=rotation_scheme,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_host=aws_host,
include_list=include_list,
exclude_list=exclude_list,
dry_run=dry_run,
prefer_recent=prefer_recent,
).rotate_backups(bucket, prefix)
def test_parse_path(self):
"""Test :func:`humanfriendly.parse_path()`."""
friendly_path = os.path.join('~', '.vimrc')
absolute_path = os.path.join(os.environ['HOME'], '.vimrc')
self.assertEqual(absolute_path, humanfriendly.parse_path(friendly_path))
def main():
"""Command line interface for the ``rotate-backups`` program."""
coloredlogs.install(syslog=True)
# Command line option defaults.
rotation_scheme = {}
kw = dict(include_list=[], exclude_list=[])
parallel = False
use_sudo = False
# Internal state.
selected_locations = []
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:],
'M:H:d:w:m:y:I:x:jpri:c:r:uC:nvqh',
[
'minutely=',
'hourly=',
'daily=',
'weekly=',
'monthly=',
'yearly=',
'include=',
'exclude=',
'parallel',
'prefer-recent',
'relaxed',
'ionice=',
'config=',
'use-sudo',
'dry-run',
'removal-command=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-M', '--minutely'):
rotation_scheme['minutely'] = coerce_retention_period(value)
elif option in ('-H', '--hourly'):
rotation_scheme['hourly'] = coerce_retention_period(value)
elif option in ('-d', '--daily'):
rotation_scheme['daily'] = coerce_retention_period(value)
elif option in ('-w', '--weekly'):
rotation_scheme['weekly'] = coerce_retention_period(value)
elif option in ('-m', '--monthly'):
rotation_scheme['monthly'] = coerce_retention_period(value)
elif option in ('-y', '--yearly'):
rotation_scheme['yearly'] = coerce_retention_period(value)
elif option in ('-I', '--include'):
kw['include_list'].append(value)
elif option in ('-x', '--exclude'):
kw['exclude_list'].append(value)
elif option in ('-j', '--parallel'):
parallel = True
elif option in ('-p', '--prefer-recent'):
kw['prefer_recent'] = True
elif option in ('-r', '--relaxed'):
kw['strict'] = False
elif option in ('-i', '--ionice'):
value = validate_ionice_class(value.lower().strip())
kw['io_scheduling_class'] = value
elif option in ('-c', '--config'):
kw['config_file'] = parse_path(value)
elif option in ('-u', '--use-sudo'):
use_sudo = True
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
kw['dry_run'] = True
elif option in ('-C', '--removal-command'):
removal_command = shlex.split(value)
logger.info("Using custom removal command: %s",
removal_command)
kw['removal_command'] = removal_command
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option! (programming error)"
if rotation_scheme:
logger.verbose("Rotation scheme defined on command line: %s",
rotation_scheme)
if arguments:
# Rotation of the locations given on the command line.
location_source = 'command line arguments'
selected_locations.extend(
coerce_location(value, sudo=use_sudo) for value in arguments)
else:
# Rotation of all configured locations.
location_source = 'configuration file'
selected_locations.extend(
location
for location, rotation_scheme, options in load_config_file(
configuration_file=kw.get('config_file'), expand=True))
# Inform the user which location(s) will be rotated.
if selected_locations:
logger.verbose("Selected %s based on %s:",
pluralize(len(selected_locations), "location"),
location_source)
for number, location in enumerate(selected_locations, start=1):
logger.verbose(" %i. %s", number, location)
else:
# Show the usage message when no directories are given nor configured.
logger.verbose("No location(s) to rotate selected.")
usage(__doc__)
return
except Exception as e:
logger.error("%s", e)
sys.exit(1)
# Rotate the backups in the selected directories.
program = RotateBackups(rotation_scheme, **kw)
if parallel:
program.rotate_concurrent(*selected_locations)
else:
for location in selected_locations:
program.rotate_backups(location)
def pathname_normalize(p):
return humanfriendly.parse_path(p)
# URL: https://github.com/xolox/python-deb-pkg-tools
"""Configuration defaults for the `deb-pkg-tools` package."""
# Standard library modules.
import os
# External dependencies.
from humanfriendly import parse_path
system_config_directory = '/etc/deb-pkg-tools'
"""The pathname of the global (system wide) configuration directory used by `deb-pkg-tools` (a string)."""
system_cache_directory = '/var/cache/deb-pkg-tools'
"""The pathname of the global (system wide) package cache directory (a string)."""
user_config_directory = parse_path('~/.deb-pkg-tools')
"""
The pathname of the current user's configuration directory used by `deb-pkg-tools` (a string).
:default: The expanded value of ``~/.deb-pkg-tools``.
"""
user_cache_directory = parse_path('~/.cache/deb-pkg-tools')
"""
The pathname of the current user's package cache directory (a string).
:default: The expanded value of ``~/.cache/deb-pkg-tools``.
"""
# The location of the package cache. If we're running as root we have write
# access to the system wide package cache so we'll pick that; the more users
