def coerce_location(value, **options):
"""
Coerce a string to a :class:`Location` object.
:param value: The value to coerce (a string or :class:`Location` object).
:param options: Any keyword arguments are passed on to
:func:`~executor.contexts.create_context()`.
:returns: A :class:`Location` object.
"""
# Location objects pass through untouched.
if not isinstance(value, Location):
# Other values are expected to be strings.
if not isinstance(value, string_types):
msg = "Expected Location object or string, got %s instead!"
raise ValueError(msg % type(value))
# Try to parse a remote location.
ssh_alias, _, directory = value.partition(':')
if ssh_alias and directory and '/' not in ssh_alias:
options['ssh_alias'] = ssh_alias
else:
directory = value
# Create the location object.
value = Location(
context=create_context(**options),
directory=parse_path(directory),
)
return value
def coerce_location(value, **options):
"""
Coerce a string to a :class:`Location` object.
:param value: The value to coerce (a string or :class:`Location` object).
:param options: Any keyword arguments are passed on to
:func:`~executor.contexts.create_context()`.
:returns: A :class:`Location` object.
"""
# Location objects pass through untouched.
if not isinstance(value, Location):
# Other values are expected to be strings.
if not isinstance(value, string_types):
msg = "Expected Location object or string, got %s instead!"
raise ValueError(msg % type(value))
# Try to parse a remote location.
ssh_alias, _, directory = value.partition(':')
if ssh_alias and directory and '/' not in ssh_alias:
options['ssh_alias'] = ssh_alias
else:
directory = value
# Create the location object.
value = Location(
context=create_context(**options),
directory=parse_path(directory),
)
return value
def main():
"""Command line interface for ``debuntu-nodejs-installer``."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
silence_urllib_logger()
# Parse the command line arguments.
action = None
context_opts = dict()
installer_opts = dict()
try:
options, arguments = getopt.getopt(sys.argv[1:], 'iV:s:r:vqh', [
'install',
'version=',
'sources-file=',
'remote-host=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-i', '--install'):
action = 'install'
elif option in ('-V', '--version'):
installer_opts['nodejs_version'] = value
elif option in ('-s', '--sources-file'):
installer_opts['sources_file'] = value
elif option in ('-r', '--remote-host'):
context_opts['ssh_alias'] = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
else:
raise Exception("Unhandled option!")
if arguments:
raise Exception(
"This program doesn't accept any positional arguments!")
if not action:
usage(__doc__)
sys.exit(0)
except Exception as e:
warning("Failed to parse command line arguments! (%s)", e)
sys.exit(1)
# Execute the requested action.
context = create_context(**context_opts)
try:
installer = NodeInstaller(context=context, **installer_opts)
getattr(installer, action)()
except (UnsupportedSystemError, ExternalCommandFailed) as e:
logger.error("%s", e)
sys.exit(1)
except Exception:
logger.exception("Encountered unexpected exception on %s!", context)
sys.exit(1)
def main():
"""Command line interface for ``debuntu-kernel-manager``."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
action = 'render_summary'
context_opts = dict()
manager_opts = dict()
try:
options, arguments = getopt.getopt(sys.argv[1:], 'cfp:r:vqh', [
'clean',
'remove',
'force',
'preserve-count=',
'remote-host=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-c', '--clean', '--remove'):
action = 'cleanup_packages'
elif option in ('-f', '--force'):
manager_opts['force'] = True
elif option in ('-p', '--preserve-count'):
manager_opts['preserve_count'] = int(value)
elif option in ('-r', '--remote-host'):
context_opts['ssh_alias'] = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
raise Exception("Unhandled option!")
# Any positional arguments are passed to apt-get.
manager_opts['apt_options'] = arguments
except Exception as e:
warning("Failed to parse command line arguments! (%s)", e)
sys.exit(1)
# Execute the requested action(s).
context = create_context(**context_opts)
try:
manager = KernelPackageManager(context=context, **manager_opts)
getattr(manager, action)()
except (CleanupError, ExternalCommandFailed) as e:
logger.error("%s", e)
sys.exit(1)
except Exception:
logger.exception("Encountered unexpected exception on %s!", context)
sys.exit(1)
def test_npm_fast_install_workaround(self):
"""Test that ``npm-fast-install`` installs ``devDependencies`` (due to the workaround in npm-accel)."""
with TemporaryDirectory() as cache_directory:
accelerator = NpmAccel(context=create_context(),
cache_directory=cache_directory,
installer_name='npm-fast-install',
production=False)
with TemporaryDirectory() as project_directory:
write_package_metadata(project_directory,
devDependencies=dict(npm='3.10.6'))
accelerator.install(project_directory)
self.check_program(project_directory, 'npm', 'help')
def main():
"""Command line interface for the ``update-dotdee`` program."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
context_opts = {}
program_opts = {}
try:
options, arguments = getopt.getopt(sys.argv[1:], 'fur:vqh', [
'force',
'use-sudo',
'remote-host=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-f', '--force'):
program_opts['force'] = True
elif option in ('-u', '--use-sudo'):
context_opts['sudo'] = True
elif option in ('-r', '--remote-host'):
context_opts['ssh_alias'] = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
sys.exit(0)
else:
# Programming error...
assert False, "Unhandled option!"
if not arguments:
usage(__doc__)
sys.exit(0)
if len(arguments) != 1:
raise Exception(
"Expected a filename as the first and only argument!")
program_opts['filename'] = arguments[0]
except Exception as e:
warning("Error: %s", e)
sys.exit(1)
# Run the program.
try:
# Initialize the execution context.
program_opts['context'] = create_context(**context_opts)
# Initialize the program and update the file.
UpdateDotDee(**program_opts).update_file()
except Exception as e:
logger.exception("Encountered unexpected exception, aborting!")
sys.exit(1)
def main():
"""Command line interface for ``debuntu-kernel-manager``."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
action = "render_summary"
context_opts = dict()
manager_opts = dict()
try:
options, arguments = getopt.getopt(
sys.argv[1:],
"cfp:r:vqh",
[
"clean", "remove", "force", "preserve-count=", "remote-host=",
"verbose", "quiet", "help"
],
)
for option, value in options:
if option in ("-c", "--clean", "--remove"):
action = "cleanup_packages"
elif option in ("-f", "--force"):
manager_opts["force"] = True
elif option in ("-p", "--preserve-count"):
manager_opts["preserve_count"] = int(value)
elif option in ("-r", "--remote-host"):
context_opts["ssh_alias"] = value
elif option in ("-v", "--verbose"):
coloredlogs.increase_verbosity()
elif option in ("-q", "--quiet"):
coloredlogs.decrease_verbosity()
elif option in ("-h", "--help"):
usage(__doc__)
return
else:
raise Exception("Unhandled option!")
# Any positional arguments are passed to apt-get.
manager_opts["apt_options"] = arguments
except Exception as e:
warning("Failed to parse command line arguments! (%s)", e)
sys.exit(1)
# Execute the requested action(s).
context = create_context(**context_opts)
try:
manager = KernelPackageManager(context=context, **manager_opts)
getattr(manager, action)()
except (CleanupError, ExternalCommandFailed) as e:
logger.error("%s", e)
sys.exit(1)
except Exception:
logger.exception("Encountered unexpected exception on %s!", context)
sys.exit(1)
def test_installer_validation(self):
"""Make sure the installer name is properly validated."""
accelerator = NpmAccel(context=create_context())
# Make sure the default installer is 'npm'.
assert accelerator.installer_name == 'npm'
assert accelerator.installer_method == accelerator.install_with_npm
# Make sure 'npm-cache' is supported.
accelerator.installer_name = 'npm-cache'
assert accelerator.installer_method == accelerator.install_with_npm_cache
# Make sure 'npm-fast-install' is supported.
accelerator.installer_name = 'npm-fast-install'
assert accelerator.installer_method == accelerator.install_with_npm_fast_install
# Make sure invalid installer names raise an error.
accelerator.installer_name = 'bogus'
self.assertRaises(ValueError, lambda: accelerator.installer_method)
def test_caching(self):
"""Verify that caching of ``node_modules`` brings a speed improvement."""
with TemporaryDirectory() as cache_directory:
with TemporaryDirectory() as project_directory:
original_dependencies = dict(npm='3.10.6')
write_package_metadata(project_directory,
original_dependencies)
accelerator = NpmAccel(
context=create_context(),
cache_directory=cache_directory,
)
# Sanity check that we're about to prime the cache.
parsed_dependencies = accelerator.extract_dependencies(
os.path.join(project_directory, 'package.json'))
assert parsed_dependencies == original_dependencies
# XXX In Python 2.x the following two expressions can both be
# True (due to implicit Unicode string coercion):
#
# 1. parsed_dependencies == original_dependencies
# 2. get_cache_file(parsed_dependencies) != get_cache_file(original_dependencies)
#
# That is to say: While you can successfully compare two
# dictionaries for equality, the repr() of the two dictionaries
# will differ, due to string keys versus Unicode keys and the
# u'' syntax in the repr() output.
file_in_cache = accelerator.get_cache_file(parsed_dependencies)
logger.debug(
"Name of file to be added to cache: %s (based on original dependencies: %s)",
file_in_cache, original_dependencies)
assert not os.path.isfile(file_in_cache)
# The first run is expected to prime the cache.
first_run = Timer(resumable=True)
with first_run:
parsed_dependencies = accelerator.install(
project_directory)
assert parsed_dependencies == original_dependencies
self.check_program(project_directory, 'npm', 'help')
# Sanity check that the cache was primed.
assert os.path.isfile(file_in_cache)
# The second run is expected to reuse the cache.
second_run = Timer(resumable=True)
with second_run:
parsed_dependencies = accelerator.install(
project_directory)
assert parsed_dependencies == original_dependencies
self.check_program(project_directory, 'npm', 'help')
# Make sure the 2nd run was significantly faster than the 1st run.
assert second_run.elapsed_time < (first_run.elapsed_time / 2)
def test_node_binary_not_found_error(self):
"""Make sure an error is raised when the Node.js interpreter is missing."""
saved_path = os.environ.get('PATH', None)
try:
# Temporarily override the search path to remove all /usr/*
# directories where a Node.js interpreter can reasonably be
# expected to be installed.
os.environ['PATH'] = '/sbin:/bin'
accelerator = NpmAccel(context=create_context())
self.assertRaises(MissingNodeInterpreterError, getattr,
accelerator, 'nodejs_interpreter')
finally:
if saved_path is not None:
os.environ['PATH'] = saved_path
else:
os.environ.pop('PATH')
def destination_context(self):
"""
The execution context of the system that stores the backup (the destination).
This is an execution context created by :mod:`executor.contexts`.
:raises: :exc:`.DestinationContextUnavailable` when the destination is
an rsync daemon module (which doesn't allow arbitrary command
execution).
"""
if self.destination.module:
raise DestinationContextUnavailable(
compact("""
Error: The execution context of the backup destination isn't
available because the destination ({dest}) is an rsync daemon
module! (tip: reconsider your command line options)
""",
dest=self.destination.expression))
else:
context_opts = dict(sudo=self.sudo_enabled)
if self.destination.hostname:
context_opts['ssh_alias'] = self.destination.hostname
context_opts['ssh_user'] = self.destination.username
return create_context(**context_opts)
def do_backup():
log = logging.getLogger()
program_opts = {
'backup_enabled': True,
'snapshot_enabled': True,
'rotate_enabled': True,
'sudo_enabled': False,
'dry_run': False,
'multi_fs': True,
'notifications_enabled': False,
'rsync_verbose_count': 1,
'rsync_show_progress': True,
'source_context': create_context(),
'source': BACKUP_SOURCE,
'destination': rsb.Destination(expression=BACKUP_DESTINATION)
}
try:
# Initialize the program with the command line
# options and execute the requested action(s).
b = rsb.RsyncSystemBackup(**program_opts).execute()
except Exception as e:
if isinstance(e, rsb.exceptions.RsyncSystemBackupError):
# Special handling when the backup disk isn't available.
if isinstance(e, rsb.exceptions.MissingBackupDiskError):
log.info("Skipping backup: %s", e)
return 1
# Known problems shouldn't produce
# an intimidating traceback to users.
log.error("Aborting due to error: %s", e)
else:
# Unhandled exceptions do get a traceback,
# because it may help fix programming errors.
log.exception("Aborting due to unhandled exception!")
return 1
else:
return 0
def test_cache_cleaning(self):
"""Make sure the automatic cache cleaning logic works as expected."""
with TemporaryDirectory() as cache_directory:
context = create_context()
accelerator = NpmAccel(context=context,
cache_directory=cache_directory)
just_above_limit = accelerator.cache_limit + 1
for i in range(just_above_limit):
# Create a fake (empty) tar archive.
fingerprint = random_string(length=40,
characters=string.hexdigits)
filename = os.path.join(cache_directory,
'%s.tar' % fingerprint)
context.write_file(filename, '')
# Create the cache metadata.
accelerator.write_metadata(filename)
# Sanity check the cache entries.
assert len(list(accelerator.find_archives())) == just_above_limit
# Run the cleanup.
accelerator.clean_cache()
# Make sure the number of cache entries decreased.
assert len(list(
accelerator.find_archives())) == accelerator.cache_limit
def main():
"""Command line interface for the ``rsync-system-backup`` program."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
context_opts = dict()
program_opts = dict()
dest_opts = dict()
try:
options, arguments = getopt.gnu_getopt(sys.argv[1:], 'bsrm:c:t:i:unx:fvqh', [
'backup', 'snapshot', 'rotate', 'mount=', 'crypto=', 'tunnel=',
'ionice=', 'no-sudo', 'dry-run', 'multi-fs', 'exclude=', 'force',
'disable-notifications', 'verbose', 'quiet', 'help',
])
for option, value in options:
if option in ('-b', '--backup'):
enable_explicit_action(program_opts, 'backup_enabled')
elif option in ('-s', '--snapshot'):
enable_explicit_action(program_opts, 'snapshot_enabled')
elif option in ('-r', '--rotate'):
enable_explicit_action(program_opts, 'rotate_enabled')
elif option in ('-m', '--mount'):
program_opts['mount_point'] = value
elif option in ('-c', '--crypto'):
program_opts['crypto_device'] = value
elif option in ('-t', '--tunnel'):
ssh_user, _, value = value.rpartition('@')
ssh_alias, _, port_number = value.partition(':')
tunnel_opts = dict(
ssh_alias=ssh_alias,
ssh_user=ssh_user,
# The port number of the rsync daemon.
remote_port=RSYNCD_PORT,
)
if port_number:
# The port number of the SSH server.
tunnel_opts['port'] = int(port_number)
dest_opts['ssh_tunnel'] = SecureTunnel(**tunnel_opts)
elif option in ('-i', '--ionice'):
value = value.lower().strip()
validate_ionice_class(value)
program_opts['ionice'] = value
elif option in ('-u', '--no-sudo'):
program_opts['sudo_enabled'] = False
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..", option)
program_opts['dry_run'] = True
elif option in ('-f', '--force'):
program_opts['force'] = True
elif option in ('-x', '--exclude'):
program_opts.setdefault('exclude_list', [])
program_opts['exclude_list'].append(value)
elif option == '--multi-fs':
program_opts['multi_fs'] = True
elif option == '--disable-notifications':
program_opts['notifications_enabled'] = False
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
raise Exception("Unhandled option! (programming error)")
if len(arguments) > 2:
msg = "Expected one or two positional arguments! (got %i)"
raise Exception(msg % len(arguments))
if len(arguments) == 2:
# Get the source from the first of two arguments.
program_opts['source'] = arguments.pop(0)
if arguments:
# Get the destination from the second (or only) argument.
dest_opts['expression'] = arguments[0]
program_opts['destination'] = Destination(**dest_opts)
elif not os.environ.get('RSYNC_MODULE_PATH'):
# Show a usage message when no destination is given.
usage(__doc__)
return
except Exception as e:
warning("Error: %s", e)
sys.exit(1)
try:
# Inject the source context into the program options.
program_opts['source_context'] = create_context(**context_opts)
# Initialize the program with the command line
# options and execute the requested action(s).
RsyncSystemBackup(**program_opts).execute()
except Exception as e:
if isinstance(e, RsyncSystemBackupError):
# Special handling when the backup disk isn't available.
if isinstance(e, MissingBackupDiskError):
# Check if we're connected to a terminal to decide whether the
# error should be propagated or silenced, the idea being that
# rsync-system-backup should keep quiet when it's being run
# from cron and the backup disk isn't available.
if not connected_to_terminal():
logger.info("Skipping backup: %s", e)
sys.exit(0)
# Known problems shouldn't produce
# an intimidating traceback to users.
logger.error("Aborting due to error: %s", e)
else:
# Unhandled exceptions do get a traceback,
# because it may help fix programming errors.
logger.exception("Aborting due to unhandled exception!")
sys.exit(1)
def test_missing_package_file_error(self):
"""Make sure an error is raised when the ``package.json`` file is missing."""
with TemporaryDirectory() as project_directory:
accelerator = NpmAccel(context=create_context())
self.assertRaises(MissingPackageFileError, accelerator.install,
project_directory)
def main():
"""Command line interface for the ``npm-accel`` program."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Command line option defaults.
program_opts = {}
context_opts = {}
directory = None
action = 'install'
# Parse the command line arguments.
try:
options, arguments = getopt.getopt(sys.argv[1:], 'pi:c:l:nbr:vqh', [
'production',
'installer=',
'cache-directory=',
'cache-limit=',
'no-cache',
'benchmark',
'remote-host=',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('p', '--production'):
program_opts['production'] = True
elif option in ('i', '--installer'):
program_opts['installer_name'] = value
elif option in ('-c', '--cache-directory'):
program_opts['cache_directory'] = parse_path(value)
elif option in ('-l', '--cache-limit'):
program_opts['cache_limit'] = int(value)
elif option in ('-n', '--no-cache'):
program_opts['write_to_cache'] = False
elif option in ('-b', '--benchmark'):
action = 'benchmark'
elif option in ('-r', '--remote-host'):
context_opts['ssh_alias'] = value
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
assert False, "Unhandled option!"
if arguments:
directory = arguments.pop(0)
if arguments:
raise Exception("Got more positional arguments than expected!")
if not directory:
if context_opts.get('ssh_alias'):
raise Exception(
"When operating on a remote system the directory needs to be specified explicitly!"
)
directory = os.getcwd()
except Exception as e:
warning("Error: Failed to parse command line arguments! (%s)" % e)
sys.exit(1)
# Perform the requested action(s).
try:
context = create_context(**context_opts)
program_opts['context'] = context
accelerator = NpmAccel(**program_opts)
method = getattr(accelerator, action)
method(directory)
except NpmAccelError as e:
warning("Error: %s", e)
sys.exit(1)
except Exception:
logger.exception("Encountered unexpected exception! Aborting ..")
sys.exit(1)
def main():
"""Command line interface for the ``rsync-system-backup`` program."""
# Initialize logging to the terminal and system log.
coloredlogs.install(syslog=True)
# Parse the command line arguments.
context_opts = dict()
program_opts = dict()
try:
options, arguments = getopt.getopt(sys.argv[1:], 'bsrm:c:i:unvqh', [
'backup',
'snapshot',
'rotate',
'mount=',
'crypto=',
'ionice=',
'no-sudo',
'dry-run',
'disable-notifications',
'verbose',
'quiet',
'help',
])
for option, value in options:
if option in ('-b', '--backup'):
enable_explicit_action(program_opts, 'backup_enabled')
elif option in ('-s', '--snapshot'):
enable_explicit_action(program_opts, 'snapshot_enabled')
elif option in ('-r', '--rotate'):
enable_explicit_action(program_opts, 'rotate_enabled')
elif option in ('-m', '--mount'):
program_opts['mount_point'] = value
elif option in ('-c', '--crypto'):
program_opts['crypto_device'] = value
elif option in ('-i', '--ionice'):
value = value.lower().strip()
validate_ionice_class(value)
program_opts['ionice'] = value
elif option in ('-u', '--no-sudo'):
program_opts['sudo_enabled'] = False
elif option in ('-n', '--dry-run'):
logger.info("Performing a dry run (because of %s option) ..",
option)
program_opts['dry_run'] = True
elif option == '--disable-notifications':
program_opts['notifications_enabled'] = False
elif option in ('-v', '--verbose'):
coloredlogs.increase_verbosity()
elif option in ('-q', '--quiet'):
coloredlogs.decrease_verbosity()
elif option in ('-h', '--help'):
usage(__doc__)
return
else:
raise Exception("Unhandled option! (programming error)")
if len(arguments) > 2:
msg = "Expected one or two positional arguments! (got %i)"
raise Exception(msg % len(arguments))
if len(arguments) == 2:
# Get the source from the first of two arguments.
program_opts['source'] = arguments.pop(0)
if arguments:
# Get the destination from the second (or only) argument.
program_opts['destination'] = arguments[0]
elif not os.environ.get('RSYNC_MODULE_PATH'):
# Show a usage message when no destination is given.
usage(__doc__)
return
except Exception as e:
warning("Error: %s", e)
sys.exit(1)
try:
# Inject the source context into the program options.
program_opts['source_context'] = create_context(**context_opts)
# Initialize the program with the command line
# options and execute the requested action(s).
RsyncSystemBackup(**program_opts).execute()
except Exception as e:
if isinstance(e, RsyncSystemBackupError):
# Known problems shouldn't produce
# an intimidating traceback to users.
logger.error("Aborting due to error: %s", e)
else:
# Unhandled exceptions do get a traceback,
# because it may help fix programming errors.
logger.exception("Aborting due to unhandled exception!")
sys.exit(1)
