def coerce_location(value, **options): """ Coerce a string to a :class:`Location` object. :param value: The value to coerce (a string or :class:`Location` object). :param options: Any keyword arguments are passed on to :func:`~executor.contexts.create_context()`. :returns: A :class:`Location` object. """ # Location objects pass through untouched. if not isinstance(value, Location): # Other values are expected to be strings. if not isinstance(value, string_types): msg = "Expected Location object or string, got %s instead!" raise ValueError(msg % type(value)) # Try to parse a remote location. ssh_alias, _, directory = value.partition(':') if ssh_alias and directory and '/' not in ssh_alias: options['ssh_alias'] = ssh_alias else: directory = value # Create the location object. value = Location( context=create_context(**options), directory=parse_path(directory), ) return value
def main(): """Command line interface for ``debuntu-nodejs-installer``.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) silence_urllib_logger() # Parse the command line arguments. action = None context_opts = dict() installer_opts = dict() try: options, arguments = getopt.getopt(sys.argv[1:], 'iV:s:r:vqh', [ 'install', 'version=', 'sources-file=', 'remote-host=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-i', '--install'): action = 'install' elif option in ('-V', '--version'): installer_opts['nodejs_version'] = value elif option in ('-s', '--sources-file'): installer_opts['sources_file'] = value elif option in ('-r', '--remote-host'): context_opts['ssh_alias'] = value elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) sys.exit(0) else: raise Exception("Unhandled option!") if arguments: raise Exception( "This program doesn't accept any positional arguments!") if not action: usage(__doc__) sys.exit(0) except Exception as e: warning("Failed to parse command line arguments! (%s)", e) sys.exit(1) # Execute the requested action. context = create_context(**context_opts) try: installer = NodeInstaller(context=context, **installer_opts) getattr(installer, action)() except (UnsupportedSystemError, ExternalCommandFailed) as e: logger.error("%s", e) sys.exit(1) except Exception: logger.exception("Encountered unexpected exception on %s!", context) sys.exit(1)
def main(): """Command line interface for ``debuntu-kernel-manager``.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) # Parse the command line arguments. action = 'render_summary' context_opts = dict() manager_opts = dict() try: options, arguments = getopt.getopt(sys.argv[1:], 'cfp:r:vqh', [ 'clean', 'remove', 'force', 'preserve-count=', 'remote-host=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-c', '--clean', '--remove'): action = 'cleanup_packages' elif option in ('-f', '--force'): manager_opts['force'] = True elif option in ('-p', '--preserve-count'): manager_opts['preserve_count'] = int(value) elif option in ('-r', '--remote-host'): context_opts['ssh_alias'] = value elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: raise Exception("Unhandled option!") # Any positional arguments are passed to apt-get. manager_opts['apt_options'] = arguments except Exception as e: warning("Failed to parse command line arguments! (%s)", e) sys.exit(1) # Execute the requested action(s). context = create_context(**context_opts) try: manager = KernelPackageManager(context=context, **manager_opts) getattr(manager, action)() except (CleanupError, ExternalCommandFailed) as e: logger.error("%s", e) sys.exit(1) except Exception: logger.exception("Encountered unexpected exception on %s!", context) sys.exit(1)
def test_npm_fast_install_workaround(self): """Test that ``npm-fast-install`` installs ``devDependencies`` (due to the workaround in npm-accel).""" with TemporaryDirectory() as cache_directory: accelerator = NpmAccel(context=create_context(), cache_directory=cache_directory, installer_name='npm-fast-install', production=False) with TemporaryDirectory() as project_directory: write_package_metadata(project_directory, devDependencies=dict(npm='3.10.6')) accelerator.install(project_directory) self.check_program(project_directory, 'npm', 'help')
def main(): """Command line interface for the ``update-dotdee`` program.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) # Parse the command line arguments. context_opts = {} program_opts = {} try: options, arguments = getopt.getopt(sys.argv[1:], 'fur:vqh', [ 'force', 'use-sudo', 'remote-host=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-f', '--force'): program_opts['force'] = True elif option in ('-u', '--use-sudo'): context_opts['sudo'] = True elif option in ('-r', '--remote-host'): context_opts['ssh_alias'] = value elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) sys.exit(0) else: # Programming error... assert False, "Unhandled option!" if not arguments: usage(__doc__) sys.exit(0) if len(arguments) != 1: raise Exception( "Expected a filename as the first and only argument!") program_opts['filename'] = arguments[0] except Exception as e: warning("Error: %s", e) sys.exit(1) # Run the program. try: # Initialize the execution context. program_opts['context'] = create_context(**context_opts) # Initialize the program and update the file. UpdateDotDee(**program_opts).update_file() except Exception as e: logger.exception("Encountered unexpected exception, aborting!") sys.exit(1)
def main(): """Command line interface for ``debuntu-kernel-manager``.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) # Parse the command line arguments. action = "render_summary" context_opts = dict() manager_opts = dict() try: options, arguments = getopt.getopt( sys.argv[1:], "cfp:r:vqh", [ "clean", "remove", "force", "preserve-count=", "remote-host=", "verbose", "quiet", "help" ], ) for option, value in options: if option in ("-c", "--clean", "--remove"): action = "cleanup_packages" elif option in ("-f", "--force"): manager_opts["force"] = True elif option in ("-p", "--preserve-count"): manager_opts["preserve_count"] = int(value) elif option in ("-r", "--remote-host"): context_opts["ssh_alias"] = value elif option in ("-v", "--verbose"): coloredlogs.increase_verbosity() elif option in ("-q", "--quiet"): coloredlogs.decrease_verbosity() elif option in ("-h", "--help"): usage(__doc__) return else: raise Exception("Unhandled option!") # Any positional arguments are passed to apt-get. manager_opts["apt_options"] = arguments except Exception as e: warning("Failed to parse command line arguments! (%s)", e) sys.exit(1) # Execute the requested action(s). context = create_context(**context_opts) try: manager = KernelPackageManager(context=context, **manager_opts) getattr(manager, action)() except (CleanupError, ExternalCommandFailed) as e: logger.error("%s", e) sys.exit(1) except Exception: logger.exception("Encountered unexpected exception on %s!", context) sys.exit(1)
def test_installer_validation(self): """Make sure the installer name is properly validated.""" accelerator = NpmAccel(context=create_context()) # Make sure the default installer is 'npm'. assert accelerator.installer_name == 'npm' assert accelerator.installer_method == accelerator.install_with_npm # Make sure 'npm-cache' is supported. accelerator.installer_name = 'npm-cache' assert accelerator.installer_method == accelerator.install_with_npm_cache # Make sure 'npm-fast-install' is supported. accelerator.installer_name = 'npm-fast-install' assert accelerator.installer_method == accelerator.install_with_npm_fast_install # Make sure invalid installer names raise an error. accelerator.installer_name = 'bogus' self.assertRaises(ValueError, lambda: accelerator.installer_method)
def test_caching(self): """Verify that caching of ``node_modules`` brings a speed improvement.""" with TemporaryDirectory() as cache_directory: with TemporaryDirectory() as project_directory: original_dependencies = dict(npm='3.10.6') write_package_metadata(project_directory, original_dependencies) accelerator = NpmAccel( context=create_context(), cache_directory=cache_directory, ) # Sanity check that we're about to prime the cache. parsed_dependencies = accelerator.extract_dependencies( os.path.join(project_directory, 'package.json')) assert parsed_dependencies == original_dependencies # XXX In Python 2.x the following two expressions can both be # True (due to implicit Unicode string coercion): # # 1. parsed_dependencies == original_dependencies # 2. get_cache_file(parsed_dependencies) != get_cache_file(original_dependencies) # # That is to say: While you can successfully compare two # dictionaries for equality, the repr() of the two dictionaries # will differ, due to string keys versus Unicode keys and the # u'' syntax in the repr() output. file_in_cache = accelerator.get_cache_file(parsed_dependencies) logger.debug( "Name of file to be added to cache: %s (based on original dependencies: %s)", file_in_cache, original_dependencies) assert not os.path.isfile(file_in_cache) # The first run is expected to prime the cache. first_run = Timer(resumable=True) with first_run: parsed_dependencies = accelerator.install( project_directory) assert parsed_dependencies == original_dependencies self.check_program(project_directory, 'npm', 'help') # Sanity check that the cache was primed. assert os.path.isfile(file_in_cache) # The second run is expected to reuse the cache. second_run = Timer(resumable=True) with second_run: parsed_dependencies = accelerator.install( project_directory) assert parsed_dependencies == original_dependencies self.check_program(project_directory, 'npm', 'help') # Make sure the 2nd run was significantly faster than the 1st run. assert second_run.elapsed_time < (first_run.elapsed_time / 2)
def test_node_binary_not_found_error(self): """Make sure an error is raised when the Node.js interpreter is missing.""" saved_path = os.environ.get('PATH', None) try: # Temporarily override the search path to remove all /usr/* # directories where a Node.js interpreter can reasonably be # expected to be installed. os.environ['PATH'] = '/sbin:/bin' accelerator = NpmAccel(context=create_context()) self.assertRaises(MissingNodeInterpreterError, getattr, accelerator, 'nodejs_interpreter') finally: if saved_path is not None: os.environ['PATH'] = saved_path else: os.environ.pop('PATH')
def destination_context(self): """ The execution context of the system that stores the backup (the destination). This is an execution context created by :mod:`executor.contexts`. :raises: :exc:`.DestinationContextUnavailable` when the destination is an rsync daemon module (which doesn't allow arbitrary command execution). """ if self.destination.module: raise DestinationContextUnavailable( compact(""" Error: The execution context of the backup destination isn't available because the destination ({dest}) is an rsync daemon module! (tip: reconsider your command line options) """, dest=self.destination.expression)) else: context_opts = dict(sudo=self.sudo_enabled) if self.destination.hostname: context_opts['ssh_alias'] = self.destination.hostname context_opts['ssh_user'] = self.destination.username return create_context(**context_opts)
def do_backup(): log = logging.getLogger() program_opts = { 'backup_enabled': True, 'snapshot_enabled': True, 'rotate_enabled': True, 'sudo_enabled': False, 'dry_run': False, 'multi_fs': True, 'notifications_enabled': False, 'rsync_verbose_count': 1, 'rsync_show_progress': True, 'source_context': create_context(), 'source': BACKUP_SOURCE, 'destination': rsb.Destination(expression=BACKUP_DESTINATION) } try: # Initialize the program with the command line # options and execute the requested action(s). b = rsb.RsyncSystemBackup(**program_opts).execute() except Exception as e: if isinstance(e, rsb.exceptions.RsyncSystemBackupError): # Special handling when the backup disk isn't available. if isinstance(e, rsb.exceptions.MissingBackupDiskError): log.info("Skipping backup: %s", e) return 1 # Known problems shouldn't produce # an intimidating traceback to users. log.error("Aborting due to error: %s", e) else: # Unhandled exceptions do get a traceback, # because it may help fix programming errors. log.exception("Aborting due to unhandled exception!") return 1 else: return 0
def test_cache_cleaning(self): """Make sure the automatic cache cleaning logic works as expected.""" with TemporaryDirectory() as cache_directory: context = create_context() accelerator = NpmAccel(context=context, cache_directory=cache_directory) just_above_limit = accelerator.cache_limit + 1 for i in range(just_above_limit): # Create a fake (empty) tar archive. fingerprint = random_string(length=40, characters=string.hexdigits) filename = os.path.join(cache_directory, '%s.tar' % fingerprint) context.write_file(filename, '') # Create the cache metadata. accelerator.write_metadata(filename) # Sanity check the cache entries. assert len(list(accelerator.find_archives())) == just_above_limit # Run the cleanup. accelerator.clean_cache() # Make sure the number of cache entries decreased. assert len(list( accelerator.find_archives())) == accelerator.cache_limit
def main(): """Command line interface for the ``rsync-system-backup`` program.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) # Parse the command line arguments. context_opts = dict() program_opts = dict() dest_opts = dict() try: options, arguments = getopt.gnu_getopt(sys.argv[1:], 'bsrm:c:t:i:unx:fvqh', [ 'backup', 'snapshot', 'rotate', 'mount=', 'crypto=', 'tunnel=', 'ionice=', 'no-sudo', 'dry-run', 'multi-fs', 'exclude=', 'force', 'disable-notifications', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-b', '--backup'): enable_explicit_action(program_opts, 'backup_enabled') elif option in ('-s', '--snapshot'): enable_explicit_action(program_opts, 'snapshot_enabled') elif option in ('-r', '--rotate'): enable_explicit_action(program_opts, 'rotate_enabled') elif option in ('-m', '--mount'): program_opts['mount_point'] = value elif option in ('-c', '--crypto'): program_opts['crypto_device'] = value elif option in ('-t', '--tunnel'): ssh_user, _, value = value.rpartition('@') ssh_alias, _, port_number = value.partition(':') tunnel_opts = dict( ssh_alias=ssh_alias, ssh_user=ssh_user, # The port number of the rsync daemon. remote_port=RSYNCD_PORT, ) if port_number: # The port number of the SSH server. tunnel_opts['port'] = int(port_number) dest_opts['ssh_tunnel'] = SecureTunnel(**tunnel_opts) elif option in ('-i', '--ionice'): value = value.lower().strip() validate_ionice_class(value) program_opts['ionice'] = value elif option in ('-u', '--no-sudo'): program_opts['sudo_enabled'] = False elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) program_opts['dry_run'] = True elif option in ('-f', '--force'): program_opts['force'] = True elif option in ('-x', '--exclude'): program_opts.setdefault('exclude_list', []) program_opts['exclude_list'].append(value) elif option == '--multi-fs': program_opts['multi_fs'] = True elif option == '--disable-notifications': program_opts['notifications_enabled'] = False elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: raise Exception("Unhandled option! (programming error)") if len(arguments) > 2: msg = "Expected one or two positional arguments! (got %i)" raise Exception(msg % len(arguments)) if len(arguments) == 2: # Get the source from the first of two arguments. program_opts['source'] = arguments.pop(0) if arguments: # Get the destination from the second (or only) argument. dest_opts['expression'] = arguments[0] program_opts['destination'] = Destination(**dest_opts) elif not os.environ.get('RSYNC_MODULE_PATH'): # Show a usage message when no destination is given. usage(__doc__) return except Exception as e: warning("Error: %s", e) sys.exit(1) try: # Inject the source context into the program options. program_opts['source_context'] = create_context(**context_opts) # Initialize the program with the command line # options and execute the requested action(s). RsyncSystemBackup(**program_opts).execute() except Exception as e: if isinstance(e, RsyncSystemBackupError): # Special handling when the backup disk isn't available. if isinstance(e, MissingBackupDiskError): # Check if we're connected to a terminal to decide whether the # error should be propagated or silenced, the idea being that # rsync-system-backup should keep quiet when it's being run # from cron and the backup disk isn't available. if not connected_to_terminal(): logger.info("Skipping backup: %s", e) sys.exit(0) # Known problems shouldn't produce # an intimidating traceback to users. logger.error("Aborting due to error: %s", e) else: # Unhandled exceptions do get a traceback, # because it may help fix programming errors. logger.exception("Aborting due to unhandled exception!") sys.exit(1)
def test_missing_package_file_error(self): """Make sure an error is raised when the ``package.json`` file is missing.""" with TemporaryDirectory() as project_directory: accelerator = NpmAccel(context=create_context()) self.assertRaises(MissingPackageFileError, accelerator.install, project_directory)
def main(): """Command line interface for the ``npm-accel`` program.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) # Command line option defaults. program_opts = {} context_opts = {} directory = None action = 'install' # Parse the command line arguments. try: options, arguments = getopt.getopt(sys.argv[1:], 'pi:c:l:nbr:vqh', [ 'production', 'installer=', 'cache-directory=', 'cache-limit=', 'no-cache', 'benchmark', 'remote-host=', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('p', '--production'): program_opts['production'] = True elif option in ('i', '--installer'): program_opts['installer_name'] = value elif option in ('-c', '--cache-directory'): program_opts['cache_directory'] = parse_path(value) elif option in ('-l', '--cache-limit'): program_opts['cache_limit'] = int(value) elif option in ('-n', '--no-cache'): program_opts['write_to_cache'] = False elif option in ('-b', '--benchmark'): action = 'benchmark' elif option in ('-r', '--remote-host'): context_opts['ssh_alias'] = value elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: assert False, "Unhandled option!" if arguments: directory = arguments.pop(0) if arguments: raise Exception("Got more positional arguments than expected!") if not directory: if context_opts.get('ssh_alias'): raise Exception( "When operating on a remote system the directory needs to be specified explicitly!" ) directory = os.getcwd() except Exception as e: warning("Error: Failed to parse command line arguments! (%s)" % e) sys.exit(1) # Perform the requested action(s). try: context = create_context(**context_opts) program_opts['context'] = context accelerator = NpmAccel(**program_opts) method = getattr(accelerator, action) method(directory) except NpmAccelError as e: warning("Error: %s", e) sys.exit(1) except Exception: logger.exception("Encountered unexpected exception! Aborting ..") sys.exit(1)
def main(): """Command line interface for the ``rsync-system-backup`` program.""" # Initialize logging to the terminal and system log. coloredlogs.install(syslog=True) # Parse the command line arguments. context_opts = dict() program_opts = dict() try: options, arguments = getopt.getopt(sys.argv[1:], 'bsrm:c:i:unvqh', [ 'backup', 'snapshot', 'rotate', 'mount=', 'crypto=', 'ionice=', 'no-sudo', 'dry-run', 'disable-notifications', 'verbose', 'quiet', 'help', ]) for option, value in options: if option in ('-b', '--backup'): enable_explicit_action(program_opts, 'backup_enabled') elif option in ('-s', '--snapshot'): enable_explicit_action(program_opts, 'snapshot_enabled') elif option in ('-r', '--rotate'): enable_explicit_action(program_opts, 'rotate_enabled') elif option in ('-m', '--mount'): program_opts['mount_point'] = value elif option in ('-c', '--crypto'): program_opts['crypto_device'] = value elif option in ('-i', '--ionice'): value = value.lower().strip() validate_ionice_class(value) program_opts['ionice'] = value elif option in ('-u', '--no-sudo'): program_opts['sudo_enabled'] = False elif option in ('-n', '--dry-run'): logger.info("Performing a dry run (because of %s option) ..", option) program_opts['dry_run'] = True elif option == '--disable-notifications': program_opts['notifications_enabled'] = False elif option in ('-v', '--verbose'): coloredlogs.increase_verbosity() elif option in ('-q', '--quiet'): coloredlogs.decrease_verbosity() elif option in ('-h', '--help'): usage(__doc__) return else: raise Exception("Unhandled option! (programming error)") if len(arguments) > 2: msg = "Expected one or two positional arguments! (got %i)" raise Exception(msg % len(arguments)) if len(arguments) == 2: # Get the source from the first of two arguments. program_opts['source'] = arguments.pop(0) if arguments: # Get the destination from the second (or only) argument. program_opts['destination'] = arguments[0] elif not os.environ.get('RSYNC_MODULE_PATH'): # Show a usage message when no destination is given. usage(__doc__) return except Exception as e: warning("Error: %s", e) sys.exit(1) try: # Inject the source context into the program options. program_opts['source_context'] = create_context(**context_opts) # Initialize the program with the command line # options and execute the requested action(s). RsyncSystemBackup(**program_opts).execute() except Exception as e: if isinstance(e, RsyncSystemBackupError): # Known problems shouldn't produce # an intimidating traceback to users. logger.error("Aborting due to error: %s", e) else: # Unhandled exceptions do get a traceback, # because it may help fix programming errors. logger.exception("Aborting due to unhandled exception!") sys.exit(1)