def show_state_host_arguments_warning(call_location): logger.warning( ( "{0}:\n\tLegacy deploy function detected! Deploys should no longer define " "`state` and `host` arguments." ).format(call_location), )
def limit(self, hosts): logger.warning(( 'Use of `State.limit` is deprecated, ' 'please use normal `if` statements instead.' )) return self.hosts(hosts)
def decorated_func(*args, **kwargs): for legacy_key, key in LEGACY_ARG_MAP.items(): if legacy_key in kwargs: kwargs[key] = kwargs.pop(legacy_key) logger.warning((f"The `{legacy_key}` has been replaced " f"by `{key}` in `postgresql.*` operations."), ) return func(*args, **kwargs)
def _warn_invalid_auth_args(args, requires_key, invalid_keys): for key in invalid_keys: value = args.get(key) if value: logger.warning(( 'Invalid auth argument: cannot use `{0}={1}` without `{2}`' ).format(key, value, requires_key))
def hosts(self, hosts): logger.warning(( 'Use of `State.hosts` is deprecated, ' 'please use normal `if` statements instead.' )) hosts = ensure_host_list(hosts, inventory=self.inventory) # Store the previous value old_limit_hosts = self.limit_hosts # Limit the new hosts to a subset of the old hosts if they existed if old_limit_hosts is not None: hosts = [ host for host in hosts if host in old_limit_hosts ] # Set the new value self.limit_hosts = hosts logger.debug('Setting limit to hosts: {0}'.format(hosts)) yield # Restore the old value self.limit_hosts = old_limit_hosts logger.debug('Reset limit to hosts: {0}'.format(old_limit_hosts))
def show(self): name = 'unknown error' if isinstance(self, HookError): name = 'hook error' elif isinstance(self, PyinfraError): name = 'pyinfra error' elif isinstance(self, IOError): name = 'local IO error' if pseudo_host.isset(): # Get any operation meta + name op_name = None current_op_hash = pseudo_state.current_op_hash current_op_meta = pseudo_state.op_meta.get(current_op_hash) if current_op_meta: op_name = ', '.join(current_op_meta['names']) sys.stderr.write('--> {0}{1}{2}: '.format( pseudo_host.print_prefix, click.style(name, 'red', bold=True), ' (operation={0})'.format(op_name) if op_name else '', )) else: sys.stderr.write( '--> {0}: '.format(click.style(name, 'red', bold=True)), ) logger.warning(self) print()
def process(self, output): m = VERSION_MATCHER.match(output[0]) if m is not None: return [int(m.group(key)) for key in ["major", "minor", "patch"]] else: logger.warning("could not parse version string from brew: %s", output[0]) return self.default()
def windows_file(*args, **kwargs): # COMPAT # TODO: remove this logger.warning(( 'Use of `windows_files.windows_file` is deprecated, ' 'please use `windows_files.file` instead.' )) return file(*args, **kwargs)
def wrapper(*args, **kwargs): logger.warning(( 'The `init.{0}` operation is deprecated, ' 'please us the `{1}.{2}` operation.' ).format( legacy_op, op_func.__module__.replace('pyinfra.operations.', ''), op_func.__name__, )) return op_func(*args, **kwargs)
def pop_global_op_kwargs(state, kwargs): ''' Pop and return operation global keyword arguments. ''' for deprecated_key in ('when', 'hosts'): if deprecated_key in kwargs: logger.warning(( 'Use of the `{0}` argument is deprecated, ' 'please use normal `if` statements instead.' ).format(deprecated_key)) meta_kwargs = state.deploy_kwargs or {} def get_kwarg(key, default=None): return kwargs.pop(key, meta_kwargs.get(key, default)) # TODO: remove hosts/when hosts = get_kwarg('hosts') hosts = ensure_host_list(hosts, inventory=state.inventory) # Filter out any hosts not in the meta kwargs (nested support) if meta_kwargs.get('hosts') is not None: hosts = [ host for host in hosts if host in meta_kwargs['hosts'] ] global_kwargs = { 'hosts': hosts, 'when': get_kwarg('when', True), } # TODO: end remove hosts/when block if 'stdin' in kwargs: show_stdin_global_warning() for _, kwarg_configs in OPERATION_KWARGS.items(): for key, config in kwarg_configs.items(): handler = None default = None if isinstance(config, dict): handler = config.get('handler') default = config.get('default') if default and callable(default): default = default(state.config) value = get_kwarg(key, default=default) if handler: value = handler(state.config, value) global_kwargs[key] = value return global_kwargs
def get_facts_and_args(commands): facts = [] current_fact = None for command in commands: if '=' in command: if not current_fact: raise CliError('Invalid fact commands: {0}'.format(commands)) key, value = command.split('=', 1) current_fact[2][key] = value continue if current_fact: facts.append(current_fact) current_fact = None if '.' not in command: args = None if ':' in command: command, args = command.split(':', 1) args = args.split(',') if not is_fact(command): raise CliError('No fact: {0}'.format(command)) fact_cls = get_fact_class(command) logger.warning(( 'Named facts are deprecated, please use the explicit import: {0}.{1}' ).format(fact_cls.__module__.replace('pyinfra.facts.', ''), fact_cls.__name__)) current_fact = (fact_cls, args, {}) else: fact_module, fact_name = command.rsplit('.', 1) try: fact_module = import_module( 'pyinfra.facts.{0}'.format(fact_module)) except ImportError: try: fact_module = import_module(str(fact_module)) except ImportError: raise CliError('No such module: {0}'.format(fact_module)) fact_cls = getattr(fact_module, fact_name, None) if not fact_cls: raise CliError('No such fact: {0}'.format(command)) current_fact = (fact_cls, (), {}) if current_fact: facts.append(current_fact) return facts
def when(self, predicate): logger.warning(('Use of `State.when` is deprecated, ' 'please use normal `if` statements instead.')) # Truth-y? Just yield/end, nothing happens here! if predicate: yield return # Otherwise limit any operations within to match no hosts with self.hosts([]): yield
def wrapper(*args, **kwargs): frameinfo = get_caller_frameinfo() kwargs['_line_number'] = frameinfo.lineno logger.warning(('The `init.{0}` operation is deprecated, ' 'please us the `{1}.{2}` operation.').format( legacy_op, op_func.__module__.replace('pyinfra.operations.', ''), op_func.__name__, )) return op_func(*args, **kwargs)
def _put_file(host, filename_or_io, remote_location): attempts = 1 try: with get_file_io(filename_or_io) as file_io: sftp = _get_sftp_connection(host) sftp.putfo(file_io, remote_location) except OSError as e: if attempts > 3: raise logger.warning(f"Failed to upload file, retrying: {e}") attempts += 1
def get_host_keys(filename): with HOST_KEYS_LOCK: host_keys = HostKeys() try: host_keys.load(filename) # When paramiko encounters a bad host keys line it sometimes bails the # entire load incorrectly. # See: https://github.com/paramiko/paramiko/pull/1990 except Exception as e: logger.warning("Failed to load host keys from {0}: {1}".format(filename, e)) return host_keys
def reboot(state, host, delay=10, interval=1, reboot_timeout=300): ''' Reboot the server and wait for reconnection. + delay: number of seconds to wait before attempting reconnect + interval: interval (s) between reconnect attempts + reboot_timeout: total time before giving up reconnecting Note: Probably want sudo enabled. Example: .. code:: python server.reboot( {'Reboot the server and wait to reconnect'}, delay=5, timeout=30, ) ''' logger.warning('The server.reboot operation is in beta!') yield { 'command': 'reboot', 'success_exit_codes': [-1], # -1 being error/disconnected } def wait_and_reconnect(state, host): # pragma: no cover sleep(delay) max_retries = round(reboot_timeout / interval) host.connection = None # remove the connection object retries = 0 while True: host.connect(state, show_errors=False) if host.connection: break if retries > max_retries: raise Exception(( 'Server did not reboot in time (reboot_timeout={0}s)' ).format(reboot_timeout)) sleep(interval) retries += 1 yield (wait_and_reconnect, (), {})
def __init__(self, **kwargs): # Always apply some env env = kwargs.pop('ENV', {}) self.ENV = env # Replace TIMEOUT -> CONNECT_TIMEOUT if 'TIMEOUT' in kwargs: logger.warning(('Config.TIMEOUT is deprecated, ' 'please use Config.CONNECT_TIMEOUT instead')) kwargs['CONNECT_TIMEOUT'] = kwargs.pop('TIMEOUT') # Apply kwargs for key, value in six.iteritems(kwargs): setattr(self, key, value)
def __getitem__(self, key): ''' DEPRECATED: please use ``Inventory.get_host`` instead. ''' # COMPAT w/ <0.4 # TODO: remove this function logger.warning(('Use of Inventory[<host_name>] is deprecated, ' 'please use `Inventory.get_host` instead.')) if key in self.hosts: return self.hosts[key] raise NoHostError('No such host: {0}'.format(key))
def __getattr__(self, key): ''' DEPRECATED: please use ``Inventory.get_group`` instead. ''' # COMPAT w/ <0.4 # TODO: remove this function logger.warning(('Use of Inventory.<group_name> is deprecated, ' 'please use `Inventory.get_group` instead.')) if key in self.groups: return self.groups[key] raise NoGroupError('No such group: {0}'.format(key))
def missing_host_key(self, client, hostname, key): logger.warning( ( f"No host key for {hostname} found in known_hosts, " "accepting & adding to host keys file" ), ) with HOST_KEYS_LOCK: host_keys = client.get_host_keys() host_keys.add(hostname, key.get_name(), key) # The paramiko client saves host keys incorrectly whereas the host keys object does # this correctly, so use that with the client filename variable. # See: https://github.com/paramiko/paramiko/pull/1989 host_keys.save(client._host_keys_filename)
def _exception(name, e, always_dump=False): print() if pseudo_host.isset(): sys.stderr.write('--> [{0}]: {1}: '.format( colored(pseudo_host.name, attrs=['bold']), colored(name, 'red', attrs=['bold']) )) else: sys.stderr.write('--> {0}: '.format(colored(name, 'red', attrs=['bold']))) if e: logger.warning(e) if arguments.get('debug') or always_dump: dump_trace(sys.exc_info()) _exit(1)
def execute(state, host, callback, *args, **kwargs): ''' [DEPRECATED], please use ``python.call``. ''' # COMPAT w/ <0.4 # TODO: remove this function logger.warning(('Use of `python.execute` is deprecated, ' 'please use `python.call` instead.')) # Pre pyinfra 0.4 the operation execution passed (state, host, host.name) # as args, so here we replicate that - hence ``python.call`` which replaces # this function going forward. args = (host.name, ) + args yield (callback, args, kwargs)
def call(function, *args, **kwargs): """ Execute a Python function within a deploy. + function: the function to execute + args: additional arguments to pass to the function + kwargs: additional keyword arguments to pass to the function Callback functions args passed the state, host, and any args/kwargs passed into the operation directly, eg: .. code:: python def my_callback(state, host, hello=None): command = 'echo hello' if hello: command = command + ' ' + hello status, stdout, stderr = host.run_shell_command(command=command, sudo=SUDO) assert status is True # ensure the command executed OK if 'hello ' not in '\\n'.join(stdout): # stdout/stderr is a *list* of lines raise Exception( f'`{command}` problem with callback stdout:{stdout} stderr:{stderr}', ) python.call( name="Run my_callback function", function=my_callback, hello="world", ) """ argspec = getfullargspec(function) if "state" in argspec.args and "host" in argspec.args: logger.warning( "Callback functions used in `python.call` operations no " f"longer take `state` and `host` arguments: {get_call_location(frame_offset=3)}", ) kwargs.pop("state", None) kwargs.pop("host", None) yield FunctionCommand(function, args, kwargs)
def run(cmd, *args, **kwargs): result = FakeInvokeResult() try: cmd = [environ["SHELL"], cmd] try: code = check_call(cmd) except CalledProcessError as e: code = e.returncode result.ok = code == 0 except Exception as e: logger.warning(( "pyinfra encountered an error loading SSH config match exec {0}: {1}" ).format( cmd, e, ), ) return result
def missing_host_key(self, client, hostname, key): should_continue = input( "No host key for {0} found in known_hosts, do you want to continue [y/n] ".format( hostname, ), ) if should_continue.lower() != "y": raise SSHException( "AskPolicy: No host key for {0} found in known_hosts".format(hostname), ) else: with HOST_KEYS_LOCK: host_keys = client.get_host_keys() host_keys.add(hostname, key.get_name(), key) # The paramiko client saves host keys incorrectly whereas the host keys object does # this correctly, so use that with the client filename variable. # See: https://github.com/paramiko/paramiko/pull/1989 host_keys.save(client._host_keys_filename) logger.warning("Added host key for {0} to known_hosts".format(hostname)) return
def add_limited_op(state, op_func, hosts, *args, **kwargs): ''' DEPRECATED: please use ``add_op`` with the ``hosts`` kwarg. ''' # COMPAT w/ <0.4 # TODO: remove this function logger.warning(('Use of `add_limited_op` is deprecated, ' 'please use `add_op` with the `hosts` kwarg instead.')) if not isinstance(hosts, (list, tuple)): hosts = [hosts] # Set the limit state.limit_hosts = hosts # Add the op add_op(state, op_func, *args, **kwargs) # Remove the limit state.limit_hosts = []
def show(self): name = 'unknown error' if isinstance(self, HookError): name = 'hook error' elif isinstance(self, PyinfraError): name = 'pyinfra error' elif isinstance(self, IOError): name = 'local IO error' if pseudo_host.isset(): sys.stderr.write('--> [{0}]: {1}: '.format( click.style(six.text_type(pseudo_host.name), bold=True), click.style(name, 'red', bold=True), )) else: sys.stderr.write( '--> {0}: '.format(click.style(name, 'red', bold=True)), ) logger.warning(self) print()
def show_state_host_arguments_warning(call_location): logger.warning(( '{0}\n\tPassing `state` and `host` as the first two arguments to deploys is ' 'deprecated, please use `state` and `host` keyword arguments.' ).format(call_location))
def _main( inventory, operations, verbosity, user, port, key, key_password, password, winrm_username, winrm_password, winrm_port, shell_executable, sudo, sudo_user, use_sudo_password, su_user, parallel, fail_percent, dry, limit, no_wait, serial, quiet, debug, debug_data, debug_facts, debug_operations, facts=None, print_operations=None, support=None, ): if not debug and not sys.warnoptions: warnings.simplefilter('ignore') # Setup logging log_level = logging.INFO if debug: log_level = logging.DEBUG elif quiet: log_level = logging.WARNING setup_logging(log_level) # Bootstrap any virtualenv init_virtualenv() deploy_dir = getcwd() potential_deploy_dirs = [] # This is the most common case: we have a deploy file so use it's # pathname - we only look at the first file as we can't have multiple # deploy directories. if operations[0].endswith('.py'): deploy_file_dir, _ = path.split(operations[0]) above_deploy_file_dir, _ = path.split(deploy_file_dir) deploy_dir = deploy_file_dir potential_deploy_dirs.extend(( deploy_file_dir, above_deploy_file_dir, )) # If we have a valid inventory, look in it's path and it's parent for # group_data or config.py to indicate deploy_dir (--fact, --run). if inventory.endswith('.py') and path.isfile(inventory): inventory_dir, _ = path.split(inventory) above_inventory_dir, _ = path.split(inventory_dir) potential_deploy_dirs.extend(( inventory_dir, above_inventory_dir, )) for potential_deploy_dir in potential_deploy_dirs: logger.debug('Checking potential directory: {0}'.format( potential_deploy_dir, )) if any(( path.isdir(path.join(potential_deploy_dir, 'group_data')), path.isfile(path.join(potential_deploy_dir, 'config.py')), )): logger.debug( 'Setting directory to: {0}'.format(potential_deploy_dir)) deploy_dir = potential_deploy_dir break # Create an empty/unitialised state object state = State() # Set the deploy directory state.deploy_dir = deploy_dir pseudo_state.set(state) if verbosity > 0: state.print_fact_info = True state.print_noop_info = True if verbosity > 1: state.print_input = state.print_fact_input = True if verbosity > 2: state.print_output = state.print_fact_output = True if not quiet: click.echo('--> Loading config...', err=True) # Load up any config.py from the filesystem config = load_config(deploy_dir) # Make a copy before we overwrite original_operations = operations # Debug (print) inventory + group data if operations[0] == 'debug-inventory': command = 'debug-inventory' # Get all non-arg facts elif operations[0] == 'all-facts': command = 'fact' fact_names = [] for fact_name in get_fact_names(): fact_class = get_fact_class(fact_name) if (not issubclass(fact_class, ShortFactBase) and not callable(fact_class.command)): fact_names.append(fact_name) operations = [(name, None) for name in fact_names] # Get one or more facts elif operations[0] == 'fact': command = 'fact' fact_names = operations[1:] facts = [] for name in fact_names: args = None if ':' in name: name, args = name.split(':', 1) args = args.split(',') if not is_fact(name): raise CliError('No fact: {0}'.format(name)) facts.append((name, args)) operations = facts # Execute a raw command with server.shell elif operations[0] == 'exec': command = 'exec' operations = operations[1:] # Execute one or more deploy files elif all(cmd.endswith('.py') for cmd in operations): command = 'deploy' operations = operations[0:] for file in operations: if not path.exists(file): raise CliError('No deploy file: {0}'.format(file)) # Operation w/optional args (<module>.<op> ARG1 ARG2 ...) elif len(operations[0].split('.')) == 2: command = 'op' operations = get_operation_and_args(operations) else: raise CliError('''Invalid operations: {0} Operation usage: pyinfra INVENTORY deploy_web.py [deploy_db.py]... pyinfra INVENTORY server.user pyinfra home=/home/pyinfra pyinfra INVENTORY exec -- echo "hello world" pyinfra INVENTORY fact os [users]...'''.format(operations)) # Load any hooks/config from the deploy file if command == 'deploy': load_deploy_config(operations[0], config) # Arg based config overrides if sudo: config.SUDO = True if sudo_user: config.SUDO_USER = sudo_user if use_sudo_password: config.USE_SUDO_PASSWORD = use_sudo_password if su_user: config.SU_USER = su_user if parallel: config.PARALLEL = parallel if shell_executable: config.SHELL = shell_executable if fail_percent is not None: config.FAIL_PERCENT = fail_percent if not quiet: click.echo('--> Loading inventory...', err=True) # Load up the inventory from the filesystem inventory, inventory_group = make_inventory( inventory, deploy_dir=deploy_dir, ssh_port=port, ssh_user=user, ssh_key=key, ssh_key_password=key_password, ssh_password=password, winrm_username=winrm_username, winrm_password=winrm_password, winrm_port=winrm_port, ) # Attach to pseudo inventory pseudo_inventory.set(inventory) # Now that we have inventory, apply --limit config override initial_limit = None if limit: all_limit_hosts = [] for limiter in limit: try: limit_hosts = inventory.get_group(limiter) except NoGroupError: limits = limiter.split(',') if len(limits) > 1: logger.warning(( 'Specifying comma separated --limit values is deprecated, ' 'please use multiple --limit options.')) limit_hosts = [ host for host in inventory if any( fnmatch(host.name, match) for match in limits) ] all_limit_hosts.extend(limit_hosts) initial_limit = list(set(all_limit_hosts)) # Initialise the state state.init(inventory, config, initial_limit=initial_limit) # If --debug-data dump & exit if command == 'debug-inventory' or debug_data: if debug_data: logger.warning( ('--debug-data is deprecated, ' 'please use `pyinfra INVENTORY debug-inventory` instead.')) print_inventory(state) _exit() # Connect to all the servers if not quiet: click.echo(err=True) click.echo('--> Connecting to hosts...', err=True) connect_all(state) # Just getting a fact? # if command == 'fact': if not quiet: click.echo(err=True) click.echo('--> Gathering facts...', err=True) state.print_fact_info = True fact_data = {} for i, command in enumerate(operations): name, args = command fact_key = name if args: fact_key = '{0}{1}'.format(name, tuple(args)) try: fact_data[fact_key] = get_facts( state, name, args=args, apply_failed_hosts=False, ) except PyinfraError: pass print_facts(fact_data) _exit() # Prepare the deploy! # # Execute a raw command with server.shell if command == 'exec': # Print the output of the command state.print_output = True add_op( state, server.shell, ' '.join(operations), _allow_cli_mode=True, ) # Deploy files(s) elif command == 'deploy': if not quiet: click.echo(err=True) click.echo('--> Preparing operations...', err=True) # Number of "steps" to make = number of files * number of hosts for i, filename in enumerate(operations): logger.info('Loading: {0}'.format(click.style(filename, bold=True))) state.current_op_file = i load_deploy_file(state, filename) # Operation w/optional args elif command == 'op': if not quiet: click.echo(err=True) click.echo('--> Preparing operation...', err=True) op, args = operations args, kwargs = args kwargs['_allow_cli_mode'] = True def print_host_ready(host): logger.info('{0}{1} {2}'.format( host.print_prefix, click.style('Ready:', 'green'), click.style(original_operations[0], bold=True), )) kwargs['_after_host_callback'] = print_host_ready add_op(state, op, *args, **kwargs) # Always show meta output if not quiet: click.echo(err=True) click.echo('--> Proposed changes:', err=True) print_meta(state) # If --debug-facts or --debug-operations, print and exit if debug_facts or debug_operations: if debug_facts: print_state_facts(state) if debug_operations: print_state_operations(state) _exit() # Run the operations we generated with the deploy file if dry: _exit() if not quiet: click.echo(err=True) if not quiet: click.echo('--> Beginning operation run...', err=True) run_ops(state, serial=serial, no_wait=no_wait) if not quiet: click.echo('--> Results:', err=True) print_results(state) _exit()
def repo( state, host, source, target, branch='master', pull=True, rebase=False, user=None, group=None, use_ssh_user=False, ssh_keyscan=False, ): ''' Manage git repositories. + source: the git source URL + target: target directory to clone to + branch: branch to pull/checkout + pull: pull any changes for the branch + rebase: when pulling, use ``--rebase`` + user: chown files to this user after + group: chown files to this group after + ssh_keyscan: keyscan the remote host if not in known_hosts before clone/pull + [DEPRECATED] use_ssh_user: whether to use the SSH user to clone/pull SSH user (deprecated, please use ``preserve_sudo_env``): This is an old hack from pyinfra <0.4 which did not support the global kwarg ``preserve_sudo_env``. It does the following: * makes the target directory writeable by all * clones/pulls w/o sudo as the connecting SSH user * removes other/group write permissions - unless group is defined, in which case only other ''' if use_ssh_user: logger.warning( 'Use of `use_ssh_user` is deprecated, please use `preserve_sudo_env` instead.', ) # Ensure our target directory exists yield files.directory(state, host, target) # If we're going to chown this after clone/pull, and we're sudo'd, we need to make the # directory writeable by the SSH user if use_ssh_user: yield chmod(target, 'go+w', recursive=True) # Do we need to scan for the remote host key? if ssh_keyscan: # Attempt to parse the domain from the git repository domain = re.match(r'^[a-zA-Z0-9]+@([0-9a-zA-Z\.\-]+)', source) if domain: yield ssh.keyscan(state, host, domain.group(1)) # Store git commands for directory prefix git_commands = [] is_repo = host.fact.directory('/'.join((target, '.git'))) # Cloning new repo? if not is_repo: git_commands.append('clone {0} --branch {1} .'.format(source, branch)) # Ensuring existing repo else: current_branch = host.fact.git_branch(target) if current_branch != branch: git_commands.append('checkout {0}'.format(branch)) if pull: if rebase: git_commands.append('pull --rebase') else: git_commands.append('pull') # Attach prefixes for directory command_prefix = 'cd {0} && git'.format(target) git_commands = [ '{0} {1}'.format(command_prefix, command) for command in git_commands ] if use_ssh_user: git_commands = [{ 'command': command, 'sudo': False, 'sudo_user': False, } for command in git_commands] for cmd in git_commands: yield cmd if use_ssh_user: # Remove write permissions from other or other+group when no group yield chmod( target, 'o-w' if group else 'go-w', recursive=True, ) # Apply any user or group if user or group: yield chown(target, user, group, recursive=True)