def connect(self, reason=None, show_errors=True): self._check_state() if not self.connection: self.state.trigger_callbacks('host_before_connect', self) try: self.connection = self.executor.connect(self.state, self) except ConnectError as e: if show_errors: log_message = '{0}{1}'.format( self.print_prefix, click.style(e.args[0], 'red'), ) logger.error(log_message) self.state.trigger_callbacks('host_connect_error', self, e) else: log_message = '{0}{1}'.format( self.print_prefix, click.style('Connected', 'green'), ) if reason: log_message = '{0}{1}'.format( log_message, ' ({0})'.format(reason), ) logger.info(log_message) self.state.trigger_callbacks('host_connect', self) return self.connection
def connect(self, reason=None, show_errors=True, raise_exceptions=False): self._check_state() if not self.connection: self.state.trigger_callbacks("host_before_connect", self) try: self.connection = self.executor.connect(self.state, self) except ConnectError as e: if show_errors: log_message = "{0}{1}".format( self.print_prefix, click.style(e.args[0], "red"), ) logger.error(log_message) self.state.trigger_callbacks("host_connect_error", self, e) if raise_exceptions: raise else: log_message = "{0}{1}".format( self.print_prefix, click.style("Connected", "green"), ) if reason: log_message = "{0}{1}".format( log_message, " ({0})".format(reason), ) logger.info(log_message) self.state.trigger_callbacks("host_connect", self) return self.connection
def put_file( state, host, filename_or_io, remote_file, sudo=False, sudo_user=None, su_user=None, print_output=False, ): ''' Upload file-ios to the specified host using SFTP. Supports uploading files with sudo by uploading to a temporary directory then moving & chowning. ''' # sudo/su are a little more complicated, as you can only sftp with the SSH # user connected, so upload to tmp and copy/chown w/sudo and/or su_user if sudo or su_user: # Get temp file location temp_file = state.get_temp_filename(remote_file) _put_file(host, filename_or_io, temp_file) if print_output: print('{0}file uploaded: {1}'.format(host.print_prefix, remote_file)) # Execute run_shell_command w/sudo and/or su_user command = 'mv {0} {1}'.format(temp_file, remote_file) # Move it to the su_user if present if su_user: command = '{0} && chown {1} {2}'.format(command, su_user, remote_file) # Otherwise any sudo_user elif sudo_user: command = '{0} && chown {1} {2}'.format(command, sudo_user, remote_file) status, _, stderr = run_shell_command( state, host, command, sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=print_output, ) if status is False: logger.error('File error: {0}'.format('\n'.join(stderr))) return False # No sudo and no su_user, so just upload it! else: _put_file(host, filename_or_io, remote_file) if print_output: print('{0}file uploaded: {1}'.format(host.print_prefix, remote_file))
def connect(self, for_fact=None, show_errors=True): self._check_state() if not self.connection: try: self.connection = self.executor.connect(self.state, self) except ConnectError as e: if show_errors: log_message = '{0}{1}'.format( self.print_prefix, click.style(e.args[0], 'red'), ) logger.error(log_message) else: log_message = '{0}{1}'.format( self.print_prefix, click.style('Connected', 'green'), ) if for_fact: log_message = '{0}{1}'.format( log_message, ' (for {0} fact)'.format(for_fact), ) logger.info(log_message) return self.connection
def get_file( state, host, remote_filename, filename_or_io, sudo=False, sudo_user=None, su_user=None, print_output=False, print_input=False, **command_kwargs ): ''' Download a file from the remote host using SFTP. Supports download files with sudo by copying to a temporary directory with read permissions, downloading and then removing the copy. ''' if sudo or su_user: # Get temp file location temp_file = state.get_temp_filename(remote_filename) # Copy the file to the tempfile location and add read permissions command = 'cp {0} {1} && chmod +r {0}'.format(remote_filename, temp_file) copy_status, _, stderr = run_shell_command( state, host, command, sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=print_output, print_input=print_input, **command_kwargs ) if copy_status is False: logger.error('File download copy temp error: {0}'.format('\n'.join(stderr))) return False try: _get_file(host, temp_file, filename_or_io) # Ensure that, even if we encounter an error, we (attempt to) remove the # temporary copy of the file. finally: remove_status, _, stderr = run_shell_command( state, host, 'rm -f {0}'.format(temp_file), sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=print_output, print_input=print_input, **command_kwargs ) if remove_status is False: logger.error('File download remove temp error: {0}'.format('\n'.join(stderr))) return False else: _get_file(host, remote_filename, filename_or_io) if print_output: click.echo( '{0}file downloaded: {1}'.format(host.print_prefix, remote_filename), err=True, ) return True
def print_host_combined_output(host, combined_output_lines): for type_, line in combined_output_lines: if type_ == 'stderr': logger.error('{0}{1}'.format( host.print_prefix, click.style(line, 'red'), )) else: logger.error('{0}{1}'.format( host.print_prefix, line, ))
def log_host_command_error(host, e, timeout=0): if isinstance(e, timeout_error): logger.error('{0}{1}'.format( host.print_prefix, click.style('Command timed out after {0}s'.format(timeout, ), 'red'), )) elif isinstance(e, (socket_error, SSHException)): logger.error('{0}{1}'.format( host.print_prefix, click.style( 'Command socket/SSH error: {0}'.format(format_exception(e)), 'red', ), ))
def print_host_combined_output(host, combined_output_lines): for type_, line in combined_output_lines: if type_ == "stderr": logger.error( "{0}{1}".format( host.print_prefix, click.style(line, "red"), ), ) else: logger.error( "{0}{1}".format( host.print_prefix, line, ), )
def put_file( state, host, filename_or_io, remote_filename, print_output=False, print_input=False, remote_temp_filename=None, # ignored **command_kwargs, ): """ Upload file by chunking and sending base64 encoded via winrm """ # TODO: fix this? Workaround for circular import from pyinfra.facts.windows_files import WindowsTempDir # Always use temp file here in case of failure temp_file = ntpath.join( host.get_fact(WindowsTempDir), "pyinfra-{0}".format(sha1_hash(remote_filename)), ) if not _put_file(state, host, filename_or_io, temp_file): return False # Execute run_shell_command w/sudo and/or su_user command = "Move-Item -Path {0} -Destination {1} -Force".format( temp_file, remote_filename) status, _, stderr = run_shell_command(state, host, command, print_output=print_output, print_input=print_input, **command_kwargs) if status is False: logger.error("File upload error: {0}".format("\n".join(stderr))) return False if print_output: click.echo( "{0}file uploaded: {1}".format(host.print_prefix, remote_filename), err=True, ) return True
def put_file(state, hostname, file_io, remote_file, sudo=False, sudo_user=None, print_output=False): ''' Upload file-ios to the specified host using SFTP. Supports uploading files with sudo by uploading to a temporary directory then moving & chowning. ''' print_prefix = '[{0}] '.format(colored(hostname, attrs=['bold'])) if not sudo: _put_file(state, hostname, file_io, remote_file) else: # sudo is a little more complicated, as you can only sftp with the SSH user # connected, so upload to tmp and copy/chown w/sudo # Get temp file location temp_file = state.get_temp_filename(remote_file) _put_file(state, hostname, file_io, temp_file) # Execute run_shell_command w/sudo to mv/chown it command = 'mv {0} {1}'.format(temp_file, remote_file) if sudo_user: command = '{0} && chown {1} {2}'.format(command, sudo_user, remote_file) channel, _, stderr = run_shell_command(state, hostname, command, sudo=sudo, sudo_user=sudo_user, print_output=print_output) if channel.exit_status > 0: logger.error('File error: {0}'.format('\n'.join(stderr))) return False if print_output: print('{0}file uploaded: {1}'.format(print_prefix, remote_file))
def _put_file(state, host, filename_or_io, remote_location, chunk_size=2048): # this should work fine on smallish files, but there will be perf issues # on larger files both due to the full read, the base64 encoding, and # the latency when sending chunks with get_file_io(filename_or_io) as file_io: data = file_io.read() for i in range(0, len(data), chunk_size): chunk = data[i:i + chunk_size] ps = ('$data = [System.Convert]::FromBase64String("{0}"); ' '{1} -Value $data -Encoding byte -Path "{2}"').format( base64.b64encode(chunk).decode('utf-8'), 'Set-Content' if i == 0 else 'Add-Content', remote_location) status, _stdout, stderr = run_shell_command(state, host, ps) if status is False: logger.error('File upload error: {0}'.format( '\n'.join(stderr))) return False return True
def put_file(state, host, filename_or_io, remote_filename, print_output=False, print_input=False, **command_kwargs): ''' Upload file by chunking and sending base64 encoded via winrm ''' # Always use temp file here in case of failure temp_file = ntpath.join( host.fact.windows_temp_dir(), 'pyinfra-{0}'.format(sha1_hash(remote_filename)), ) if not _put_file(state, host, filename_or_io, temp_file): return False # Execute run_shell_command w/sudo and/or su_user command = 'Move-Item -Path {0} -Destination {1} -Force'.format( temp_file, remote_filename) status, _, stderr = run_shell_command(state, host, command, print_output=print_output, print_input=print_input, **command_kwargs) if status is False: logger.error('File upload error: {0}'.format('\n'.join(stderr))) return False if print_output: click.echo( '{0}file uploaded: {1}'.format(host.print_prefix, remote_filename), err=True, ) return True
def log_host_command_error(host, e, timeout=0): if isinstance(e, timeout_error): logger.error( "{0}{1}".format( host.print_prefix, click.style( "Command timed out after {0}s".format( timeout, ), "red", ), ), ) elif isinstance(e, (socket_error, SSHException)): logger.error( "{0}{1}".format( host.print_prefix, click.style( "Command socket/SSH error: {0}".format(format_exception(e)), "red", ), ), ) elif isinstance(e, IOError): logger.error( "{0}{1}".format( host.print_prefix, click.style( "Command IO error: {0}".format(format_exception(e)), "red", ), ), ) # Still here? Re-raise! else: raise e
def _log_connect_error(host, message, data): logger.error('{0}{1} ({2})'.format( host.print_prefix, click.style(message, 'red'), data, ))
def missing_host_key(self, client, hostname, key): logger.error("No host key for {0} found in known_hosts".format(hostname)) raise SSHException( "StrictPolicy: No host key for {0} found in known_hosts".format(hostname), )
def _run_server_op(state, host, op_hash): # Noop for this host? if op_hash not in state.ops[host]: logger.info('{0}{1}'.format( host.print_prefix, click.style( 'Skipped', 'blue', ), )) return True op_data = state.ops[host][op_hash] op_meta = state.op_meta[op_hash] logger.debug('Starting operation {0} on {1}'.format( ', '.join(op_meta['names']), host, )) state.ops_run.add(op_hash) # ...loop through each command for i, command in enumerate(op_data['commands']): status = False sudo = op_meta['sudo'] sudo_user = op_meta['sudo_user'] su_user = op_meta['su_user'] preserve_sudo_env = op_meta['preserve_sudo_env'] # As dicts, individual commands can override meta settings (ie on a # per-host basis generated during deploy). if isinstance(command, dict): if 'sudo' in command: sudo = command['sudo'] if 'sudo_user' in command: sudo_user = command['sudo_user'] if 'su_user' in command: su_user = command['su_user'] command = command['command'] # Now we attempt to execute the command # Tuples stand for callbacks & file uploads if isinstance(command, tuple): # If first element is function, it's a callback if isinstance(command[0], FunctionType): func, args, kwargs = command try: status = func(state, host, *args, **kwargs) # Custom functions could do anything, so expect anything! except Exception as e: logger.error('{0}{1}'.format( host.print_prefix, click.style( 'Unexpected error in Python callback: {0}'.format( format_exception(e), ), 'red', ), )) # Non-function mean files to copy else: file_io, remote_filename = command try: status = host.put_file( state, file_io, remote_filename, sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=state.print_output, ) except (timeout_error, socket_error, SSHException, IOError) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) # Must be a string/shell command: execute it on the server w/op-level preferences else: stdout = [] stderr = [] try: status, stdout, stderr = host.run_shell_command( state, command.strip(), sudo=sudo, sudo_user=sudo_user, su_user=su_user, preserve_sudo_env=preserve_sudo_env, timeout=op_meta['timeout'], get_pty=op_meta['get_pty'], env=op_meta['env'], print_output=state.print_output, ) except (timeout_error, socket_error, SSHException) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) # If we failed and have no already printed the stderr, print it if status is False and not state.print_output: has_stderr = False for line in stderr: has_stderr = True logger.error('{0}{1}'.format( host.print_prefix, click.style(line, 'red'), )) # Not all (like, most) programs output their errors to stderr! if not has_stderr: for line in stdout: logger.error('{0}{1}'.format( host.print_prefix, click.style(line, 'red'), )) # Break the loop to trigger a failure if status is False: break else: state.results[host]['commands'] += 1 # Commands didn't break, so count our successes & return True! else: # Count success state.results[host]['ops'] += 1 state.results[host]['success_ops'] += 1 logger.info('{0}{1}'.format( host.print_prefix, click.style( 'Success' if len(op_data['commands']) > 0 else 'No changes', 'green', ), )) # Trigger any success handler if op_meta['on_success']: op_meta['on_success'](state, host, op_hash) return True # Up error_ops & log state.results[host]['error_ops'] += 1 if op_meta['ignore_errors']: logger.warning('{0}{1}'.format( host.print_prefix, click.style('Error (ignored)', 'yellow'), )) else: logger.error('{0}{1}'.format( host.print_prefix, click.style('Error', 'red'), )) # Always trigger any error handler if op_meta['on_error']: op_meta['on_error'](state, host, op_hash) # Ignored, op "completes" w/ ignored error if op_meta['ignore_errors']: state.results[host]['ops'] += 1 # Unignored error -> False return False
def _run_server_op(state, host, op_hash): if op_hash not in state.ops[host]: logger.info('{0}{1}'.format(host.print_prefix, click.style('Skipped', 'blue'))) return True op_data = state.ops[host][op_hash] op_meta = state.op_meta[op_hash] logger.debug('Starting operation {0} on {1}'.format( ', '.join(op_meta['names']), host, )) state.ops_run.add(op_hash) # ...loop through each command for i, command in enumerate(op_data['commands']): if not isinstance(command, PyinfraCommand): raise TypeError( 'Command: {0} is not a valid pyinfra command!'.format(command)) status = False executor_kwarg_keys = get_executor_kwarg_keys() executor_kwargs = { key: op_meta[key] for key in executor_kwarg_keys if key in op_meta } executor_kwargs.update(command.executor_kwargs) # Now we attempt to execute the command # if isinstance(command, FunctionCommand): try: status = command.function(state, host, *command.args, **command.kwargs) except Exception as e: # Custom functions could do anything, so expect anything! logger.warning(traceback.format_exc()) logger.error('{0}{1}'.format( host.print_prefix, click.style( 'Unexpected error in Python callback: {0}'.format( format_exception(e), ), 'red', ), )) elif isinstance(command, FileUploadCommand): try: status = host.put_file(command.src, command.dest, print_output=state.print_output, print_input=state.print_input, **executor_kwargs) except (timeout_error, socket_error, SSHException, IOError) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) elif isinstance(command, FileDownloadCommand): try: status = host.get_file(command.src, command.dest, print_output=state.print_output, print_input=state.print_input, **executor_kwargs) except (timeout_error, socket_error, SSHException, IOError) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) elif isinstance(command, StringCommand): combined_output_lines = [] try: status, combined_output_lines = host.run_shell_command( command, print_output=state.print_output, print_input=state.print_input, return_combined_output=True, **executor_kwargs) except (timeout_error, socket_error, SSHException) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) # If we failed and have no already printed the stderr, print it if status is False and not state.print_output: for type_, line in combined_output_lines: if type_ == 'stderr': logger.error('{0}{1}'.format( host.print_prefix, click.style(line, 'red'), )) else: logger.error('{0}{1}'.format( host.print_prefix, line, )) else: raise TypeError( '{0} is an invalid pyinfra command!'.format(command)) # Break the loop to trigger a failure if status is False: break else: state.results[host]['commands'] += 1 # Commands didn't break, so count our successes & return True! else: # Count success state.results[host]['ops'] += 1 state.results[host]['success_ops'] += 1 logger.info('{0}{1}'.format( host.print_prefix, click.style( 'Success' if len(op_data['commands']) > 0 else 'No changes', 'green', ), )) # Trigger any success handler if op_meta['on_success']: op_meta['on_success'](state, host, op_hash) return True # Up error_ops & log state.results[host]['error_ops'] += 1 if op_meta['ignore_errors']: logger.warning('{0}{1}'.format( host.print_prefix, click.style('Error (ignored)', 'yellow'), )) else: logger.error('{0}{1}'.format( host.print_prefix, click.style('Error', 'red'), )) # Always trigger any error handler if op_meta['on_error']: op_meta['on_error'](state, host, op_hash) # Ignored, op "completes" w/ ignored error if op_meta['ignore_errors']: state.results[host]['ops'] += 1 # Unignored error -> False return False
def _run_op(state, hostname, op_hash): # Noop for this host? if op_hash not in state.ops[hostname]: logger.debug('(Skipping) no op {0} on {1}'.format(op_hash, hostname)) return True op_data = state.ops[hostname][op_hash] op_meta = state.op_meta[op_hash] stderr_buffer = [] print_prefix = '{}: '.format(hostname, attrs=['bold']) logger.debug('Starting operation {0} on {1}'.format( ', '.join(op_meta['names']), hostname)) state.ops_run.add(op_hash) # ...loop through each command for i, command in enumerate(op_data['commands']): status = True sudo = op_meta['sudo'] sudo_user = op_meta['sudo_user'] su_user = op_meta['su_user'] # As dicts, individual commands can override meta settings (ie on a per-host # basis generated during deploy). if isinstance(command, dict): if 'sudo' in command: sudo = command['sudo'] if 'sudo_user' in command: sudo_user = command['sudo_user'] if 'su_user' in command: su_user = command['su_user'] command = command['command'] # Tuples stand for callbacks & file uploads elif isinstance(command, tuple): # If first element is function, it's a callback if isinstance(command[0], FunctionType): status = command[0](state, state.inventory[hostname], hostname, *command[1], **command[2]) # Non-function mean files to copy else: status = put_file(state, hostname, *command, sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=state.print_output) # Must be a string/shell command: execute it on the server w/op-level preferences else: try: channel, _, stderr = run_shell_command( state, hostname, command.strip(), sudo=sudo, sudo_user=sudo_user, su_user=su_user, timeout=op_meta['timeout'], env=op_data['env'], print_output=state.print_output) # Keep stderr in case of error stderr_buffer.extend(stderr) status = channel.exit_status == 0 except timeout_error: timeout_message = 'Operation timeout after {0}s'.format( op_meta['timeout']) stderr_buffer.append(timeout_message) status = False # Print the timeout error as not printed by run_shell_command if state.print_output: print('{0}{1}'.format(print_prefix, timeout_message)) if status is False: break else: state.results[hostname]['commands'] += 1 # Commands didn't break, so count our successes & return True! else: # Count success state.results[hostname]['ops'] += 1 state.results[hostname]['success_ops'] += 1 logger.info('[{0}] {1}'.format( colored(hostname, attrs=['bold']), colored( 'Success' if len(op_data['commands']) > 0 else 'No changes', 'green'))) return True # If the op failed somewhere, print stderr (if not already printed!) if not state.print_output: for line in stderr_buffer: print(' {0}{1}'.format(print_prefix, colored(line, 'red'))) # Up error_ops & log state.results[hostname]['error_ops'] += 1 if op_meta['ignore_errors']: logger.warning('[{0}] {1}'.format(hostname, colored('Error (ignored)', 'yellow'))) else: logger.error('[{0}] {1}'.format(hostname, colored('Error', 'red'))) # Ignored, op "completes" w/ ignored error if op_meta['ignore_errors']: state.results[hostname]['ops'] += 1 return None # Unignored error -> False return False
def connect(host, **kwargs): ''' Connect to a single host. Returns the SSH client if succesful. Stateless by design so can be run in parallel. ''' logger.debug('Connecting to: {0} ({1})'.format(host.name, kwargs)) name = host.name hostname = host.data.ssh_hostname or name try: # Create new client & connect to the host client = SSHClient() client.set_missing_host_key_policy(MissingHostKeyPolicy()) client.connect(hostname, **kwargs) # Enable SSH forwarding session = client.get_transport().open_session() AgentRequestHandler(session) # Log logger.info('[{0}] {1}'.format(colored(name, attrs=['bold']), colored('Connected', 'green'))) return client except AuthenticationException as e: logger.error('Auth error on: {0}, {1}'.format(name, e)) except SSHException as e: logger.error('SSH error on: {0}, {1}'.format(name, e)) except gaierror: if hostname != name: logger.error('Could not resolve {0} host: {1}'.format( name, hostname)) else: logger.error('Could not resolve {0}'.format(name)) except socket_error as e: logger.error('Could not connect: {0}:{1}, {2}'.format( name, kwargs.get('port', 22), e)) except EOFError as e: logger.error('EOF error connecting to {0}: {1}'.format(name, e))
def get_facts(state, name, args=None, sudo=False, sudo_user=None, su_user=None): ''' Get a single fact for all hosts in the state. ''' sudo = sudo or state.config.SUDO sudo_user = sudo_user or state.config.SUDO_USER su_user = su_user or state.config.SU_USER ignore_errors = state.config.IGNORE_ERRORS # If inside an operation, fetch config meta if state.current_op_meta: sudo, sudo_user, su_user, ignore_errors = state.current_op_meta # Create an instance of the fact fact = FACTS[name]() # If we're inactive or (pipelining & inside an op): just return the defaults if not state.active or (state.pipelining and state.in_op): return {host.name: fact.default for host in state.inventory} command = fact.command if args: command = command(*args) # Make a hash which keeps facts unique - but usable cross-deploy/threads. Locks are # used to maintain order. fact_hash = make_hash( (name, command, sudo, sudo_user, su_user, ignore_errors)) # Lock! state.fact_locks.setdefault(fact_hash, Semaphore()).acquire() # Already got this fact? Unlock and return 'em if state.facts.get(fact_hash): state.fact_locks[fact_hash].release() return state.facts[fact_hash] # Execute the command for each state inventory in a greenlet greenlets = { host.name: state.fact_pool.spawn(run_shell_command, state, host.name, command, sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=state.print_fact_output) for host in state.inventory if host not in state.ready_hosts } hostname_facts = {} failed_hosts = set() # Collect the facts and any failures for hostname, greenlet in six.iteritems(greenlets): try: channel, stdout, stderr = greenlet.get() if stdout: data = fact.process(stdout) else: data = fact.default hostname_facts[hostname] = data except (timeout_error, SSHException): if ignore_errors: logger.warning('[{0}] {1}'.format( hostname, colored('Fact error (ignored)', 'yellow'))) else: failed_hosts.add(hostname) logger.error('[{0}] {1}'.format(hostname, colored('Fact error', 'red'))) log_name = colored(name, attrs=['bold']) if args: log = 'Loaded fact {0}: {1}'.format(log_name, args) else: log = 'Loaded fact {0}'.format(log_name) if state.print_fact_info: logger.info(log) else: logger.debug(log) # Check we've not failed if not ignore_errors: state.fail_hosts(failed_hosts) # Assign the facts state.facts[fact_hash] = hostname_facts # Release the lock, return the data state.fact_locks[fact_hash].release() return state.facts[fact_hash]
def _run_server_op(state, host, op_hash): state.trigger_callbacks('operation_host_start', host, op_hash) if op_hash not in state.ops[host]: logger.info('{0}{1}'.format(host.print_prefix, click.style('Skipped', 'blue'))) return True op_data = state.get_op_data(host, op_hash) global_kwargs = op_data['global_kwargs'] op_meta = state.get_op_meta(op_hash) ignore_errors = global_kwargs['ignore_errors'] logger.debug('Starting operation {0} on {1}'.format( ', '.join(op_meta['names']), host, )) executor_kwarg_keys = get_executor_kwarg_keys() base_executor_kwargs = { key: global_kwargs[key] for key in executor_kwarg_keys if key in global_kwargs } precondition = global_kwargs['precondition'] if precondition: show_pre_or_post_condition_warning('precondition') if precondition and not _run_shell_command( state, host, StringCommand(precondition), global_kwargs, base_executor_kwargs, ): log_error_or_warning( host, ignore_errors, description='precondition failed: {0}'.format(precondition), ) if not ignore_errors: state.trigger_callbacks('operation_host_error', host, op_hash) return False state.ops_run.add(op_hash) # ...loop through each command for i, command in enumerate(op_data['commands']): status = False executor_kwargs = base_executor_kwargs.copy() executor_kwargs.update(command.executor_kwargs) # Now we attempt to execute the command # if not isinstance(command, PyinfraCommand): raise TypeError( '{0} is an invalid pyinfra command!'.format(command)) if isinstance(command, FunctionCommand): try: status = command.execute(state, host, executor_kwargs) except Exception as e: # Custom functions could do anything, so expect anything! logger.warning(traceback.format_exc()) logger.error('{0}{1}'.format( host.print_prefix, click.style( 'Unexpected error in Python callback: {0}'.format( format_exception(e), ), 'red', ), )) elif isinstance(command, StringCommand): status = _run_shell_command(state, host, command, global_kwargs, executor_kwargs) else: try: status = command.execute(state, host, executor_kwargs) except (timeout_error, socket_error, SSHException, IOError) as e: log_host_command_error( host, e, timeout=global_kwargs['timeout'], ) # Break the loop to trigger a failure if status is False: break state.results[host]['commands'] += 1 # Commands didn't break, so count our successes & return True! else: postcondition = global_kwargs['postcondition'] if postcondition: show_pre_or_post_condition_warning('postcondition') if postcondition and not _run_shell_command( state, host, StringCommand(postcondition), global_kwargs, base_executor_kwargs, ): log_error_or_warning( host, ignore_errors, description='postcondition failed: {0}'.format(postcondition), ) if not ignore_errors: state.trigger_callbacks('operation_host_error', host, op_hash) return False # Count success state.results[host]['ops'] += 1 state.results[host]['success_ops'] += 1 logger.info('{0}{1}'.format( host.print_prefix, click.style( 'Success' if len(op_data['commands']) > 0 else 'No changes', 'green', ), )) # Trigger any success handler if global_kwargs['on_success']: global_kwargs['on_success'](state, host, op_hash) state.trigger_callbacks('operation_host_success', host, op_hash) return True # Up error_ops & log state.results[host]['error_ops'] += 1 log_error_or_warning(host, ignore_errors) # Always trigger any error handler if global_kwargs['on_error']: global_kwargs['on_error'](state, host, op_hash) # Ignored, op "completes" w/ ignored error if ignore_errors: state.results[host]['ops'] += 1 # Unignored error -> False state.trigger_callbacks('operation_host_error', host, op_hash) if ignore_errors: return True return False
def put_file( state, host, filename_or_io, remote_filename, remote_temp_filename=None, sudo=False, sudo_user=None, su_user=None, print_output=False, print_input=False, **command_kwargs, ): """ Upload file-ios to the specified host using SFTP. Supports uploading files with sudo by uploading to a temporary directory then moving & chowning. """ # sudo/su are a little more complicated, as you can only sftp with the SSH # user connected, so upload to tmp and copy/chown w/sudo and/or su_user if sudo or su_user: # Get temp file location temp_file = remote_temp_filename or state.get_temp_filename( remote_filename) _put_file(host, filename_or_io, temp_file) # Make sure our sudo/su user can access the file if su_user: command = StringCommand("setfacl", "-m", "u:{0}:r".format(su_user), temp_file) elif sudo_user: command = StringCommand("setfacl -m u:{0}:r".format(sudo_user), temp_file) if su_user or sudo_user: status, _, stderr = run_shell_command( state, host, command, sudo=False, print_output=print_output, print_input=print_input, **command_kwargs, ) if status is False: logger.error("Error on handover to sudo/su user: {0}".format( "\n".join(stderr))) return False # Execute run_shell_command w/sudo and/or su_user command = StringCommand("cp", temp_file, QuoteString(remote_filename)) status, _, stderr = run_shell_command( state, host, command, sudo=sudo, sudo_user=sudo_user, su_user=su_user, print_output=print_output, print_input=print_input, **command_kwargs, ) if status is False: logger.error("File upload error: {0}".format("\n".join(stderr))) return False # Delete the temporary file now that we've successfully copied it command = StringCommand("rm", "-f", temp_file) status, _, stderr = run_shell_command( state, host, command, sudo=False, print_output=print_output, print_input=print_input, **command_kwargs, ) if status is False: logger.error("Unable to remove temporary file: {0}".format( "\n".join(stderr))) return False # No sudo and no su_user, so just upload it! else: _put_file(host, filename_or_io, remote_filename) if print_output: click.echo( "{0}file uploaded: {1}".format(host.print_prefix, remote_filename), err=True, ) return True
def run_host_op(state, host, op_hash): state.trigger_callbacks("operation_host_start", host, op_hash) if op_hash not in state.ops[host]: logger.info("{0}{1}".format(host.print_prefix, click.style("Skipped", "blue"))) return True op_data = state.get_op_data(host, op_hash) global_kwargs = op_data["global_kwargs"] op_meta = state.get_op_meta(op_hash) ignore_errors = global_kwargs["ignore_errors"] continue_on_error = global_kwargs["continue_on_error"] logger.debug("Starting operation %r on %s", op_meta["names"], host) executor_kwarg_keys = get_executor_kwarg_keys() base_executor_kwargs = { key: global_kwargs[key] for key in executor_kwarg_keys if key in global_kwargs } precondition = global_kwargs["precondition"] if precondition: show_pre_or_post_condition_warning("precondition") if precondition and not _run_shell_command( state, host, StringCommand(precondition), global_kwargs, base_executor_kwargs, ): log_error_or_warning( host, ignore_errors, description="precondition failed: {0}".format(precondition), ) if not ignore_errors: state.trigger_callbacks("operation_host_error", host, op_hash) return False state.ops_run.add(op_hash) if host.executing_op_hash is None: host.executing_op_hash = op_hash else: host.nested_executing_op_hash = op_hash return_status = False did_error = False executed_commands = 0 all_combined_output_lines = [] for i, command in enumerate(op_data["commands"]): status = False executor_kwargs = base_executor_kwargs.copy() executor_kwargs.update(command.executor_kwargs) # Now we attempt to execute the command # if not isinstance(command, PyinfraCommand): raise TypeError( "{0} is an invalid pyinfra command!".format(command)) if isinstance(command, FunctionCommand): try: status = command.execute(state, host, executor_kwargs) except Exception as e: # Custom functions could do anything, so expect anything! logger.warning(traceback.format_exc()) logger.error( "{0}{1}".format( host.print_prefix, click.style( "Unexpected error in Python callback: {0}".format( format_exception(e), ), "red", ), ), ) elif isinstance(command, StringCommand): status, combined_output_lines = _run_shell_command( state, host, command, global_kwargs, executor_kwargs, return_combined_output=True, ) all_combined_output_lines.extend(combined_output_lines) else: try: status = command.execute(state, host, executor_kwargs) except (timeout_error, socket_error, SSHException, IOError) as e: log_host_command_error( host, e, timeout=global_kwargs["timeout"], ) # Break the loop to trigger a failure if status is False: if continue_on_error is True: did_error = True continue break executed_commands += 1 state.results[host]["commands"] += 1 # Commands didn't break, so count our successes & return True! else: postcondition = global_kwargs["postcondition"] if postcondition: show_pre_or_post_condition_warning("postcondition") if postcondition and not _run_shell_command( state, host, StringCommand(postcondition), global_kwargs, base_executor_kwargs, ): log_error_or_warning( host, ignore_errors, description="postcondition failed: {0}".format(postcondition), ) if not ignore_errors: state.trigger_callbacks("operation_host_error", host, op_hash) return False if not did_error: return_status = True if return_status is True: state.results[host]["ops"] += 1 state.results[host]["success_ops"] += 1 logger.info( "{0}{1}".format( host.print_prefix, click.style( "Success" if len(op_data["commands"]) > 0 else "No changes", "green", ), ), ) # Trigger any success handler if global_kwargs["on_success"]: global_kwargs["on_success"](state, host, op_hash) state.trigger_callbacks("operation_host_success", host, op_hash) else: if ignore_errors: state.results[host]["ignored_error_ops"] += 1 else: state.results[host]["error_ops"] += 1 if executed_commands: state.results[host]["partial_ops"] += 1 log_error_or_warning( host, ignore_errors, continue_on_error=continue_on_error, description= f"executed {executed_commands}/{len(op_data['commands'])} commands", ) # Always trigger any error handler if global_kwargs["on_error"]: global_kwargs["on_error"](state, host, op_hash) # Ignored, op "completes" w/ ignored error if ignore_errors: state.results[host]["ops"] += 1 # Unignored error -> False state.trigger_callbacks("operation_host_error", host, op_hash) if ignore_errors: return_status = True op_data["operation_meta"].set_combined_output_lines( all_combined_output_lines) if host.nested_executing_op_hash: host.nested_executing_op_hash = None else: host.executing_op_hash = None return return_status
def _run_server_op(state, host, op_hash): if op_hash not in state.ops[host]: logger.info('{0}{1}'.format(host.print_prefix, click.style('Skipped', 'blue'))) return True op_data = state.ops[host][op_hash] op_meta = state.op_meta[op_hash] logger.debug('Starting operation {0} on {1}'.format( ', '.join(op_meta['names']), host, )) state.ops_run.add(op_hash) # ...loop through each command for i, command in enumerate(op_data['commands']): status = False executor_kwarg_keys = get_executor_kwarg_keys() executor_kwargs = { key: op_meta[key] for key in executor_kwarg_keys if key in op_meta } # As dicts, individual commands can override meta settings (ie on a # per-host basis generated during deploy). if isinstance(command, dict): for key in executor_kwarg_keys: if key in command: executor_kwargs[key] = command[key] command = command['command'] # Now we attempt to execute the command # Tuples stand for callbacks & file uploads if isinstance(command, tuple): # If first element is function, it's a callback if isinstance(command[0], FunctionType): func, args, kwargs = command try: status = func(state, host, *args, **kwargs) # Custom functions could do anything, so expect anything! except Exception as e: logger.debug(traceback.format_exc()) logger.error('{0}{1}'.format( host.print_prefix, click.style( 'Unexpected error in Python callback: {0}'.format( format_exception(e), ), 'red', ), )) # Non-function mean files to copy else: method_type, first_file, second_file = command if method_type == 'upload': method = host.put_file elif method_type == 'download': method = host.get_file else: raise TypeError( '{0} is an invalid pyinfra command!'.format(command)) try: status = method(state, first_file, second_file, print_output=state.print_output, print_input=state.print_input, **executor_kwargs) except (timeout_error, socket_error, SSHException, IOError) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) # Must be a string/shell command: execute it on the server w/op-level preferences elif isinstance(command, six.string_types): combined_output_lines = [] try: status, combined_output_lines = host.run_shell_command( state, command.strip(), print_output=state.print_output, print_input=state.print_input, return_combined_output=True, **executor_kwargs) except (timeout_error, socket_error, SSHException) as e: log_host_command_error( host, e, timeout=op_meta['timeout'], ) # If we failed and have no already printed the stderr, print it if status is False and not state.print_output: for type_, line in combined_output_lines: if type_ == 'stderr': logger.error('{0}{1}'.format( host.print_prefix, click.style(line, 'red'), )) else: logger.error('{0}{1}'.format( host.print_prefix, line, )) else: raise TypeError( '{0} is an invalid pyinfra command!'.format(command)) # Break the loop to trigger a failure if status is False: break else: state.results[host]['commands'] += 1 # Commands didn't break, so count our successes & return True! else: # Count success state.results[host]['ops'] += 1 state.results[host]['success_ops'] += 1 logger.info('{0}{1}'.format( host.print_prefix, click.style( 'Success' if len(op_data['commands']) > 0 else 'No changes', 'green', ), )) # Trigger any success handler if op_meta['on_success']: op_meta['on_success'](state, host, op_hash) return True # Up error_ops & log state.results[host]['error_ops'] += 1 if op_meta['ignore_errors']: logger.warning('{0}{1}'.format( host.print_prefix, click.style('Error (ignored)', 'yellow'), )) else: logger.error('{0}{1}'.format( host.print_prefix, click.style('Error', 'red'), )) # Always trigger any error handler if op_meta['on_error']: op_meta['on_error'](state, host, op_hash) # Ignored, op "completes" w/ ignored error if op_meta['ignore_errors']: state.results[host]['ops'] += 1 # Unignored error -> False return False