def connect(self, reason=None, show_errors=True):
self._check_state()
if not self.connection:
self.state.trigger_callbacks('host_before_connect', self)
try:
self.connection = self.executor.connect(self.state, self)
except ConnectError as e:
if show_errors:
log_message = '{0}{1}'.format(
self.print_prefix,
click.style(e.args[0], 'red'),
)
logger.error(log_message)
self.state.trigger_callbacks('host_connect_error', self, e)
else:
log_message = '{0}{1}'.format(
self.print_prefix,
click.style('Connected', 'green'),
)
if reason:
log_message = '{0}{1}'.format(
log_message,
' ({0})'.format(reason),
)
logger.info(log_message)
self.state.trigger_callbacks('host_connect', self)
return self.connection
def connect(self, reason=None, show_errors=True, raise_exceptions=False):
self._check_state()
if not self.connection:
self.state.trigger_callbacks("host_before_connect", self)
try:
self.connection = self.executor.connect(self.state, self)
except ConnectError as e:
if show_errors:
log_message = "{0}{1}".format(
self.print_prefix,
click.style(e.args[0], "red"),
)
logger.error(log_message)
self.state.trigger_callbacks("host_connect_error", self, e)
if raise_exceptions:
raise
else:
log_message = "{0}{1}".format(
self.print_prefix,
click.style("Connected", "green"),
)
if reason:
log_message = "{0}{1}".format(
log_message,
" ({0})".format(reason),
)
logger.info(log_message)
self.state.trigger_callbacks("host_connect", self)
return self.connection
def put_file(
state,
host,
filename_or_io,
remote_file,
sudo=False,
sudo_user=None,
su_user=None,
print_output=False,
):
'''
Upload file-ios to the specified host using SFTP. Supports uploading files
with sudo by uploading to a temporary directory then moving & chowning.
'''
# sudo/su are a little more complicated, as you can only sftp with the SSH
# user connected, so upload to tmp and copy/chown w/sudo and/or su_user
if sudo or su_user:
# Get temp file location
temp_file = state.get_temp_filename(remote_file)
_put_file(host, filename_or_io, temp_file)
if print_output:
print('{0}file uploaded: {1}'.format(host.print_prefix,
remote_file))
# Execute run_shell_command w/sudo and/or su_user
command = 'mv {0} {1}'.format(temp_file, remote_file)
# Move it to the su_user if present
if su_user:
command = '{0} && chown {1} {2}'.format(command, su_user,
remote_file)
# Otherwise any sudo_user
elif sudo_user:
command = '{0} && chown {1} {2}'.format(command, sudo_user,
remote_file)
status, _, stderr = run_shell_command(
state,
host,
command,
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
print_output=print_output,
)
if status is False:
logger.error('File error: {0}'.format('\n'.join(stderr)))
return False
# No sudo and no su_user, so just upload it!
else:
_put_file(host, filename_or_io, remote_file)
if print_output:
print('{0}file uploaded: {1}'.format(host.print_prefix,
remote_file))
def connect(self, for_fact=None, show_errors=True):
self._check_state()
if not self.connection:
try:
self.connection = self.executor.connect(self.state, self)
except ConnectError as e:
if show_errors:
log_message = '{0}{1}'.format(
self.print_prefix,
click.style(e.args[0], 'red'),
)
logger.error(log_message)
else:
log_message = '{0}{1}'.format(
self.print_prefix,
click.style('Connected', 'green'),
)
if for_fact:
log_message = '{0}{1}'.format(
log_message,
' (for {0} fact)'.format(for_fact),
)
logger.info(log_message)
return self.connection
def get_file(
state, host, remote_filename, filename_or_io,
sudo=False, sudo_user=None, su_user=None,
print_output=False, print_input=False,
**command_kwargs
):
'''
Download a file from the remote host using SFTP. Supports download files
with sudo by copying to a temporary directory with read permissions,
downloading and then removing the copy.
'''
if sudo or su_user:
# Get temp file location
temp_file = state.get_temp_filename(remote_filename)
# Copy the file to the tempfile location and add read permissions
command = 'cp {0} {1} && chmod +r {0}'.format(remote_filename, temp_file)
copy_status, _, stderr = run_shell_command(
state, host, command,
sudo=sudo, sudo_user=sudo_user, su_user=su_user,
print_output=print_output,
print_input=print_input,
**command_kwargs
)
if copy_status is False:
logger.error('File download copy temp error: {0}'.format('\n'.join(stderr)))
return False
try:
_get_file(host, temp_file, filename_or_io)
# Ensure that, even if we encounter an error, we (attempt to) remove the
# temporary copy of the file.
finally:
remove_status, _, stderr = run_shell_command(
state, host, 'rm -f {0}'.format(temp_file),
sudo=sudo, sudo_user=sudo_user, su_user=su_user,
print_output=print_output,
print_input=print_input,
**command_kwargs
)
if remove_status is False:
logger.error('File download remove temp error: {0}'.format('\n'.join(stderr)))
return False
else:
_get_file(host, remote_filename, filename_or_io)
if print_output:
click.echo(
'{0}file downloaded: {1}'.format(host.print_prefix, remote_filename),
err=True,
)
return True
def print_host_combined_output(host, combined_output_lines):
for type_, line in combined_output_lines:
if type_ == 'stderr':
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(line, 'red'),
))
else:
logger.error('{0}{1}'.format(
host.print_prefix,
line,
))
def log_host_command_error(host, e, timeout=0):
if isinstance(e, timeout_error):
logger.error('{0}{1}'.format(
host.print_prefix,
click.style('Command timed out after {0}s'.format(timeout, ),
'red'),
))
elif isinstance(e, (socket_error, SSHException)):
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(
'Command socket/SSH error: {0}'.format(format_exception(e)),
'red',
),
))
def print_host_combined_output(host, combined_output_lines):
for type_, line in combined_output_lines:
if type_ == "stderr":
logger.error(
"{0}{1}".format(
host.print_prefix,
click.style(line, "red"),
),
)
else:
logger.error(
"{0}{1}".format(
host.print_prefix,
line,
),
)
def put_file(
state,
host,
filename_or_io,
remote_filename,
print_output=False,
print_input=False,
remote_temp_filename=None, # ignored
**command_kwargs,
):
"""
Upload file by chunking and sending base64 encoded via winrm
"""
# TODO: fix this? Workaround for circular import
from pyinfra.facts.windows_files import WindowsTempDir
# Always use temp file here in case of failure
temp_file = ntpath.join(
host.get_fact(WindowsTempDir),
"pyinfra-{0}".format(sha1_hash(remote_filename)),
)
if not _put_file(state, host, filename_or_io, temp_file):
return False
# Execute run_shell_command w/sudo and/or su_user
command = "Move-Item -Path {0} -Destination {1} -Force".format(
temp_file, remote_filename)
status, _, stderr = run_shell_command(state,
host,
command,
print_output=print_output,
print_input=print_input,
**command_kwargs)
if status is False:
logger.error("File upload error: {0}".format("\n".join(stderr)))
return False
if print_output:
click.echo(
"{0}file uploaded: {1}".format(host.print_prefix, remote_filename),
err=True,
)
return True
def put_file(state,
hostname,
file_io,
remote_file,
sudo=False,
sudo_user=None,
print_output=False):
'''
Upload file-ios to the specified host using SFTP. Supports uploading files with sudo
by uploading to a temporary directory then moving & chowning.
'''
print_prefix = '[{0}] '.format(colored(hostname, attrs=['bold']))
if not sudo:
_put_file(state, hostname, file_io, remote_file)
else:
# sudo is a little more complicated, as you can only sftp with the SSH user
# connected, so upload to tmp and copy/chown w/sudo
# Get temp file location
temp_file = state.get_temp_filename(remote_file)
_put_file(state, hostname, file_io, temp_file)
# Execute run_shell_command w/sudo to mv/chown it
command = 'mv {0} {1}'.format(temp_file, remote_file)
if sudo_user:
command = '{0} && chown {1} {2}'.format(command, sudo_user,
remote_file)
channel, _, stderr = run_shell_command(state,
hostname,
command,
sudo=sudo,
sudo_user=sudo_user,
print_output=print_output)
if channel.exit_status > 0:
logger.error('File error: {0}'.format('\n'.join(stderr)))
return False
if print_output:
print('{0}file uploaded: {1}'.format(print_prefix, remote_file))
def _put_file(state, host, filename_or_io, remote_location, chunk_size=2048):
# this should work fine on smallish files, but there will be perf issues
# on larger files both due to the full read, the base64 encoding, and
# the latency when sending chunks
with get_file_io(filename_or_io) as file_io:
data = file_io.read()
for i in range(0, len(data), chunk_size):
chunk = data[i:i + chunk_size]
ps = ('$data = [System.Convert]::FromBase64String("{0}"); '
'{1} -Value $data -Encoding byte -Path "{2}"').format(
base64.b64encode(chunk).decode('utf-8'),
'Set-Content' if i == 0 else 'Add-Content',
remote_location)
status, _stdout, stderr = run_shell_command(state, host, ps)
if status is False:
logger.error('File upload error: {0}'.format(
'\n'.join(stderr)))
return False
return True
def put_file(state,
host,
filename_or_io,
remote_filename,
print_output=False,
print_input=False,
**command_kwargs):
'''
Upload file by chunking and sending base64 encoded via winrm
'''
# Always use temp file here in case of failure
temp_file = ntpath.join(
host.fact.windows_temp_dir(),
'pyinfra-{0}'.format(sha1_hash(remote_filename)),
)
if not _put_file(state, host, filename_or_io, temp_file):
return False
# Execute run_shell_command w/sudo and/or su_user
command = 'Move-Item -Path {0} -Destination {1} -Force'.format(
temp_file, remote_filename)
status, _, stderr = run_shell_command(state,
host,
command,
print_output=print_output,
print_input=print_input,
**command_kwargs)
if status is False:
logger.error('File upload error: {0}'.format('\n'.join(stderr)))
return False
if print_output:
click.echo(
'{0}file uploaded: {1}'.format(host.print_prefix, remote_filename),
err=True,
)
return True
def log_host_command_error(host, e, timeout=0):
if isinstance(e, timeout_error):
logger.error(
"{0}{1}".format(
host.print_prefix,
click.style(
"Command timed out after {0}s".format(
timeout,
),
"red",
),
),
)
elif isinstance(e, (socket_error, SSHException)):
logger.error(
"{0}{1}".format(
host.print_prefix,
click.style(
"Command socket/SSH error: {0}".format(format_exception(e)),
"red",
),
),
)
elif isinstance(e, IOError):
logger.error(
"{0}{1}".format(
host.print_prefix,
click.style(
"Command IO error: {0}".format(format_exception(e)),
"red",
),
),
)
# Still here? Re-raise!
else:
raise e
def _log_connect_error(host, message, data):
logger.error('{0}{1} ({2})'.format(
host.print_prefix,
click.style(message, 'red'),
data,
))
def missing_host_key(self, client, hostname, key):
logger.error("No host key for {0} found in known_hosts".format(hostname))
raise SSHException(
"StrictPolicy: No host key for {0} found in known_hosts".format(hostname),
)
def _run_server_op(state, host, op_hash):
# Noop for this host?
if op_hash not in state.ops[host]:
logger.info('{0}{1}'.format(
host.print_prefix,
click.style(
'Skipped',
'blue',
),
))
return True
op_data = state.ops[host][op_hash]
op_meta = state.op_meta[op_hash]
logger.debug('Starting operation {0} on {1}'.format(
', '.join(op_meta['names']),
host,
))
state.ops_run.add(op_hash)
# ...loop through each command
for i, command in enumerate(op_data['commands']):
status = False
sudo = op_meta['sudo']
sudo_user = op_meta['sudo_user']
su_user = op_meta['su_user']
preserve_sudo_env = op_meta['preserve_sudo_env']
# As dicts, individual commands can override meta settings (ie on a
# per-host basis generated during deploy).
if isinstance(command, dict):
if 'sudo' in command:
sudo = command['sudo']
if 'sudo_user' in command:
sudo_user = command['sudo_user']
if 'su_user' in command:
su_user = command['su_user']
command = command['command']
# Now we attempt to execute the command
# Tuples stand for callbacks & file uploads
if isinstance(command, tuple):
# If first element is function, it's a callback
if isinstance(command[0], FunctionType):
func, args, kwargs = command
try:
status = func(state, host, *args, **kwargs)
# Custom functions could do anything, so expect anything!
except Exception as e:
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(
'Unexpected error in Python callback: {0}'.format(
format_exception(e), ),
'red',
),
))
# Non-function mean files to copy
else:
file_io, remote_filename = command
try:
status = host.put_file(
state,
file_io,
remote_filename,
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
print_output=state.print_output,
)
except (timeout_error, socket_error, SSHException,
IOError) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
# Must be a string/shell command: execute it on the server w/op-level preferences
else:
stdout = []
stderr = []
try:
status, stdout, stderr = host.run_shell_command(
state,
command.strip(),
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
preserve_sudo_env=preserve_sudo_env,
timeout=op_meta['timeout'],
get_pty=op_meta['get_pty'],
env=op_meta['env'],
print_output=state.print_output,
)
except (timeout_error, socket_error, SSHException) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
# If we failed and have no already printed the stderr, print it
if status is False and not state.print_output:
has_stderr = False
for line in stderr:
has_stderr = True
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(line, 'red'),
))
# Not all (like, most) programs output their errors to stderr!
if not has_stderr:
for line in stdout:
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(line, 'red'),
))
# Break the loop to trigger a failure
if status is False:
break
else:
state.results[host]['commands'] += 1
# Commands didn't break, so count our successes & return True!
else:
# Count success
state.results[host]['ops'] += 1
state.results[host]['success_ops'] += 1
logger.info('{0}{1}'.format(
host.print_prefix,
click.style(
'Success' if len(op_data['commands']) > 0 else 'No changes',
'green',
),
))
# Trigger any success handler
if op_meta['on_success']:
op_meta['on_success'](state, host, op_hash)
return True
# Up error_ops & log
state.results[host]['error_ops'] += 1
if op_meta['ignore_errors']:
logger.warning('{0}{1}'.format(
host.print_prefix,
click.style('Error (ignored)', 'yellow'),
))
else:
logger.error('{0}{1}'.format(
host.print_prefix,
click.style('Error', 'red'),
))
# Always trigger any error handler
if op_meta['on_error']:
op_meta['on_error'](state, host, op_hash)
# Ignored, op "completes" w/ ignored error
if op_meta['ignore_errors']:
state.results[host]['ops'] += 1
# Unignored error -> False
return False
def _run_server_op(state, host, op_hash):
if op_hash not in state.ops[host]:
logger.info('{0}{1}'.format(host.print_prefix,
click.style('Skipped', 'blue')))
return True
op_data = state.ops[host][op_hash]
op_meta = state.op_meta[op_hash]
logger.debug('Starting operation {0} on {1}'.format(
', '.join(op_meta['names']),
host,
))
state.ops_run.add(op_hash)
# ...loop through each command
for i, command in enumerate(op_data['commands']):
if not isinstance(command, PyinfraCommand):
raise TypeError(
'Command: {0} is not a valid pyinfra command!'.format(command))
status = False
executor_kwarg_keys = get_executor_kwarg_keys()
executor_kwargs = {
key: op_meta[key]
for key in executor_kwarg_keys if key in op_meta
}
executor_kwargs.update(command.executor_kwargs)
# Now we attempt to execute the command
#
if isinstance(command, FunctionCommand):
try:
status = command.function(state, host, *command.args,
**command.kwargs)
except Exception as e: # Custom functions could do anything, so expect anything!
logger.warning(traceback.format_exc())
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(
'Unexpected error in Python callback: {0}'.format(
format_exception(e), ),
'red',
),
))
elif isinstance(command, FileUploadCommand):
try:
status = host.put_file(command.src,
command.dest,
print_output=state.print_output,
print_input=state.print_input,
**executor_kwargs)
except (timeout_error, socket_error, SSHException, IOError) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
elif isinstance(command, FileDownloadCommand):
try:
status = host.get_file(command.src,
command.dest,
print_output=state.print_output,
print_input=state.print_input,
**executor_kwargs)
except (timeout_error, socket_error, SSHException, IOError) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
elif isinstance(command, StringCommand):
combined_output_lines = []
try:
status, combined_output_lines = host.run_shell_command(
command,
print_output=state.print_output,
print_input=state.print_input,
return_combined_output=True,
**executor_kwargs)
except (timeout_error, socket_error, SSHException) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
# If we failed and have no already printed the stderr, print it
if status is False and not state.print_output:
for type_, line in combined_output_lines:
if type_ == 'stderr':
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(line, 'red'),
))
else:
logger.error('{0}{1}'.format(
host.print_prefix,
line,
))
else:
raise TypeError(
'{0} is an invalid pyinfra command!'.format(command))
# Break the loop to trigger a failure
if status is False:
break
else:
state.results[host]['commands'] += 1
# Commands didn't break, so count our successes & return True!
else:
# Count success
state.results[host]['ops'] += 1
state.results[host]['success_ops'] += 1
logger.info('{0}{1}'.format(
host.print_prefix,
click.style(
'Success' if len(op_data['commands']) > 0 else 'No changes',
'green',
),
))
# Trigger any success handler
if op_meta['on_success']:
op_meta['on_success'](state, host, op_hash)
return True
# Up error_ops & log
state.results[host]['error_ops'] += 1
if op_meta['ignore_errors']:
logger.warning('{0}{1}'.format(
host.print_prefix,
click.style('Error (ignored)', 'yellow'),
))
else:
logger.error('{0}{1}'.format(
host.print_prefix,
click.style('Error', 'red'),
))
# Always trigger any error handler
if op_meta['on_error']:
op_meta['on_error'](state, host, op_hash)
# Ignored, op "completes" w/ ignored error
if op_meta['ignore_errors']:
state.results[host]['ops'] += 1
# Unignored error -> False
return False
def _run_op(state, hostname, op_hash):
# Noop for this host?
if op_hash not in state.ops[hostname]:
logger.debug('(Skipping) no op {0} on {1}'.format(op_hash, hostname))
return True
op_data = state.ops[hostname][op_hash]
op_meta = state.op_meta[op_hash]
stderr_buffer = []
print_prefix = '{}: '.format(hostname, attrs=['bold'])
logger.debug('Starting operation {0} on {1}'.format(
', '.join(op_meta['names']), hostname))
state.ops_run.add(op_hash)
# ...loop through each command
for i, command in enumerate(op_data['commands']):
status = True
sudo = op_meta['sudo']
sudo_user = op_meta['sudo_user']
su_user = op_meta['su_user']
# As dicts, individual commands can override meta settings (ie on a per-host
# basis generated during deploy).
if isinstance(command, dict):
if 'sudo' in command:
sudo = command['sudo']
if 'sudo_user' in command:
sudo_user = command['sudo_user']
if 'su_user' in command:
su_user = command['su_user']
command = command['command']
# Tuples stand for callbacks & file uploads
elif isinstance(command, tuple):
# If first element is function, it's a callback
if isinstance(command[0], FunctionType):
status = command[0](state, state.inventory[hostname], hostname,
*command[1], **command[2])
# Non-function mean files to copy
else:
status = put_file(state,
hostname,
*command,
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
print_output=state.print_output)
# Must be a string/shell command: execute it on the server w/op-level preferences
else:
try:
channel, _, stderr = run_shell_command(
state,
hostname,
command.strip(),
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
timeout=op_meta['timeout'],
env=op_data['env'],
print_output=state.print_output)
# Keep stderr in case of error
stderr_buffer.extend(stderr)
status = channel.exit_status == 0
except timeout_error:
timeout_message = 'Operation timeout after {0}s'.format(
op_meta['timeout'])
stderr_buffer.append(timeout_message)
status = False
# Print the timeout error as not printed by run_shell_command
if state.print_output:
print('{0}{1}'.format(print_prefix, timeout_message))
if status is False:
break
else:
state.results[hostname]['commands'] += 1
# Commands didn't break, so count our successes & return True!
else:
# Count success
state.results[hostname]['ops'] += 1
state.results[hostname]['success_ops'] += 1
logger.info('[{0}] {1}'.format(
colored(hostname, attrs=['bold']),
colored(
'Success' if len(op_data['commands']) > 0 else 'No changes',
'green')))
return True
# If the op failed somewhere, print stderr (if not already printed!)
if not state.print_output:
for line in stderr_buffer:
print(' {0}{1}'.format(print_prefix, colored(line, 'red')))
# Up error_ops & log
state.results[hostname]['error_ops'] += 1
if op_meta['ignore_errors']:
logger.warning('[{0}] {1}'.format(hostname,
colored('Error (ignored)',
'yellow')))
else:
logger.error('[{0}] {1}'.format(hostname, colored('Error', 'red')))
# Ignored, op "completes" w/ ignored error
if op_meta['ignore_errors']:
state.results[hostname]['ops'] += 1
return None
# Unignored error -> False
return False
def connect(host, **kwargs):
'''
Connect to a single host. Returns the SSH client if succesful. Stateless by design so
can be run in parallel.
'''
logger.debug('Connecting to: {0} ({1})'.format(host.name, kwargs))
name = host.name
hostname = host.data.ssh_hostname or name
try:
# Create new client & connect to the host
client = SSHClient()
client.set_missing_host_key_policy(MissingHostKeyPolicy())
client.connect(hostname, **kwargs)
# Enable SSH forwarding
session = client.get_transport().open_session()
AgentRequestHandler(session)
# Log
logger.info('[{0}] {1}'.format(colored(name, attrs=['bold']),
colored('Connected', 'green')))
return client
except AuthenticationException as e:
logger.error('Auth error on: {0}, {1}'.format(name, e))
except SSHException as e:
logger.error('SSH error on: {0}, {1}'.format(name, e))
except gaierror:
if hostname != name:
logger.error('Could not resolve {0} host: {1}'.format(
name, hostname))
else:
logger.error('Could not resolve {0}'.format(name))
except socket_error as e:
logger.error('Could not connect: {0}:{1}, {2}'.format(
name, kwargs.get('port', 22), e))
except EOFError as e:
logger.error('EOF error connecting to {0}: {1}'.format(name, e))
def get_facts(state,
name,
args=None,
sudo=False,
sudo_user=None,
su_user=None):
'''
Get a single fact for all hosts in the state.
'''
sudo = sudo or state.config.SUDO
sudo_user = sudo_user or state.config.SUDO_USER
su_user = su_user or state.config.SU_USER
ignore_errors = state.config.IGNORE_ERRORS
# If inside an operation, fetch config meta
if state.current_op_meta:
sudo, sudo_user, su_user, ignore_errors = state.current_op_meta
# Create an instance of the fact
fact = FACTS[name]()
# If we're inactive or (pipelining & inside an op): just return the defaults
if not state.active or (state.pipelining and state.in_op):
return {host.name: fact.default for host in state.inventory}
command = fact.command
if args:
command = command(*args)
# Make a hash which keeps facts unique - but usable cross-deploy/threads. Locks are
# used to maintain order.
fact_hash = make_hash(
(name, command, sudo, sudo_user, su_user, ignore_errors))
# Lock!
state.fact_locks.setdefault(fact_hash, Semaphore()).acquire()
# Already got this fact? Unlock and return 'em
if state.facts.get(fact_hash):
state.fact_locks[fact_hash].release()
return state.facts[fact_hash]
# Execute the command for each state inventory in a greenlet
greenlets = {
host.name: state.fact_pool.spawn(run_shell_command,
state,
host.name,
command,
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
print_output=state.print_fact_output)
for host in state.inventory if host not in state.ready_hosts
}
hostname_facts = {}
failed_hosts = set()
# Collect the facts and any failures
for hostname, greenlet in six.iteritems(greenlets):
try:
channel, stdout, stderr = greenlet.get()
if stdout:
data = fact.process(stdout)
else:
data = fact.default
hostname_facts[hostname] = data
except (timeout_error, SSHException):
if ignore_errors:
logger.warning('[{0}] {1}'.format(
hostname, colored('Fact error (ignored)', 'yellow')))
else:
failed_hosts.add(hostname)
logger.error('[{0}] {1}'.format(hostname,
colored('Fact error', 'red')))
log_name = colored(name, attrs=['bold'])
if args:
log = 'Loaded fact {0}: {1}'.format(log_name, args)
else:
log = 'Loaded fact {0}'.format(log_name)
if state.print_fact_info:
logger.info(log)
else:
logger.debug(log)
# Check we've not failed
if not ignore_errors:
state.fail_hosts(failed_hosts)
# Assign the facts
state.facts[fact_hash] = hostname_facts
# Release the lock, return the data
state.fact_locks[fact_hash].release()
return state.facts[fact_hash]
def _run_server_op(state, host, op_hash):
state.trigger_callbacks('operation_host_start', host, op_hash)
if op_hash not in state.ops[host]:
logger.info('{0}{1}'.format(host.print_prefix,
click.style('Skipped', 'blue')))
return True
op_data = state.get_op_data(host, op_hash)
global_kwargs = op_data['global_kwargs']
op_meta = state.get_op_meta(op_hash)
ignore_errors = global_kwargs['ignore_errors']
logger.debug('Starting operation {0} on {1}'.format(
', '.join(op_meta['names']),
host,
))
executor_kwarg_keys = get_executor_kwarg_keys()
base_executor_kwargs = {
key: global_kwargs[key]
for key in executor_kwarg_keys if key in global_kwargs
}
precondition = global_kwargs['precondition']
if precondition:
show_pre_or_post_condition_warning('precondition')
if precondition and not _run_shell_command(
state,
host,
StringCommand(precondition),
global_kwargs,
base_executor_kwargs,
):
log_error_or_warning(
host,
ignore_errors,
description='precondition failed: {0}'.format(precondition),
)
if not ignore_errors:
state.trigger_callbacks('operation_host_error', host, op_hash)
return False
state.ops_run.add(op_hash)
# ...loop through each command
for i, command in enumerate(op_data['commands']):
status = False
executor_kwargs = base_executor_kwargs.copy()
executor_kwargs.update(command.executor_kwargs)
# Now we attempt to execute the command
#
if not isinstance(command, PyinfraCommand):
raise TypeError(
'{0} is an invalid pyinfra command!'.format(command))
if isinstance(command, FunctionCommand):
try:
status = command.execute(state, host, executor_kwargs)
except Exception as e: # Custom functions could do anything, so expect anything!
logger.warning(traceback.format_exc())
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(
'Unexpected error in Python callback: {0}'.format(
format_exception(e), ),
'red',
),
))
elif isinstance(command, StringCommand):
status = _run_shell_command(state, host, command, global_kwargs,
executor_kwargs)
else:
try:
status = command.execute(state, host, executor_kwargs)
except (timeout_error, socket_error, SSHException, IOError) as e:
log_host_command_error(
host,
e,
timeout=global_kwargs['timeout'],
)
# Break the loop to trigger a failure
if status is False:
break
state.results[host]['commands'] += 1
# Commands didn't break, so count our successes & return True!
else:
postcondition = global_kwargs['postcondition']
if postcondition:
show_pre_or_post_condition_warning('postcondition')
if postcondition and not _run_shell_command(
state,
host,
StringCommand(postcondition),
global_kwargs,
base_executor_kwargs,
):
log_error_or_warning(
host,
ignore_errors,
description='postcondition failed: {0}'.format(postcondition),
)
if not ignore_errors:
state.trigger_callbacks('operation_host_error', host, op_hash)
return False
# Count success
state.results[host]['ops'] += 1
state.results[host]['success_ops'] += 1
logger.info('{0}{1}'.format(
host.print_prefix,
click.style(
'Success' if len(op_data['commands']) > 0 else 'No changes',
'green',
),
))
# Trigger any success handler
if global_kwargs['on_success']:
global_kwargs['on_success'](state, host, op_hash)
state.trigger_callbacks('operation_host_success', host, op_hash)
return True
# Up error_ops & log
state.results[host]['error_ops'] += 1
log_error_or_warning(host, ignore_errors)
# Always trigger any error handler
if global_kwargs['on_error']:
global_kwargs['on_error'](state, host, op_hash)
# Ignored, op "completes" w/ ignored error
if ignore_errors:
state.results[host]['ops'] += 1
# Unignored error -> False
state.trigger_callbacks('operation_host_error', host, op_hash)
if ignore_errors:
return True
return False
def put_file(
state,
host,
filename_or_io,
remote_filename,
remote_temp_filename=None,
sudo=False,
sudo_user=None,
su_user=None,
print_output=False,
print_input=False,
**command_kwargs,
):
"""
Upload file-ios to the specified host using SFTP. Supports uploading files
with sudo by uploading to a temporary directory then moving & chowning.
"""
# sudo/su are a little more complicated, as you can only sftp with the SSH
# user connected, so upload to tmp and copy/chown w/sudo and/or su_user
if sudo or su_user:
# Get temp file location
temp_file = remote_temp_filename or state.get_temp_filename(
remote_filename)
_put_file(host, filename_or_io, temp_file)
# Make sure our sudo/su user can access the file
if su_user:
command = StringCommand("setfacl", "-m", "u:{0}:r".format(su_user),
temp_file)
elif sudo_user:
command = StringCommand("setfacl -m u:{0}:r".format(sudo_user),
temp_file)
if su_user or sudo_user:
status, _, stderr = run_shell_command(
state,
host,
command,
sudo=False,
print_output=print_output,
print_input=print_input,
**command_kwargs,
)
if status is False:
logger.error("Error on handover to sudo/su user: {0}".format(
"\n".join(stderr)))
return False
# Execute run_shell_command w/sudo and/or su_user
command = StringCommand("cp", temp_file, QuoteString(remote_filename))
status, _, stderr = run_shell_command(
state,
host,
command,
sudo=sudo,
sudo_user=sudo_user,
su_user=su_user,
print_output=print_output,
print_input=print_input,
**command_kwargs,
)
if status is False:
logger.error("File upload error: {0}".format("\n".join(stderr)))
return False
# Delete the temporary file now that we've successfully copied it
command = StringCommand("rm", "-f", temp_file)
status, _, stderr = run_shell_command(
state,
host,
command,
sudo=False,
print_output=print_output,
print_input=print_input,
**command_kwargs,
)
if status is False:
logger.error("Unable to remove temporary file: {0}".format(
"\n".join(stderr)))
return False
# No sudo and no su_user, so just upload it!
else:
_put_file(host, filename_or_io, remote_filename)
if print_output:
click.echo(
"{0}file uploaded: {1}".format(host.print_prefix, remote_filename),
err=True,
)
return True
def run_host_op(state, host, op_hash):
state.trigger_callbacks("operation_host_start", host, op_hash)
if op_hash not in state.ops[host]:
logger.info("{0}{1}".format(host.print_prefix,
click.style("Skipped", "blue")))
return True
op_data = state.get_op_data(host, op_hash)
global_kwargs = op_data["global_kwargs"]
op_meta = state.get_op_meta(op_hash)
ignore_errors = global_kwargs["ignore_errors"]
continue_on_error = global_kwargs["continue_on_error"]
logger.debug("Starting operation %r on %s", op_meta["names"], host)
executor_kwarg_keys = get_executor_kwarg_keys()
base_executor_kwargs = {
key: global_kwargs[key]
for key in executor_kwarg_keys if key in global_kwargs
}
precondition = global_kwargs["precondition"]
if precondition:
show_pre_or_post_condition_warning("precondition")
if precondition and not _run_shell_command(
state,
host,
StringCommand(precondition),
global_kwargs,
base_executor_kwargs,
):
log_error_or_warning(
host,
ignore_errors,
description="precondition failed: {0}".format(precondition),
)
if not ignore_errors:
state.trigger_callbacks("operation_host_error", host, op_hash)
return False
state.ops_run.add(op_hash)
if host.executing_op_hash is None:
host.executing_op_hash = op_hash
else:
host.nested_executing_op_hash = op_hash
return_status = False
did_error = False
executed_commands = 0
all_combined_output_lines = []
for i, command in enumerate(op_data["commands"]):
status = False
executor_kwargs = base_executor_kwargs.copy()
executor_kwargs.update(command.executor_kwargs)
# Now we attempt to execute the command
#
if not isinstance(command, PyinfraCommand):
raise TypeError(
"{0} is an invalid pyinfra command!".format(command))
if isinstance(command, FunctionCommand):
try:
status = command.execute(state, host, executor_kwargs)
except Exception as e: # Custom functions could do anything, so expect anything!
logger.warning(traceback.format_exc())
logger.error(
"{0}{1}".format(
host.print_prefix,
click.style(
"Unexpected error in Python callback: {0}".format(
format_exception(e), ),
"red",
),
), )
elif isinstance(command, StringCommand):
status, combined_output_lines = _run_shell_command(
state,
host,
command,
global_kwargs,
executor_kwargs,
return_combined_output=True,
)
all_combined_output_lines.extend(combined_output_lines)
else:
try:
status = command.execute(state, host, executor_kwargs)
except (timeout_error, socket_error, SSHException, IOError) as e:
log_host_command_error(
host,
e,
timeout=global_kwargs["timeout"],
)
# Break the loop to trigger a failure
if status is False:
if continue_on_error is True:
did_error = True
continue
break
executed_commands += 1
state.results[host]["commands"] += 1
# Commands didn't break, so count our successes & return True!
else:
postcondition = global_kwargs["postcondition"]
if postcondition:
show_pre_or_post_condition_warning("postcondition")
if postcondition and not _run_shell_command(
state,
host,
StringCommand(postcondition),
global_kwargs,
base_executor_kwargs,
):
log_error_or_warning(
host,
ignore_errors,
description="postcondition failed: {0}".format(postcondition),
)
if not ignore_errors:
state.trigger_callbacks("operation_host_error", host, op_hash)
return False
if not did_error:
return_status = True
if return_status is True:
state.results[host]["ops"] += 1
state.results[host]["success_ops"] += 1
logger.info(
"{0}{1}".format(
host.print_prefix,
click.style(
"Success"
if len(op_data["commands"]) > 0 else "No changes",
"green",
),
), )
# Trigger any success handler
if global_kwargs["on_success"]:
global_kwargs["on_success"](state, host, op_hash)
state.trigger_callbacks("operation_host_success", host, op_hash)
else:
if ignore_errors:
state.results[host]["ignored_error_ops"] += 1
else:
state.results[host]["error_ops"] += 1
if executed_commands:
state.results[host]["partial_ops"] += 1
log_error_or_warning(
host,
ignore_errors,
continue_on_error=continue_on_error,
description=
f"executed {executed_commands}/{len(op_data['commands'])} commands",
)
# Always trigger any error handler
if global_kwargs["on_error"]:
global_kwargs["on_error"](state, host, op_hash)
# Ignored, op "completes" w/ ignored error
if ignore_errors:
state.results[host]["ops"] += 1
# Unignored error -> False
state.trigger_callbacks("operation_host_error", host, op_hash)
if ignore_errors:
return_status = True
op_data["operation_meta"].set_combined_output_lines(
all_combined_output_lines)
if host.nested_executing_op_hash:
host.nested_executing_op_hash = None
else:
host.executing_op_hash = None
return return_status
def _run_server_op(state, host, op_hash):
if op_hash not in state.ops[host]:
logger.info('{0}{1}'.format(host.print_prefix,
click.style('Skipped', 'blue')))
return True
op_data = state.ops[host][op_hash]
op_meta = state.op_meta[op_hash]
logger.debug('Starting operation {0} on {1}'.format(
', '.join(op_meta['names']),
host,
))
state.ops_run.add(op_hash)
# ...loop through each command
for i, command in enumerate(op_data['commands']):
status = False
executor_kwarg_keys = get_executor_kwarg_keys()
executor_kwargs = {
key: op_meta[key]
for key in executor_kwarg_keys if key in op_meta
}
# As dicts, individual commands can override meta settings (ie on a
# per-host basis generated during deploy).
if isinstance(command, dict):
for key in executor_kwarg_keys:
if key in command:
executor_kwargs[key] = command[key]
command = command['command']
# Now we attempt to execute the command
# Tuples stand for callbacks & file uploads
if isinstance(command, tuple):
# If first element is function, it's a callback
if isinstance(command[0], FunctionType):
func, args, kwargs = command
try:
status = func(state, host, *args, **kwargs)
# Custom functions could do anything, so expect anything!
except Exception as e:
logger.debug(traceback.format_exc())
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(
'Unexpected error in Python callback: {0}'.format(
format_exception(e), ),
'red',
),
))
# Non-function mean files to copy
else:
method_type, first_file, second_file = command
if method_type == 'upload':
method = host.put_file
elif method_type == 'download':
method = host.get_file
else:
raise TypeError(
'{0} is an invalid pyinfra command!'.format(command))
try:
status = method(state,
first_file,
second_file,
print_output=state.print_output,
print_input=state.print_input,
**executor_kwargs)
except (timeout_error, socket_error, SSHException,
IOError) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
# Must be a string/shell command: execute it on the server w/op-level preferences
elif isinstance(command, six.string_types):
combined_output_lines = []
try:
status, combined_output_lines = host.run_shell_command(
state,
command.strip(),
print_output=state.print_output,
print_input=state.print_input,
return_combined_output=True,
**executor_kwargs)
except (timeout_error, socket_error, SSHException) as e:
log_host_command_error(
host,
e,
timeout=op_meta['timeout'],
)
# If we failed and have no already printed the stderr, print it
if status is False and not state.print_output:
for type_, line in combined_output_lines:
if type_ == 'stderr':
logger.error('{0}{1}'.format(
host.print_prefix,
click.style(line, 'red'),
))
else:
logger.error('{0}{1}'.format(
host.print_prefix,
line,
))
else:
raise TypeError(
'{0} is an invalid pyinfra command!'.format(command))
# Break the loop to trigger a failure
if status is False:
break
else:
state.results[host]['commands'] += 1
# Commands didn't break, so count our successes & return True!
else:
# Count success
state.results[host]['ops'] += 1
state.results[host]['success_ops'] += 1
logger.info('{0}{1}'.format(
host.print_prefix,
click.style(
'Success' if len(op_data['commands']) > 0 else 'No changes',
'green',
),
))
# Trigger any success handler
if op_meta['on_success']:
op_meta['on_success'](state, host, op_hash)
return True
# Up error_ops & log
state.results[host]['error_ops'] += 1
if op_meta['ignore_errors']:
logger.warning('{0}{1}'.format(
host.print_prefix,
click.style('Error (ignored)', 'yellow'),
))
else:
logger.error('{0}{1}'.format(
host.print_prefix,
click.style('Error', 'red'),
))
# Always trigger any error handler
if op_meta['on_error']:
op_meta['on_error'](state, host, op_hash)
# Ignored, op "completes" w/ ignored error
if op_meta['ignore_errors']:
state.results[host]['ops'] += 1
# Unignored error -> False
return False
