def get_mech_config(limit=None):
logger.info('Getting Mech config...')
if limit and not isinstance(limit, (list, tuple)):
limit = [limit]
# Note: There is no "--machine-readable" option to 'mech status'
with progress_spinner({'mech ls'}) as progress:
output = local.shell(
'mech ls',
splitlines=True,
)
progress('mech ls')
targets = []
for line in output:
address = ''
data = line.split()
target = data[0]
if len(data) == 5:
address = data[1]
# Skip anything not in the limit
if limit is not None and target not in limit:
continue
# For each vm that has an address, fetch it's SSH config in a thread
if address != '' and address[0].isdigit():
targets.append(target)
threads = []
config_queue = Queue()
with progress_spinner(targets) as progress:
for target in targets:
thread = Thread(
target=_get_mech_ssh_config,
args=(config_queue, progress, target),
)
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
queue_items = list(config_queue.queue)
lines = []
for output in queue_items:
lines.extend(output)
return lines
def get_vagrant_config(limit=None):
logger.info("Getting Vagrant config...")
if limit and not isinstance(limit, (list, tuple)):
limit = [limit]
with progress_spinner({"vagrant status"}) as progress:
output = local.shell(
"vagrant status --machine-readable",
splitlines=True,
)
progress("vagrant status")
targets = []
for line in output:
line = line.strip()
_, target, type_, data = line.split(",", 3)
# Skip anything not in the limit
if limit is not None and target not in limit:
continue
# For each running container - fetch it's SSH config in a thread - this
# is because Vagrant *really* slow to run each command.
if type_ == "state" and data == "running":
targets.append(target)
threads = []
config_queue = Queue()
with progress_spinner(targets) as progress:
for target in targets:
thread = Thread(
target=_get_vagrant_ssh_config,
args=(config_queue, progress, target),
)
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
queue_items = list(config_queue.queue)
lines = []
for output in queue_items:
lines.extend([ln.strip() for ln in output])
return lines
def connect(state, host):
if not host.connection:
host.connection = ssh.connect(state, host)
if "docker_container_id" in host.host_data: # user can provide a docker_container_id
return host.connection
try:
with progress_spinner({"docker run"}):
# last line is the container ID
status, stdout, stderr = ssh.run_shell_command(
state,
host,
"docker run -d {0} tail -f /dev/null".format(
host.data.docker_image),
)
if not status:
raise IOError("\n".join(stderr))
container_id = stdout[-1]
except PyinfraError as e:
host.connection = None # fail connection
raise ConnectError(e.args[0])
host.host_data["docker_container_id"] = container_id
return host.connection
def connect_all(state):
'''
Connect to all the configured servers in parallel. Reads/writes state.inventory.
Args:
state (``pyinfra.api.State`` obj): the state containing an inventory to connect to
'''
hosts = [host for host in state.inventory if state.is_host_in_limit(host)]
greenlet_to_host = {
state.pool.spawn(host.connect, state): host
for host in hosts
}
with progress_spinner(greenlet_to_host.values()) as progress:
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
progress(host)
# Get/set the results
failed_hosts = set()
for greenlet, host in six.iteritems(greenlet_to_host):
# Raise any unexpected exception
greenlet.get()
if host.connection:
state.activate_host(host)
else:
failed_hosts.add(host)
# Remove those that failed, triggering FAIL_PERCENT check
state.fail_hosts(failed_hosts, activated_count=len(hosts))
def disconnect(state, host):
container_id = host.host_data['docker_container_id'][:12]
with progress_spinner({'docker commit'}):
image_id = local.shell(
'docker commit {0}'.format(container_id),
splitlines=True,
)[-1][7:19] # last line is the image ID, get sha256:[XXXXXXXXXX]...
with progress_spinner({'docker rm'}):
local.shell('docker rm -f {0}'.format(container_id), )
logger.info('{0}docker build complete, image ID: {1}'.format(
host.print_prefix,
click.style(image_id, bold=True),
))
def _get_vagrant_config(limit=None):
if limit and not isinstance(limit, list):
limit = [limit]
with progress_spinner({'vagrant status'}) as progress:
output = local.shell(
'vagrant status --machine-readable',
splitlines=True,
)
progress('vagrant status')
targets = []
for line in output:
_, target, type_, data = line.split(',', 3)
# Skip anything not in the limit
if limit is not None and target not in limit:
continue
# For each running container - fetch it's SSH config in a thread - this
# is because Vagrant *really* slow to run each command.
if type_ == 'state' and data == 'running':
targets.append(target)
threads = []
config_queue = Queue()
with progress_spinner(targets) as progress:
for target in targets:
thread = Thread(
target=_get_vagrant_ssh_config,
args=(config_queue, progress, target),
)
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
queue_items = list(config_queue.queue)
lines = []
for output in queue_items:
lines.extend(output)
return lines
def disconnect(state, host):
container_id = host.host_data["docker_container_id"][:12]
with progress_spinner({"docker commit"}):
image_id = ssh.run_shell_command(
state, host, "docker commit {0}".format(container_id))[1][-1][
7:19] # last line is the image ID, get sha256:[XXXXXXXXXX]...
with progress_spinner({"docker rm"}):
ssh.run_shell_command(
state,
host,
"docker rm -f {0}".format(container_id),
)
logger.info(
"{0}docker build complete, image ID: {1}".format(
host.print_prefix,
click.style(image_id, bold=True),
), )
def connect(state, host, for_fact=None):
if 'docker_container_id' in host.host_data: # user can provide a docker_container_id
return True
with progress_spinner({'docker run'}):
container_id = local.shell(
'docker run -d {0} sleep 10000'.format(host.name),
splitlines=True,
)[-1] # last line is the container ID
host.host_data['docker_container_id'] = container_id
return True
def connect(state, host, for_fact=None):
chroot_directory = host.data.chroot_directory
try:
with progress_spinner({'chroot run'}):
local.shell(
'chroot {0} ls'.format(chroot_directory), splitlines=True,
)
except PyinfraError as e:
raise ConnectError(e.args[0])
host.host_data['chroot_directory'] = chroot_directory
return True
def connect(state, host):
chroot_directory = host.data.chroot_directory
try:
with progress_spinner({"chroot run"}):
local.shell(
"chroot {0} ls".format(chroot_directory),
splitlines=True,
)
except PyinfraError as e:
raise ConnectError(e.args[0])
host.connector_data["chroot_directory"] = chroot_directory
return True
def connect(state, host, for_fact=None):
if 'docker_container_id' in host.host_data: # user can provide a docker_container_id
return True
try:
with progress_spinner({'docker run'}):
container_id = local.shell(
'docker run -d {0} tail -f /dev/null'.format(host.data.docker_image),
splitlines=True,
)[-1] # last line is the container ID
except PyinfraError as e:
raise ConnectError(e.args[0])
host.host_data['docker_container_id'] = container_id
return True
def _run_serial_ops(state):
'''
Run all ops for all servers, one server at a time.
'''
for host in list(state.inventory.iter_active_hosts()):
host_operations = product([host], state.get_op_order())
with progress_spinner(host_operations) as progress:
try:
_run_server_ops(
state, host,
progress=progress,
)
except PyinfraError:
state.fail_hosts({host})
def _run_no_wait_ops(state):
'''
Run all ops for all servers at once.
'''
hosts_operations = product(state.inventory.iter_active_hosts(), state.get_op_order())
with progress_spinner(hosts_operations) as progress:
# Spawn greenlet for each host to run *all* ops
greenlets = [
state.pool.spawn(
_run_server_ops, state, host,
progress=progress,
)
for host in state.inventory.iter_active_hosts()
]
gevent.joinall(greenlets)
def connect(state, host):
if 'docker_container_id' in host.host_data: # user can provide a docker_container_id
host.host_data['docker_container_no_disconnect'] = True
return True
with progress_spinner({'prepare docker container'}):
try:
# Check if the provided @docker/X is an existing container ID
_find_start_docker_container(host.data.docker_image)
except PyinfraError:
container_id = _start_docker_image(host.data.docker_image)
else:
container_id = host.data.docker_image
host.host_data['docker_container_no_disconnect'] = True
host.host_data['docker_container_id'] = container_id
return True
def get_facts(state, *args, **kwargs):
def get_fact_with_context(state, host, *args, **kwargs):
with ctx_state.use(state):
with ctx_host.use(host):
return get_fact(state, host, *args, **kwargs)
greenlet_to_host = {
state.pool.spawn(get_fact_with_context, state, host, *args, **kwargs):
host
for host in state.inventory.iter_active_hosts()
}
results = {}
with progress_spinner(greenlet_to_host.values()) as progress:
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
results[host] = greenlet.get()
progress(host)
return results
def load_deploy_file(state, filename):
state.current_deploy_filename = filename
def load_file(local_host):
with ctx_config.use(state.config.copy()):
with ctx_host.use(local_host):
exec_file(filename)
logger.info(
"{0}{1} {2}".format(
local_host.print_prefix,
click.style("Ready:", "green"),
click.style(filename, bold=True),
),
)
greenlet_to_host = {
state.pool.spawn(load_file, host): host for host in state.inventory.iter_active_hosts()
}
with progress_spinner(greenlet_to_host.values()) as progress:
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
greenlet.get()
progress(host)
def make_names_data(output_key=None):
show_warning()
if not output_key:
raise InventoryError("No Terraform output key!")
with progress_spinner({"fetch terraform output"}):
tf_output_raw = local.shell("terraform output -json")
tf_output = json.loads(tf_output_raw)
tf_output = _flatten_dict(tf_output)
if output_key not in tf_output:
raise InventoryError(f"No Terraform output with key: `{output_key}`")
tf_output_value = tf_output[output_key]
if not isinstance(tf_output_value, list):
raise InventoryError(
f"Invalid Terraform output type, should be list, got `{type(tf_output_value)}`",
)
for ssh_target in tf_output_value:
data = {"ssh_hostname": ssh_target}
yield "@terraform/{0}".format(ssh_target), data, ["@terraform"]
def _run_single_op(state, op_hash):
'''
Run a single operation for all servers. Can be configured to run in serial.
'''
op_meta = state.op_meta[op_hash]
op_types = []
if op_meta['serial']:
op_types.append('serial')
if op_meta['run_once']:
op_types.append('run once')
logger.info('{0} {1} {2}'.format(
click.style(
'--> Starting{0}operation:'.format(
' {0} '.format(', '.join(op_types)) if op_types else ' ', ),
'blue'),
click.style(', '.join(op_meta['names']), bold=True),
tuple(op_meta['args']) if op_meta['args'] else '',
))
failed_hosts = set()
if op_meta['serial']:
with progress_spinner(state.inventory) as progress:
# For each host, run the op
for host in state.inventory:
result = _run_server_op(state, host, op_hash)
progress(host)
if not result:
failed_hosts.add(host)
else:
# Start with the whole inventory in one batch
batches = [state.inventory]
# If parallel set break up the inventory into a series of batches
if op_meta['parallel']:
parallel = op_meta['parallel']
hosts = list(state.inventory)
batches = [
hosts[i:i + parallel] for i in range(0, len(hosts), parallel)
]
for batch in batches:
with progress_spinner(batch) as progress:
# Spawn greenlet for each host
greenlet_to_host = {
state.pool.spawn(_run_server_op, state, host, op_hash):
host
for host in batch
}
# Trigger CLI progress as hosts complete if provided
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
progress(host)
# Get all the results
for greenlet, host in six.iteritems(greenlet_to_host):
if not greenlet.get():
failed_hosts.add(host)
# Now all the batches/hosts are complete, fail any failures
if not op_meta['ignore_errors']:
state.fail_hosts(failed_hosts)
if pyinfra.is_cli:
print()
def get_facts(
state,
name,
args=None,
kwargs=None,
ensure_hosts=None,
apply_failed_hosts=True,
):
'''
Get a single fact for all hosts in the state.
'''
fact = get_fact_class(name)()
if isinstance(fact, ShortFactBase):
return get_short_facts(state,
fact,
args=args,
ensure_hosts=ensure_hosts)
args = args or ()
kwargs = kwargs or {}
if args or kwargs:
# Merges args & kwargs into a single kwargs dictionary
kwargs = getcallargs(fact.command, *args, **kwargs)
logger.debug('Getting fact: {0} {1} (ensure_hosts: {2})'.format(
name,
get_kwargs_str(kwargs),
ensure_hosts,
))
# Apply args or defaults
sudo = state.config.SUDO
sudo_user = state.config.SUDO_USER
su_user = state.config.SU_USER
ignore_errors = state.config.IGNORE_ERRORS
shell_executable = state.config.SHELL
use_sudo_password = state.config.USE_SUDO_PASSWORD
# Facts can override the shell (winrm powershell vs cmd support)
if fact.shell_executable:
shell_executable = fact.shell_executable
# Timeout for operations !== timeout for connect (config.CONNECT_TIMEOUT)
timeout = None
# If inside an operation, fetch global arguments
current_global_kwargs = state.current_op_global_kwargs
if current_global_kwargs:
sudo = current_global_kwargs['sudo']
sudo_user = current_global_kwargs['sudo_user']
use_sudo_password = current_global_kwargs['use_sudo_password']
su_user = current_global_kwargs['su_user']
ignore_errors = current_global_kwargs['ignore_errors']
timeout = current_global_kwargs['timeout']
# Make a hash which keeps facts unique - but usable cross-deploy/threads.
# Locks are used to maintain order.
fact_hash = make_hash(
(name, kwargs, sudo, sudo_user, su_user, ignore_errors))
# Already got this fact? Unlock and return them
current_facts = state.facts.get(fact_hash, {})
if current_facts:
if not ensure_hosts or all(host in current_facts
for host in ensure_hosts):
return current_facts
with FACT_LOCK:
# Add any hosts we must have, whether considered in the inventory or not
# (these hosts might be outside the --limit or current op limit_hosts).
hosts = set(state.inventory.iter_active_hosts())
if ensure_hosts:
hosts.update(ensure_hosts)
# Execute the command for each state inventory in a greenlet
greenlet_to_host = {}
for host in hosts:
if host in current_facts:
continue
# Generate actual arguments by passing strings as jinja2 templates
host_kwargs = {
key: get_arg_value(state, host, arg)
for key, arg in kwargs.items()
}
command = _make_command(fact.command, host_kwargs)
requires_command = _make_command(fact.requires_command,
host_kwargs)
if requires_command:
command = '! command -v {0} > /dev/null || ({1})'.format(
requires_command,
command,
)
greenlet = state.fact_pool.spawn(
host.run_shell_command,
command,
sudo=sudo,
sudo_user=sudo_user,
use_sudo_password=use_sudo_password,
su_user=su_user,
timeout=timeout,
shell_executable=shell_executable,
print_output=state.print_fact_output,
print_input=state.print_fact_input,
return_combined_output=True,
)
greenlet_to_host[greenlet] = host
# Wait for all the commands to execute
progress_prefix = 'fact: {0} {1}'.format(name, get_kwargs_str(kwargs))
with progress_spinner(
greenlet_to_host.values(),
prefix_message=progress_prefix,
) as progress:
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
progress(host)
hostname_facts = {}
failed_hosts = set()
# Collect the facts and any failures
for greenlet, host in six.iteritems(greenlet_to_host):
status = False
stdout = []
try:
status, combined_output_lines = greenlet.get()
except (timeout_error, socket_error, SSHException) as e:
failed_hosts.add(host)
log_host_command_error(
host,
e,
timeout=timeout,
)
stdout, stderr = split_combined_output(combined_output_lines)
data = fact.default()
if status:
if stdout:
data = fact.process(stdout)
elif stderr:
first_line = stderr[0]
if (sudo_user and state.will_add_user(sudo_user)
and re.match(SUDO_REGEX, first_line)):
status = True
if (su_user and state.will_add_user(su_user) and any(
re.match(regex, first_line) for regex in SU_REGEXES)):
status = True
if not status:
failed_hosts.add(host)
if not state.print_fact_output:
print_host_combined_output(host, combined_output_lines)
log_error_or_warning(
host,
ignore_errors,
description=('could not load fact: {0} {1}').format(
name, get_kwargs_str(kwargs)))
hostname_facts[host] = data
log = 'Loaded fact {0} {1}'.format(click.style(name, bold=True),
get_kwargs_str(kwargs))
if state.print_fact_info:
logger.info(log)
else:
logger.debug(log)
# Check we've not failed
if not ignore_errors and apply_failed_hosts:
state.fail_hosts(failed_hosts)
# Assign the facts
state.facts.setdefault(fact_hash, {}).update(hostname_facts)
return state.facts[fact_hash]
def get_facts(state,
name,
args=None,
ensure_hosts=None,
apply_failed_hosts=True):
'''
Get a single fact for all hosts in the state.
'''
# Create an instance of the fact
fact = get_fact_class(name)()
if isinstance(fact, ShortFactBase):
return get_short_facts(state,
fact,
args=args,
ensure_hosts=ensure_hosts)
logger.debug('Getting fact: {0} (ensure_hosts: {1})'.format(
name,
ensure_hosts,
))
args = args or []
# Apply args or defaults
sudo = state.config.SUDO
sudo_user = state.config.SUDO_USER
su_user = state.config.SU_USER
ignore_errors = state.config.IGNORE_ERRORS
shell_executable = state.config.SHELL
use_sudo_password = state.config.USE_SUDO_PASSWORD
# Facts can override the shell (winrm powershell vs cmd support)
if fact.shell_executable:
shell_executable = fact.shell_executable
# Timeout for operations !== timeout for connect (config.CONNECT_TIMEOUT)
timeout = None
# Get the current op meta
current_op_hash = state.current_op_hash
current_op_meta = state.op_meta.get(current_op_hash)
# If inside an operation, fetch config meta
if current_op_meta:
sudo = current_op_meta['sudo']
sudo_user = current_op_meta['sudo_user']
use_sudo_password = current_op_meta['use_sudo_password']
su_user = current_op_meta['su_user']
ignore_errors = current_op_meta['ignore_errors']
timeout = current_op_meta['timeout']
# Make a hash which keeps facts unique - but usable cross-deploy/threads.
# Locks are used to maintain order.
fact_hash = make_hash(
(name, args, sudo, sudo_user, su_user, ignore_errors))
# Already got this fact? Unlock and return them
current_facts = state.facts.get(fact_hash, {})
if current_facts:
if not ensure_hosts or all(host in current_facts
for host in ensure_hosts):
return current_facts
with FACT_LOCK:
# Add any hosts we must have, whether considered in the inventory or not
# (these hosts might be outside the --limit or current op limit_hosts).
hosts = set(state.inventory)
if ensure_hosts:
hosts.update(ensure_hosts)
# Execute the command for each state inventory in a greenlet
greenlet_to_host = {}
for host in hosts:
if host in current_facts:
continue
# if host in state.ready_hosts:
# continue
# Work out the command
command = fact.command
if callable(command):
# Generate actual arguments by passing strings as jinja2 templates
host_args = [get_arg_value(state, host, arg) for arg in args]
command = command(*host_args)
greenlet = state.fact_pool.spawn(
host.run_shell_command,
command,
sudo=sudo,
sudo_user=sudo_user,
use_sudo_password=use_sudo_password,
su_user=su_user,
timeout=timeout,
shell_executable=shell_executable,
print_output=state.print_fact_output,
print_input=state.print_fact_input,
)
greenlet_to_host[greenlet] = host
# Wait for all the commands to execute
progress_prefix = 'fact: {0}'.format(name)
if args:
progress_prefix = '{0}{1}'.format(progress_prefix, args[0])
with progress_spinner(
greenlet_to_host.values(),
prefix_message=progress_prefix,
) as progress:
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
progress(host)
hostname_facts = {}
failed_hosts = set()
# Collect the facts and any failures
for greenlet, host in six.iteritems(greenlet_to_host):
status = False
stdout = []
try:
status, stdout, _ = greenlet.get()
except (timeout_error, socket_error, SSHException) as e:
failed_hosts.add(host)
log_host_command_error(
host,
e,
timeout=timeout,
)
data = fact.default()
if status:
if stdout:
data = fact.process(stdout)
elif not fact.use_default_on_error:
failed_hosts.add(host)
hostname_facts[host] = data
log_name = click.style(name, bold=True)
filtered_args = list(filter(None, args))
if filtered_args:
log = 'Loaded fact {0}: {1}'.format(log_name, tuple(filtered_args))
else:
log = 'Loaded fact {0}'.format(log_name)
if state.print_fact_info:
logger.info(log)
else:
logger.debug(log)
# Check we've not failed
if not ignore_errors and apply_failed_hosts:
state.fail_hosts(failed_hosts)
# Assign the facts
state.facts.setdefault(fact_hash, {}).update(hostname_facts)
return state.facts[fact_hash]
def _run_single_op(state, op_hash):
'''
Run a single operation for all servers. Can be configured to run in serial.
'''
state.trigger_callbacks('operation_start', op_hash)
op_meta = state.get_op_meta(op_hash)
_log_operation_start(op_meta)
failed_hosts = set()
if op_meta['serial']:
with progress_spinner(state.inventory.iter_active_hosts()) as progress:
# For each host, run the op
for host in state.inventory.iter_active_hosts():
result = _run_server_op(state, host, op_hash)
progress(host)
if not result:
failed_hosts.add(host)
else:
# Start with the whole inventory in one batch
batches = [list(state.inventory.iter_active_hosts())]
# If parallel set break up the inventory into a series of batches
if op_meta['parallel']:
parallel = op_meta['parallel']
hosts = list(state.inventory.iter_active_hosts())
batches = [
hosts[i:i + parallel] for i in range(0, len(hosts), parallel)
]
for batch in batches:
with progress_spinner(batch) as progress:
# Spawn greenlet for each host
greenlet_to_host = {
state.pool.spawn(_run_server_op, state, host, op_hash):
host
for host in batch
}
# Trigger CLI progress as hosts complete if provided
for greenlet in gevent.iwait(greenlet_to_host.keys()):
host = greenlet_to_host[greenlet]
progress(host)
# Get all the results
for greenlet, host in six.iteritems(greenlet_to_host):
if not greenlet.get():
failed_hosts.add(host)
# Now all the batches/hosts are complete, fail any failures
state.fail_hosts(failed_hosts)
if pyinfra.is_cli:
click.echo(err=True)
state.trigger_callbacks('operation_end', op_hash)
