def get_mech_config(limit=None): logger.info('Getting Mech config...') if limit and not isinstance(limit, (list, tuple)): limit = [limit] # Note: There is no "--machine-readable" option to 'mech status' with progress_spinner({'mech ls'}) as progress: output = local.shell( 'mech ls', splitlines=True, ) progress('mech ls') targets = [] for line in output: address = '' data = line.split() target = data[0] if len(data) == 5: address = data[1] # Skip anything not in the limit if limit is not None and target not in limit: continue # For each vm that has an address, fetch it's SSH config in a thread if address != '' and address[0].isdigit(): targets.append(target) threads = [] config_queue = Queue() with progress_spinner(targets) as progress: for target in targets: thread = Thread( target=_get_mech_ssh_config, args=(config_queue, progress, target), ) threads.append(thread) thread.start() for thread in threads: thread.join() queue_items = list(config_queue.queue) lines = [] for output in queue_items: lines.extend(output) return lines
def get_vagrant_config(limit=None): logger.info("Getting Vagrant config...") if limit and not isinstance(limit, (list, tuple)): limit = [limit] with progress_spinner({"vagrant status"}) as progress: output = local.shell( "vagrant status --machine-readable", splitlines=True, ) progress("vagrant status") targets = [] for line in output: line = line.strip() _, target, type_, data = line.split(",", 3) # Skip anything not in the limit if limit is not None and target not in limit: continue # For each running container - fetch it's SSH config in a thread - this # is because Vagrant *really* slow to run each command. if type_ == "state" and data == "running": targets.append(target) threads = [] config_queue = Queue() with progress_spinner(targets) as progress: for target in targets: thread = Thread( target=_get_vagrant_ssh_config, args=(config_queue, progress, target), ) threads.append(thread) thread.start() for thread in threads: thread.join() queue_items = list(config_queue.queue) lines = [] for output in queue_items: lines.extend([ln.strip() for ln in output]) return lines
def connect(state, host): if not host.connection: host.connection = ssh.connect(state, host) if "docker_container_id" in host.host_data: # user can provide a docker_container_id return host.connection try: with progress_spinner({"docker run"}): # last line is the container ID status, stdout, stderr = ssh.run_shell_command( state, host, "docker run -d {0} tail -f /dev/null".format( host.data.docker_image), ) if not status: raise IOError("\n".join(stderr)) container_id = stdout[-1] except PyinfraError as e: host.connection = None # fail connection raise ConnectError(e.args[0]) host.host_data["docker_container_id"] = container_id return host.connection
def connect_all(state): ''' Connect to all the configured servers in parallel. Reads/writes state.inventory. Args: state (``pyinfra.api.State`` obj): the state containing an inventory to connect to ''' hosts = [host for host in state.inventory if state.is_host_in_limit(host)] greenlet_to_host = { state.pool.spawn(host.connect, state): host for host in hosts } with progress_spinner(greenlet_to_host.values()) as progress: for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] progress(host) # Get/set the results failed_hosts = set() for greenlet, host in six.iteritems(greenlet_to_host): # Raise any unexpected exception greenlet.get() if host.connection: state.activate_host(host) else: failed_hosts.add(host) # Remove those that failed, triggering FAIL_PERCENT check state.fail_hosts(failed_hosts, activated_count=len(hosts))
def disconnect(state, host): container_id = host.host_data['docker_container_id'][:12] with progress_spinner({'docker commit'}): image_id = local.shell( 'docker commit {0}'.format(container_id), splitlines=True, )[-1][7:19] # last line is the image ID, get sha256:[XXXXXXXXXX]... with progress_spinner({'docker rm'}): local.shell('docker rm -f {0}'.format(container_id), ) logger.info('{0}docker build complete, image ID: {1}'.format( host.print_prefix, click.style(image_id, bold=True), ))
def _get_vagrant_config(limit=None): if limit and not isinstance(limit, list): limit = [limit] with progress_spinner({'vagrant status'}) as progress: output = local.shell( 'vagrant status --machine-readable', splitlines=True, ) progress('vagrant status') targets = [] for line in output: _, target, type_, data = line.split(',', 3) # Skip anything not in the limit if limit is not None and target not in limit: continue # For each running container - fetch it's SSH config in a thread - this # is because Vagrant *really* slow to run each command. if type_ == 'state' and data == 'running': targets.append(target) threads = [] config_queue = Queue() with progress_spinner(targets) as progress: for target in targets: thread = Thread( target=_get_vagrant_ssh_config, args=(config_queue, progress, target), ) threads.append(thread) thread.start() for thread in threads: thread.join() queue_items = list(config_queue.queue) lines = [] for output in queue_items: lines.extend(output) return lines
def disconnect(state, host): container_id = host.host_data["docker_container_id"][:12] with progress_spinner({"docker commit"}): image_id = ssh.run_shell_command( state, host, "docker commit {0}".format(container_id))[1][-1][ 7:19] # last line is the image ID, get sha256:[XXXXXXXXXX]... with progress_spinner({"docker rm"}): ssh.run_shell_command( state, host, "docker rm -f {0}".format(container_id), ) logger.info( "{0}docker build complete, image ID: {1}".format( host.print_prefix, click.style(image_id, bold=True), ), )
def connect(state, host, for_fact=None): if 'docker_container_id' in host.host_data: # user can provide a docker_container_id return True with progress_spinner({'docker run'}): container_id = local.shell( 'docker run -d {0} sleep 10000'.format(host.name), splitlines=True, )[-1] # last line is the container ID host.host_data['docker_container_id'] = container_id return True
def connect(state, host, for_fact=None): chroot_directory = host.data.chroot_directory try: with progress_spinner({'chroot run'}): local.shell( 'chroot {0} ls'.format(chroot_directory), splitlines=True, ) except PyinfraError as e: raise ConnectError(e.args[0]) host.host_data['chroot_directory'] = chroot_directory return True
def connect(state, host): chroot_directory = host.data.chroot_directory try: with progress_spinner({"chroot run"}): local.shell( "chroot {0} ls".format(chroot_directory), splitlines=True, ) except PyinfraError as e: raise ConnectError(e.args[0]) host.connector_data["chroot_directory"] = chroot_directory return True
def connect(state, host, for_fact=None): if 'docker_container_id' in host.host_data: # user can provide a docker_container_id return True try: with progress_spinner({'docker run'}): container_id = local.shell( 'docker run -d {0} tail -f /dev/null'.format(host.data.docker_image), splitlines=True, )[-1] # last line is the container ID except PyinfraError as e: raise ConnectError(e.args[0]) host.host_data['docker_container_id'] = container_id return True
def _run_serial_ops(state): ''' Run all ops for all servers, one server at a time. ''' for host in list(state.inventory.iter_active_hosts()): host_operations = product([host], state.get_op_order()) with progress_spinner(host_operations) as progress: try: _run_server_ops( state, host, progress=progress, ) except PyinfraError: state.fail_hosts({host})
def _run_no_wait_ops(state): ''' Run all ops for all servers at once. ''' hosts_operations = product(state.inventory.iter_active_hosts(), state.get_op_order()) with progress_spinner(hosts_operations) as progress: # Spawn greenlet for each host to run *all* ops greenlets = [ state.pool.spawn( _run_server_ops, state, host, progress=progress, ) for host in state.inventory.iter_active_hosts() ] gevent.joinall(greenlets)
def connect(state, host): if 'docker_container_id' in host.host_data: # user can provide a docker_container_id host.host_data['docker_container_no_disconnect'] = True return True with progress_spinner({'prepare docker container'}): try: # Check if the provided @docker/X is an existing container ID _find_start_docker_container(host.data.docker_image) except PyinfraError: container_id = _start_docker_image(host.data.docker_image) else: container_id = host.data.docker_image host.host_data['docker_container_no_disconnect'] = True host.host_data['docker_container_id'] = container_id return True
def get_facts(state, *args, **kwargs): def get_fact_with_context(state, host, *args, **kwargs): with ctx_state.use(state): with ctx_host.use(host): return get_fact(state, host, *args, **kwargs) greenlet_to_host = { state.pool.spawn(get_fact_with_context, state, host, *args, **kwargs): host for host in state.inventory.iter_active_hosts() } results = {} with progress_spinner(greenlet_to_host.values()) as progress: for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] results[host] = greenlet.get() progress(host) return results
def load_deploy_file(state, filename): state.current_deploy_filename = filename def load_file(local_host): with ctx_config.use(state.config.copy()): with ctx_host.use(local_host): exec_file(filename) logger.info( "{0}{1} {2}".format( local_host.print_prefix, click.style("Ready:", "green"), click.style(filename, bold=True), ), ) greenlet_to_host = { state.pool.spawn(load_file, host): host for host in state.inventory.iter_active_hosts() } with progress_spinner(greenlet_to_host.values()) as progress: for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] greenlet.get() progress(host)
def make_names_data(output_key=None): show_warning() if not output_key: raise InventoryError("No Terraform output key!") with progress_spinner({"fetch terraform output"}): tf_output_raw = local.shell("terraform output -json") tf_output = json.loads(tf_output_raw) tf_output = _flatten_dict(tf_output) if output_key not in tf_output: raise InventoryError(f"No Terraform output with key: `{output_key}`") tf_output_value = tf_output[output_key] if not isinstance(tf_output_value, list): raise InventoryError( f"Invalid Terraform output type, should be list, got `{type(tf_output_value)}`", ) for ssh_target in tf_output_value: data = {"ssh_hostname": ssh_target} yield "@terraform/{0}".format(ssh_target), data, ["@terraform"]
def _run_single_op(state, op_hash): ''' Run a single operation for all servers. Can be configured to run in serial. ''' op_meta = state.op_meta[op_hash] op_types = [] if op_meta['serial']: op_types.append('serial') if op_meta['run_once']: op_types.append('run once') logger.info('{0} {1} {2}'.format( click.style( '--> Starting{0}operation:'.format( ' {0} '.format(', '.join(op_types)) if op_types else ' ', ), 'blue'), click.style(', '.join(op_meta['names']), bold=True), tuple(op_meta['args']) if op_meta['args'] else '', )) failed_hosts = set() if op_meta['serial']: with progress_spinner(state.inventory) as progress: # For each host, run the op for host in state.inventory: result = _run_server_op(state, host, op_hash) progress(host) if not result: failed_hosts.add(host) else: # Start with the whole inventory in one batch batches = [state.inventory] # If parallel set break up the inventory into a series of batches if op_meta['parallel']: parallel = op_meta['parallel'] hosts = list(state.inventory) batches = [ hosts[i:i + parallel] for i in range(0, len(hosts), parallel) ] for batch in batches: with progress_spinner(batch) as progress: # Spawn greenlet for each host greenlet_to_host = { state.pool.spawn(_run_server_op, state, host, op_hash): host for host in batch } # Trigger CLI progress as hosts complete if provided for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] progress(host) # Get all the results for greenlet, host in six.iteritems(greenlet_to_host): if not greenlet.get(): failed_hosts.add(host) # Now all the batches/hosts are complete, fail any failures if not op_meta['ignore_errors']: state.fail_hosts(failed_hosts) if pyinfra.is_cli: print()
def get_facts( state, name, args=None, kwargs=None, ensure_hosts=None, apply_failed_hosts=True, ): ''' Get a single fact for all hosts in the state. ''' fact = get_fact_class(name)() if isinstance(fact, ShortFactBase): return get_short_facts(state, fact, args=args, ensure_hosts=ensure_hosts) args = args or () kwargs = kwargs or {} if args or kwargs: # Merges args & kwargs into a single kwargs dictionary kwargs = getcallargs(fact.command, *args, **kwargs) logger.debug('Getting fact: {0} {1} (ensure_hosts: {2})'.format( name, get_kwargs_str(kwargs), ensure_hosts, )) # Apply args or defaults sudo = state.config.SUDO sudo_user = state.config.SUDO_USER su_user = state.config.SU_USER ignore_errors = state.config.IGNORE_ERRORS shell_executable = state.config.SHELL use_sudo_password = state.config.USE_SUDO_PASSWORD # Facts can override the shell (winrm powershell vs cmd support) if fact.shell_executable: shell_executable = fact.shell_executable # Timeout for operations !== timeout for connect (config.CONNECT_TIMEOUT) timeout = None # If inside an operation, fetch global arguments current_global_kwargs = state.current_op_global_kwargs if current_global_kwargs: sudo = current_global_kwargs['sudo'] sudo_user = current_global_kwargs['sudo_user'] use_sudo_password = current_global_kwargs['use_sudo_password'] su_user = current_global_kwargs['su_user'] ignore_errors = current_global_kwargs['ignore_errors'] timeout = current_global_kwargs['timeout'] # Make a hash which keeps facts unique - but usable cross-deploy/threads. # Locks are used to maintain order. fact_hash = make_hash( (name, kwargs, sudo, sudo_user, su_user, ignore_errors)) # Already got this fact? Unlock and return them current_facts = state.facts.get(fact_hash, {}) if current_facts: if not ensure_hosts or all(host in current_facts for host in ensure_hosts): return current_facts with FACT_LOCK: # Add any hosts we must have, whether considered in the inventory or not # (these hosts might be outside the --limit or current op limit_hosts). hosts = set(state.inventory.iter_active_hosts()) if ensure_hosts: hosts.update(ensure_hosts) # Execute the command for each state inventory in a greenlet greenlet_to_host = {} for host in hosts: if host in current_facts: continue # Generate actual arguments by passing strings as jinja2 templates host_kwargs = { key: get_arg_value(state, host, arg) for key, arg in kwargs.items() } command = _make_command(fact.command, host_kwargs) requires_command = _make_command(fact.requires_command, host_kwargs) if requires_command: command = '! command -v {0} > /dev/null || ({1})'.format( requires_command, command, ) greenlet = state.fact_pool.spawn( host.run_shell_command, command, sudo=sudo, sudo_user=sudo_user, use_sudo_password=use_sudo_password, su_user=su_user, timeout=timeout, shell_executable=shell_executable, print_output=state.print_fact_output, print_input=state.print_fact_input, return_combined_output=True, ) greenlet_to_host[greenlet] = host # Wait for all the commands to execute progress_prefix = 'fact: {0} {1}'.format(name, get_kwargs_str(kwargs)) with progress_spinner( greenlet_to_host.values(), prefix_message=progress_prefix, ) as progress: for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] progress(host) hostname_facts = {} failed_hosts = set() # Collect the facts and any failures for greenlet, host in six.iteritems(greenlet_to_host): status = False stdout = [] try: status, combined_output_lines = greenlet.get() except (timeout_error, socket_error, SSHException) as e: failed_hosts.add(host) log_host_command_error( host, e, timeout=timeout, ) stdout, stderr = split_combined_output(combined_output_lines) data = fact.default() if status: if stdout: data = fact.process(stdout) elif stderr: first_line = stderr[0] if (sudo_user and state.will_add_user(sudo_user) and re.match(SUDO_REGEX, first_line)): status = True if (su_user and state.will_add_user(su_user) and any( re.match(regex, first_line) for regex in SU_REGEXES)): status = True if not status: failed_hosts.add(host) if not state.print_fact_output: print_host_combined_output(host, combined_output_lines) log_error_or_warning( host, ignore_errors, description=('could not load fact: {0} {1}').format( name, get_kwargs_str(kwargs))) hostname_facts[host] = data log = 'Loaded fact {0} {1}'.format(click.style(name, bold=True), get_kwargs_str(kwargs)) if state.print_fact_info: logger.info(log) else: logger.debug(log) # Check we've not failed if not ignore_errors and apply_failed_hosts: state.fail_hosts(failed_hosts) # Assign the facts state.facts.setdefault(fact_hash, {}).update(hostname_facts) return state.facts[fact_hash]
def get_facts(state, name, args=None, ensure_hosts=None, apply_failed_hosts=True): ''' Get a single fact for all hosts in the state. ''' # Create an instance of the fact fact = get_fact_class(name)() if isinstance(fact, ShortFactBase): return get_short_facts(state, fact, args=args, ensure_hosts=ensure_hosts) logger.debug('Getting fact: {0} (ensure_hosts: {1})'.format( name, ensure_hosts, )) args = args or [] # Apply args or defaults sudo = state.config.SUDO sudo_user = state.config.SUDO_USER su_user = state.config.SU_USER ignore_errors = state.config.IGNORE_ERRORS shell_executable = state.config.SHELL use_sudo_password = state.config.USE_SUDO_PASSWORD # Facts can override the shell (winrm powershell vs cmd support) if fact.shell_executable: shell_executable = fact.shell_executable # Timeout for operations !== timeout for connect (config.CONNECT_TIMEOUT) timeout = None # Get the current op meta current_op_hash = state.current_op_hash current_op_meta = state.op_meta.get(current_op_hash) # If inside an operation, fetch config meta if current_op_meta: sudo = current_op_meta['sudo'] sudo_user = current_op_meta['sudo_user'] use_sudo_password = current_op_meta['use_sudo_password'] su_user = current_op_meta['su_user'] ignore_errors = current_op_meta['ignore_errors'] timeout = current_op_meta['timeout'] # Make a hash which keeps facts unique - but usable cross-deploy/threads. # Locks are used to maintain order. fact_hash = make_hash( (name, args, sudo, sudo_user, su_user, ignore_errors)) # Already got this fact? Unlock and return them current_facts = state.facts.get(fact_hash, {}) if current_facts: if not ensure_hosts or all(host in current_facts for host in ensure_hosts): return current_facts with FACT_LOCK: # Add any hosts we must have, whether considered in the inventory or not # (these hosts might be outside the --limit or current op limit_hosts). hosts = set(state.inventory) if ensure_hosts: hosts.update(ensure_hosts) # Execute the command for each state inventory in a greenlet greenlet_to_host = {} for host in hosts: if host in current_facts: continue # if host in state.ready_hosts: # continue # Work out the command command = fact.command if callable(command): # Generate actual arguments by passing strings as jinja2 templates host_args = [get_arg_value(state, host, arg) for arg in args] command = command(*host_args) greenlet = state.fact_pool.spawn( host.run_shell_command, command, sudo=sudo, sudo_user=sudo_user, use_sudo_password=use_sudo_password, su_user=su_user, timeout=timeout, shell_executable=shell_executable, print_output=state.print_fact_output, print_input=state.print_fact_input, ) greenlet_to_host[greenlet] = host # Wait for all the commands to execute progress_prefix = 'fact: {0}'.format(name) if args: progress_prefix = '{0}{1}'.format(progress_prefix, args[0]) with progress_spinner( greenlet_to_host.values(), prefix_message=progress_prefix, ) as progress: for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] progress(host) hostname_facts = {} failed_hosts = set() # Collect the facts and any failures for greenlet, host in six.iteritems(greenlet_to_host): status = False stdout = [] try: status, stdout, _ = greenlet.get() except (timeout_error, socket_error, SSHException) as e: failed_hosts.add(host) log_host_command_error( host, e, timeout=timeout, ) data = fact.default() if status: if stdout: data = fact.process(stdout) elif not fact.use_default_on_error: failed_hosts.add(host) hostname_facts[host] = data log_name = click.style(name, bold=True) filtered_args = list(filter(None, args)) if filtered_args: log = 'Loaded fact {0}: {1}'.format(log_name, tuple(filtered_args)) else: log = 'Loaded fact {0}'.format(log_name) if state.print_fact_info: logger.info(log) else: logger.debug(log) # Check we've not failed if not ignore_errors and apply_failed_hosts: state.fail_hosts(failed_hosts) # Assign the facts state.facts.setdefault(fact_hash, {}).update(hostname_facts) return state.facts[fact_hash]
def _run_single_op(state, op_hash): ''' Run a single operation for all servers. Can be configured to run in serial. ''' state.trigger_callbacks('operation_start', op_hash) op_meta = state.get_op_meta(op_hash) _log_operation_start(op_meta) failed_hosts = set() if op_meta['serial']: with progress_spinner(state.inventory.iter_active_hosts()) as progress: # For each host, run the op for host in state.inventory.iter_active_hosts(): result = _run_server_op(state, host, op_hash) progress(host) if not result: failed_hosts.add(host) else: # Start with the whole inventory in one batch batches = [list(state.inventory.iter_active_hosts())] # If parallel set break up the inventory into a series of batches if op_meta['parallel']: parallel = op_meta['parallel'] hosts = list(state.inventory.iter_active_hosts()) batches = [ hosts[i:i + parallel] for i in range(0, len(hosts), parallel) ] for batch in batches: with progress_spinner(batch) as progress: # Spawn greenlet for each host greenlet_to_host = { state.pool.spawn(_run_server_op, state, host, op_hash): host for host in batch } # Trigger CLI progress as hosts complete if provided for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] progress(host) # Get all the results for greenlet, host in six.iteritems(greenlet_to_host): if not greenlet.get(): failed_hosts.add(host) # Now all the batches/hosts are complete, fail any failures state.fail_hosts(failed_hosts) if pyinfra.is_cli: click.echo(err=True) state.trigger_callbacks('operation_end', op_hash)