def create_fact(self, fact_cls, data=None, kwargs=None): try: fact = self.get_fact(fact_cls) except KeyError: fact_key = self._get_fact_key(fact_cls) fact = self.fact[fact_key] = {} if kwargs: fact[_sort_kwargs_str(get_kwargs_str(kwargs))] = data else: fact_key = self._get_fact_key(fact_cls) self.fact[fact_key] = data
def delete_fact(self, fact_cls, kwargs=None): try: fact = self.get_fact(fact_cls) except KeyError: return ordered_kwargs = _sort_kwargs_str(get_kwargs_str(kwargs)) for key in fact.keys(): ordered_key = _sort_kwargs_str(key) if ordered_key == ordered_kwargs: fact.pop(key) break
def get_fact(self, fact_cls, **kwargs): fact_key = self._get_fact_key(fact_cls) fact = getattr(self.fact, fact_key, None) if fact is None: raise KeyError("Missing test fact data: {0}".format(fact_key)) if kwargs: self._check_fact_args(fact_cls, kwargs) fact_ordered_keys = { _sort_kwargs_str(key): value for key, value in fact.items() } kwargs_str = _sort_kwargs_str(get_kwargs_str(kwargs)) if kwargs_str not in fact: logger.info( "Possible missing fact key: {0}".format(kwargs_str)) return fact_ordered_keys.get(kwargs_str) return fact
def _get_fact( state, host, cls, args=None, kwargs=None, ensure_hosts=None, apply_failed_hosts=True, fact_hash=None, ): fact = cls() name = fact.name args = args or () kwargs = kwargs or {} # Get the defaults *and* overrides by popping from kwargs, executor kwargs passed # into get_fact override everything else (applied below). override_kwargs, override_kwarg_keys = pop_global_arguments( kwargs, state=state, host=host, keys_to_check=get_executor_kwarg_keys(), ) executor_kwargs = _get_executor_kwargs( state, host, override_kwargs=override_kwargs, override_kwarg_keys=override_kwarg_keys, ) if args or kwargs: # Merges args & kwargs into a single kwargs dictionary kwargs = getcallargs(fact.command, *args, **kwargs) kwargs_str = get_kwargs_str(kwargs) logger.debug( "Getting fact: %s (%s) (ensure_hosts: %r)", name, kwargs_str, ensure_hosts, ) if not host.connected: host.connect( reason=f"to load fact: {name} ({kwargs_str})", raise_exceptions=True, ) ignore_errors = (host.current_op_global_kwargs or {}).get( "ignore_errors", state.config.IGNORE_ERRORS, ) # Facts can override the shell (winrm powershell vs cmd support) if fact.shell_executable: executor_kwargs["shell_executable"] = fact.shell_executable command = _make_command(fact.command, kwargs) requires_command = _make_command(fact.requires_command, kwargs) if requires_command: command = StringCommand( # Command doesn't exist, return 0 *or* run & return fact command "!", "command", "-v", requires_command, ">/dev/null", "||", command, ) status = False stdout = [] combined_output_lines = [] try: status, combined_output_lines = host.run_shell_command( command, print_output=state.print_fact_output, print_input=state.print_fact_input, return_combined_output=True, **executor_kwargs, ) except (timeout_error, socket_error, SSHException) as e: log_host_command_error( host, e, timeout=executor_kwargs["timeout"], ) stdout, stderr = split_combined_output(combined_output_lines) data = fact.default() if status: if stdout: data = fact.process(stdout) elif stderr: # If we have error output and that error is sudo or su stating the user # does not exist, do not fail but instead return the default fact value. # This allows for users that don't currently but may be created during # other operations. first_line = stderr[0] if executor_kwargs["sudo_user"] and re.match(SUDO_REGEX, first_line): status = True if executor_kwargs["su_user"] and any( re.match(regex, first_line) for regex in SU_REGEXES): status = True if status: log_message = "{0}{1}".format( host.print_prefix, "Loaded fact {0}{1}".format( click.style(name, bold=True), f" ({get_kwargs_str(kwargs)})" if kwargs else "", ), ) if state.print_fact_info: logger.info(log_message) else: logger.debug(log_message) else: if not state.print_fact_output: print_host_combined_output(host, combined_output_lines) log_error_or_warning( host, ignore_errors, description=("could not load fact: {0} {1}").format( name, get_kwargs_str(kwargs)), ) # Check we've not failed if not status and not ignore_errors and apply_failed_hosts: state.fail_hosts({host}) if fact_hash: host.facts[fact_hash] = data return data
def get_facts( state, name, args=None, kwargs=None, ensure_hosts=None, apply_failed_hosts=True, ): ''' Get a single fact for all hosts in the state. ''' fact = get_fact_class(name)() if isinstance(fact, ShortFactBase): return get_short_facts(state, fact, args=args, ensure_hosts=ensure_hosts) args = args or () kwargs = kwargs or {} if args or kwargs: # Merges args & kwargs into a single kwargs dictionary kwargs = getcallargs(fact.command, *args, **kwargs) logger.debug('Getting fact: {0} {1} (ensure_hosts: {2})'.format( name, get_kwargs_str(kwargs), ensure_hosts, )) # Apply args or defaults sudo = state.config.SUDO sudo_user = state.config.SUDO_USER su_user = state.config.SU_USER ignore_errors = state.config.IGNORE_ERRORS shell_executable = state.config.SHELL use_sudo_password = state.config.USE_SUDO_PASSWORD # Facts can override the shell (winrm powershell vs cmd support) if fact.shell_executable: shell_executable = fact.shell_executable # Timeout for operations !== timeout for connect (config.CONNECT_TIMEOUT) timeout = None # If inside an operation, fetch global arguments current_global_kwargs = state.current_op_global_kwargs if current_global_kwargs: sudo = current_global_kwargs['sudo'] sudo_user = current_global_kwargs['sudo_user'] use_sudo_password = current_global_kwargs['use_sudo_password'] su_user = current_global_kwargs['su_user'] ignore_errors = current_global_kwargs['ignore_errors'] timeout = current_global_kwargs['timeout'] # Make a hash which keeps facts unique - but usable cross-deploy/threads. # Locks are used to maintain order. fact_hash = make_hash( (name, kwargs, sudo, sudo_user, su_user, ignore_errors)) # Already got this fact? Unlock and return them current_facts = state.facts.get(fact_hash, {}) if current_facts: if not ensure_hosts or all(host in current_facts for host in ensure_hosts): return current_facts with FACT_LOCK: # Add any hosts we must have, whether considered in the inventory or not # (these hosts might be outside the --limit or current op limit_hosts). hosts = set(state.inventory.iter_active_hosts()) if ensure_hosts: hosts.update(ensure_hosts) # Execute the command for each state inventory in a greenlet greenlet_to_host = {} for host in hosts: if host in current_facts: continue # Generate actual arguments by passing strings as jinja2 templates host_kwargs = { key: get_arg_value(state, host, arg) for key, arg in kwargs.items() } command = _make_command(fact.command, host_kwargs) requires_command = _make_command(fact.requires_command, host_kwargs) if requires_command: command = '! command -v {0} > /dev/null || ({1})'.format( requires_command, command, ) greenlet = state.fact_pool.spawn( host.run_shell_command, command, sudo=sudo, sudo_user=sudo_user, use_sudo_password=use_sudo_password, su_user=su_user, timeout=timeout, shell_executable=shell_executable, print_output=state.print_fact_output, print_input=state.print_fact_input, return_combined_output=True, ) greenlet_to_host[greenlet] = host # Wait for all the commands to execute progress_prefix = 'fact: {0} {1}'.format(name, get_kwargs_str(kwargs)) with progress_spinner( greenlet_to_host.values(), prefix_message=progress_prefix, ) as progress: for greenlet in gevent.iwait(greenlet_to_host.keys()): host = greenlet_to_host[greenlet] progress(host) hostname_facts = {} failed_hosts = set() # Collect the facts and any failures for greenlet, host in six.iteritems(greenlet_to_host): status = False stdout = [] try: status, combined_output_lines = greenlet.get() except (timeout_error, socket_error, SSHException) as e: failed_hosts.add(host) log_host_command_error( host, e, timeout=timeout, ) stdout, stderr = split_combined_output(combined_output_lines) data = fact.default() if status: if stdout: data = fact.process(stdout) elif stderr: first_line = stderr[0] if (sudo_user and state.will_add_user(sudo_user) and re.match(SUDO_REGEX, first_line)): status = True if (su_user and state.will_add_user(su_user) and any( re.match(regex, first_line) for regex in SU_REGEXES)): status = True if not status: failed_hosts.add(host) if not state.print_fact_output: print_host_combined_output(host, combined_output_lines) log_error_or_warning( host, ignore_errors, description=('could not load fact: {0} {1}').format( name, get_kwargs_str(kwargs))) hostname_facts[host] = data log = 'Loaded fact {0} {1}'.format(click.style(name, bold=True), get_kwargs_str(kwargs)) if state.print_fact_info: logger.info(log) else: logger.debug(log) # Check we've not failed if not ignore_errors and apply_failed_hosts: state.fail_hosts(failed_hosts) # Assign the facts state.facts.setdefault(fact_hash, {}).update(hostname_facts) return state.facts[fact_hash]
def _main( inventory, operations, verbosity, ssh_user, ssh_port, ssh_key, ssh_key_password, ssh_password, winrm_username, winrm_password, winrm_port, winrm_transport, shell_executable, sudo, sudo_user, use_sudo_password, su_user, parallel, fail_percent, dry, limit, no_wait, serial, quiet, debug, debug_data, debug_facts, debug_operations, facts=None, print_operations=None, support=None, ): if not debug and not sys.warnoptions: warnings.simplefilter('ignore') # Setup logging log_level = logging.INFO if debug: log_level = logging.DEBUG elif quiet: log_level = logging.WARNING setup_logging(log_level) # Bootstrap any virtualenv init_virtualenv() deploy_dir = getcwd() potential_deploy_dirs = [] # This is the most common case: we have a deploy file so use it's # pathname - we only look at the first file as we can't have multiple # deploy directories. if operations[0].endswith('.py'): deploy_file_dir, _ = path.split(operations[0]) above_deploy_file_dir, _ = path.split(deploy_file_dir) deploy_dir = deploy_file_dir potential_deploy_dirs.extend(( deploy_file_dir, above_deploy_file_dir, )) # If we have a valid inventory, look in it's path and it's parent for # group_data or config.py to indicate deploy_dir (--fact, --run). if inventory.endswith('.py') and path.isfile(inventory): inventory_dir, _ = path.split(inventory) above_inventory_dir, _ = path.split(inventory_dir) potential_deploy_dirs.extend(( inventory_dir, above_inventory_dir, )) for potential_deploy_dir in potential_deploy_dirs: logger.debug('Checking potential directory: {0}'.format( potential_deploy_dir, )) if any(( path.isdir(path.join(potential_deploy_dir, 'group_data')), path.isfile(path.join(potential_deploy_dir, 'config.py')), )): logger.debug( 'Setting directory to: {0}'.format(potential_deploy_dir)) deploy_dir = potential_deploy_dir break # Make sure imported files (deploy.py/etc) behave as if imported from the cwd sys.path.append(deploy_dir) # Create an empty/unitialised state object state = State() # Set the deploy directory state.deploy_dir = deploy_dir pseudo_state.set(state) if verbosity > 0: state.print_fact_info = True state.print_noop_info = True if verbosity > 1: state.print_input = state.print_fact_input = True if verbosity > 2: state.print_output = state.print_fact_output = True if not quiet: click.echo('--> Loading config...', err=True) # Load up any config.py from the filesystem config = load_config(deploy_dir) # Make a copy before we overwrite original_operations = operations # Debug (print) inventory + group data if operations[0] == 'debug-inventory': command = 'debug-inventory' # Get all non-arg facts elif operations[0] == 'all-facts': click.echo(click.style( 'all-facts is deprecated and will be removed in the future.', 'yellow', ), err=True) command = 'fact' fact_ops = [] for fact_name in get_fact_names(): fact_class = get_fact_class(fact_name) if (not issubclass(fact_class, ShortFactBase) and not callable(fact_class.command)): fact_ops.append((fact_class, None, None)) operations = fact_ops # Get one or more facts elif operations[0] == 'fact': command = 'fact' operations = get_facts_and_args(operations[1:]) # Execute a raw command with server.shell elif operations[0] == 'exec': command = 'exec' operations = operations[1:] # Execute one or more deploy files elif all(cmd.endswith('.py') for cmd in operations): command = 'deploy' operations = operations[0:] for file in operations: if not path.exists(file): raise CliError('No deploy file: {0}'.format(file)) # Operation w/optional args (<module>.<op> ARG1 ARG2 ...) elif len(operations[0].split('.')) == 2: command = 'op' operations = get_operation_and_args(operations) else: raise CliError('''Invalid operations: {0} Operation usage: pyinfra INVENTORY deploy_web.py [deploy_db.py]... pyinfra INVENTORY server.user pyinfra home=/home/pyinfra pyinfra INVENTORY exec -- echo "hello world" pyinfra INVENTORY fact os [users]...'''.format(operations)) # Load any hooks/config from the deploy file if command == 'deploy': load_deploy_config(operations[0], config) # Arg based config overrides if sudo: config.SUDO = True if sudo_user: config.SUDO_USER = sudo_user if use_sudo_password: config.USE_SUDO_PASSWORD = use_sudo_password if su_user: config.SU_USER = su_user if parallel: config.PARALLEL = parallel if shell_executable: config.SHELL = shell_executable if fail_percent is not None: config.FAIL_PERCENT = fail_percent if not quiet: click.echo('--> Loading inventory...', err=True) # Load up the inventory from the filesystem inventory, inventory_group = make_inventory( inventory, deploy_dir=deploy_dir, ssh_port=ssh_port, ssh_user=ssh_user, ssh_key=ssh_key, ssh_key_password=ssh_key_password, ssh_password=ssh_password, winrm_username=winrm_username, winrm_password=winrm_password, winrm_port=winrm_port, winrm_transport=winrm_transport, ) # Attach to pseudo inventory pseudo_inventory.set(inventory) # Now that we have inventory, apply --limit config override initial_limit = None if limit: all_limit_hosts = [] for limiter in limit: try: limit_hosts = inventory.get_group(limiter) except NoGroupError: limits = limiter.split(',') if len(limits) > 1: logger.warning(( 'Specifying comma separated --limit values is deprecated, ' 'please use multiple --limit options.')) limit_hosts = [ host for host in inventory if any( fnmatch(host.name, match) for match in limits) ] all_limit_hosts.extend(limit_hosts) initial_limit = list(set(all_limit_hosts)) # Initialise the state state.init(inventory, config, initial_limit=initial_limit) # If --debug-data dump & exit if command == 'debug-inventory' or debug_data: if debug_data: logger.warning( ('--debug-data is deprecated, ' 'please use `pyinfra INVENTORY debug-inventory` instead.')) print_inventory(state) _exit() # Connect to all the servers if not quiet: click.echo(err=True) click.echo('--> Connecting to hosts...', err=True) connect_all(state) # Just getting a fact? # if command == 'fact': if not quiet: click.echo(err=True) click.echo('--> Gathering facts...', err=True) state.print_fact_info = True fact_data = {} for i, command in enumerate(operations): fact_cls, args, kwargs = command fact_key = fact_cls.name if args or kwargs: fact_key = '{0}{1} {2}'.format( fact_cls.name, args or '', get_kwargs_str(kwargs) if kwargs else '', ) try: fact_data[fact_key] = get_facts( state, fact_cls, args=args, kwargs=kwargs, apply_failed_hosts=False, ) except PyinfraError: pass print_facts(fact_data) _exit() # Prepare the deploy! # # Execute a raw command with server.shell if command == 'exec': # Print the output of the command state.print_output = True add_op( state, server.shell, ' '.join(operations), _allow_cli_mode=True, ) # Deploy files(s) elif command == 'deploy': if not quiet: click.echo(err=True) click.echo('--> Preparing operations...', err=True) # Number of "steps" to make = number of files * number of hosts for i, filename in enumerate(operations): logger.info('Loading: {0}'.format(click.style(filename, bold=True))) load_deploy_file(state, filename) # Operation w/optional args elif command == 'op': if not quiet: click.echo(err=True) click.echo('--> Preparing operation...', err=True) op, args = operations args, kwargs = args kwargs['_allow_cli_mode'] = True def print_host_ready(host): logger.info('{0}{1} {2}'.format( host.print_prefix, click.style('Ready:', 'green'), click.style(original_operations[0], bold=True), )) kwargs['_after_host_callback'] = print_host_ready add_op(state, op, *args, **kwargs) # Always show meta output if not quiet: click.echo(err=True) click.echo('--> Proposed changes:', err=True) print_meta(state) # If --debug-facts or --debug-operations, print and exit if debug_facts or debug_operations: if debug_facts: print_state_facts(state) if debug_operations: print_state_operations(state) _exit() # Run the operations we generated with the deploy file if dry: _exit() if not quiet: click.echo(err=True) if not quiet: click.echo('--> Beginning operation run...', err=True) run_ops(state, serial=serial, no_wait=no_wait) if not quiet: click.echo('--> Results:', err=True) print_results(state) _exit()
def _main( inventory, operations, verbosity, chdir, ssh_user, ssh_port, ssh_key, ssh_key_password, ssh_password, winrm_username, winrm_password, winrm_port, winrm_transport, shell_executable, sudo, sudo_user, use_sudo_password, su_user, parallel, fail_percent, data, group_data, config_filename, dry, limit, no_wait, serial, quiet, debug, debug_facts, debug_operations, support=None, ): # Setup working directory # if chdir: os_chdir(chdir) # Setup logging # if not debug and not sys.warnoptions: warnings.simplefilter("ignore") log_level = logging.INFO if debug: log_level = logging.DEBUG elif quiet: log_level = logging.WARNING setup_logging(log_level) # Bootstrap any virtualenv init_virtualenv() # Check operations are valid and setup command # # Make a copy before we overwrite original_operations = operations # Debug (print) inventory + group data if operations[0] == "debug-inventory": command = "debug-inventory" # Get one or more facts elif operations[0] == "fact": command = "fact" operations = get_facts_and_args(operations[1:]) # Execute a raw command with server.shell elif operations[0] == "exec": command = "exec" operations = operations[1:] # Execute one or more deploy files elif all(cmd.endswith(".py") for cmd in operations): command = "deploy" filenames = [] for filename in operations[0:]: if path.exists(filename): filenames.append(filename) continue if chdir and filename.startswith(chdir): correct_filename = path.relpath(filename, chdir) logger.warning( ( "Fixing deploy filename under `--chdir` argument: " f"{filename} -> {correct_filename}" ), ) filenames.append(correct_filename) continue raise CliError( "No deploy file: {0}".format( path.join(chdir, filename) if chdir else filename, ), ) operations = filenames # Operation w/optional args (<module>.<op> ARG1 ARG2 ...) elif len(operations[0].split(".")) == 2: command = "op" operations = get_operation_and_args(operations) else: raise CliError( """Invalid operations: {0} Operation usage: pyinfra INVENTORY deploy_web.py [deploy_db.py]... pyinfra INVENTORY server.user pyinfra home=/home/pyinfra pyinfra INVENTORY exec -- echo "hello world" pyinfra INVENTORY fact os [users]...""".format( operations, ), ) # Setup state, config & inventory # cwd = getcwd() if cwd not in sys.path: # ensure cwd is present in sys.path sys.path.append(cwd) state = State() state.cwd = cwd ctx_state.set(state) if verbosity > 0: state.print_fact_info = True state.print_noop_info = True if verbosity > 1: state.print_input = state.print_fact_input = True if verbosity > 2: state.print_output = state.print_fact_output = True if not quiet: click.echo("--> Loading config...", err=True) config = Config() ctx_config.set(config) # Load up any config.py from the filesystem config_filename = path.join(state.cwd, config_filename) if path.exists(config_filename): exec_file(config_filename) # Lock the current config, this allows us to restore this version after # executing deploy files that may alter them. config.lock_current_state() # Arg based config overrides if sudo: config.SUDO = True if sudo_user: config.SUDO_USER = sudo_user if use_sudo_password: config.USE_SUDO_PASSWORD = use_sudo_password if su_user: config.SU_USER = su_user if parallel: config.PARALLEL = parallel if shell_executable: config.SHELL = None if shell_executable in ("None", "null") else shell_executable if fail_percent is not None: config.FAIL_PERCENT = fail_percent if not quiet: click.echo("--> Loading inventory...", err=True) override_data = {} for arg in data: key, value = arg.split("=", 1) override_data[key] = value override_data = {key: parse_cli_arg(value) for key, value in override_data.items()} for key, value in ( ("ssh_user", ssh_user), ("ssh_key", ssh_key), ("ssh_key_password", ssh_key_password), ("ssh_port", ssh_port), ("ssh_password", ssh_password), ("winrm_username", winrm_username), ("winrm_password", winrm_password), ("winrm_port", winrm_port), ("winrm_transport", winrm_transport), ): if value: override_data[key] = value # Load up the inventory from the filesystem inventory, inventory_group = make_inventory( inventory, cwd=state.cwd, override_data=override_data, group_data_directories=group_data, ) ctx_inventory.set(inventory) # Now that we have inventory, apply --limit config override initial_limit = None if limit: all_limit_hosts = [] for limiter in limit: try: limit_hosts = inventory.get_group(limiter) except NoGroupError: limit_hosts = [host for host in inventory if fnmatch(host.name, limiter)] if not limit_hosts: logger.warning("No host matches found for --limit pattern: {0}".format(limiter)) all_limit_hosts.extend(limit_hosts) initial_limit = list(set(all_limit_hosts)) # Initialise the state state.init(inventory, config, initial_limit=initial_limit) if command == "debug-inventory": print_inventory(state) _exit() # Connect to the hosts & start handling the user commands # if not quiet: click.echo(err=True) click.echo("--> Connecting to hosts...", err=True) connect_all(state) if command == "fact": if not quiet: click.echo(err=True) click.echo("--> Gathering facts...", err=True) state.print_fact_info = True fact_data = {} for i, command in enumerate(operations): fact_cls, args, kwargs = command fact_key = fact_cls.name if args or kwargs: fact_key = "{0}{1}{2}".format( fact_cls.name, args or "", " ({0})".format(get_kwargs_str(kwargs)) if kwargs else "", ) try: fact_data[fact_key] = get_facts( state, fact_cls, args=args, kwargs=kwargs, apply_failed_hosts=False, ) except PyinfraError: pass print_facts(fact_data) _exit() if command == "exec": state.print_output = True add_op( state, server.shell, " ".join(operations), _allow_cli_mode=True, ) elif command == "deploy": if not quiet: click.echo(err=True) click.echo("--> Preparing operations...", err=True) # Number of "steps" to make = number of files * number of hosts for i, filename in enumerate(operations): logger.info("Loading: {0}".format(click.style(filename, bold=True))) state.current_op_file_number = i load_deploy_file(state, filename) # Remove any config changes introduced by the deploy file & any includes config.reset_locked_state() elif command == "op": if not quiet: click.echo(err=True) click.echo("--> Preparing operation...", err=True) op, args = operations args, kwargs = args kwargs["_allow_cli_mode"] = True def print_host_ready(host): logger.info( "{0}{1} {2}".format( host.print_prefix, click.style("Ready:", "green"), click.style(original_operations[0], bold=True), ), ) kwargs["_after_host_callback"] = print_host_ready add_op(state, op, *args, **kwargs) # Print proposed changes, execute unless --dry, and exit # if not quiet: click.echo(err=True) click.echo("--> Proposed changes:", err=True) print_meta(state) # If --debug-facts or --debug-operations, print and exit if debug_facts or debug_operations: if debug_facts: print_state_facts(state) if debug_operations: print_state_operations(state) _exit() if dry: _exit() if not quiet: click.echo(err=True) if not quiet: click.echo("--> Beginning operation run...", err=True) run_ops(state, serial=serial, no_wait=no_wait) if not quiet: click.echo("--> Results:", err=True) print_results(state) _exit()