def _main(
inventory,
operations,
verbosity,
user,
port,
key,
key_password,
password,
winrm_username,
winrm_password,
winrm_port,
shell_executable,
sudo,
sudo_user,
use_sudo_password,
su_user,
parallel,
fail_percent,
dry,
limit,
no_wait,
serial,
quiet,
debug,
debug_data,
debug_facts,
debug_operations,
facts=None,
print_operations=None,
support=None,
):
if not debug and not sys.warnoptions:
warnings.simplefilter('ignore')
# Setup logging
log_level = logging.INFO
if debug:
log_level = logging.DEBUG
elif quiet:
log_level = logging.WARNING
setup_logging(log_level)
# Bootstrap any virtualenv
init_virtualenv()
deploy_dir = getcwd()
potential_deploy_dirs = []
# This is the most common case: we have a deploy file so use it's
# pathname - we only look at the first file as we can't have multiple
# deploy directories.
if operations[0].endswith('.py'):
deploy_file_dir, _ = path.split(operations[0])
above_deploy_file_dir, _ = path.split(deploy_file_dir)
deploy_dir = deploy_file_dir
potential_deploy_dirs.extend((
deploy_file_dir,
above_deploy_file_dir,
))
# If we have a valid inventory, look in it's path and it's parent for
# group_data or config.py to indicate deploy_dir (--fact, --run).
if inventory.endswith('.py') and path.isfile(inventory):
inventory_dir, _ = path.split(inventory)
above_inventory_dir, _ = path.split(inventory_dir)
potential_deploy_dirs.extend((
inventory_dir,
above_inventory_dir,
))
for potential_deploy_dir in potential_deploy_dirs:
logger.debug('Checking potential directory: {0}'.format(
potential_deploy_dir, ))
if any((
path.isdir(path.join(potential_deploy_dir, 'group_data')),
path.isfile(path.join(potential_deploy_dir, 'config.py')),
)):
logger.debug(
'Setting directory to: {0}'.format(potential_deploy_dir))
deploy_dir = potential_deploy_dir
break
# Create an empty/unitialised state object
state = State()
# Set the deploy directory
state.deploy_dir = deploy_dir
pseudo_state.set(state)
if verbosity > 0:
state.print_fact_info = True
state.print_noop_info = True
if verbosity > 1:
state.print_input = state.print_fact_input = True
if verbosity > 2:
state.print_output = state.print_fact_output = True
if not quiet:
click.echo('--> Loading config...', err=True)
# Load up any config.py from the filesystem
config = load_config(deploy_dir)
# Make a copy before we overwrite
original_operations = operations
# Debug (print) inventory + group data
if operations[0] == 'debug-inventory':
command = 'debug-inventory'
# Get all non-arg facts
elif operations[0] == 'all-facts':
command = 'fact'
fact_names = []
for fact_name in get_fact_names():
fact_class = get_fact_class(fact_name)
if (not issubclass(fact_class, ShortFactBase)
and not callable(fact_class.command)):
fact_names.append(fact_name)
operations = [(name, None) for name in fact_names]
# Get one or more facts
elif operations[0] == 'fact':
command = 'fact'
fact_names = operations[1:]
facts = []
for name in fact_names:
args = None
if ':' in name:
name, args = name.split(':', 1)
args = args.split(',')
if not is_fact(name):
raise CliError('No fact: {0}'.format(name))
facts.append((name, args))
operations = facts
# Execute a raw command with server.shell
elif operations[0] == 'exec':
command = 'exec'
operations = operations[1:]
# Execute one or more deploy files
elif all(cmd.endswith('.py') for cmd in operations):
command = 'deploy'
operations = operations[0:]
for file in operations:
if not path.exists(file):
raise CliError('No deploy file: {0}'.format(file))
# Operation w/optional args (<module>.<op> ARG1 ARG2 ...)
elif len(operations[0].split('.')) == 2:
command = 'op'
operations = get_operation_and_args(operations)
else:
raise CliError('''Invalid operations: {0}
Operation usage:
pyinfra INVENTORY deploy_web.py [deploy_db.py]...
pyinfra INVENTORY server.user pyinfra home=/home/pyinfra
pyinfra INVENTORY exec -- echo "hello world"
pyinfra INVENTORY fact os [users]...'''.format(operations))
# Load any hooks/config from the deploy file
if command == 'deploy':
load_deploy_config(operations[0], config)
# Arg based config overrides
if sudo:
config.SUDO = True
if sudo_user:
config.SUDO_USER = sudo_user
if use_sudo_password:
config.USE_SUDO_PASSWORD = use_sudo_password
if su_user:
config.SU_USER = su_user
if parallel:
config.PARALLEL = parallel
if shell_executable:
config.SHELL = shell_executable
if fail_percent is not None:
config.FAIL_PERCENT = fail_percent
if not quiet:
click.echo('--> Loading inventory...', err=True)
# Load up the inventory from the filesystem
inventory, inventory_group = make_inventory(
inventory,
deploy_dir=deploy_dir,
ssh_port=port,
ssh_user=user,
ssh_key=key,
ssh_key_password=key_password,
ssh_password=password,
winrm_username=winrm_username,
winrm_password=winrm_password,
winrm_port=winrm_port,
)
# Attach to pseudo inventory
pseudo_inventory.set(inventory)
# Now that we have inventory, apply --limit config override
initial_limit = None
if limit:
all_limit_hosts = []
for limiter in limit:
try:
limit_hosts = inventory.get_group(limiter)
except NoGroupError:
limits = limiter.split(',')
if len(limits) > 1:
logger.warning((
'Specifying comma separated --limit values is deprecated, '
'please use multiple --limit options.'))
limit_hosts = [
host for host in inventory if any(
fnmatch(host.name, match) for match in limits)
]
all_limit_hosts.extend(limit_hosts)
initial_limit = list(set(all_limit_hosts))
# Initialise the state
state.init(inventory, config, initial_limit=initial_limit)
# If --debug-data dump & exit
if command == 'debug-inventory' or debug_data:
if debug_data:
logger.warning(
('--debug-data is deprecated, '
'please use `pyinfra INVENTORY debug-inventory` instead.'))
print_inventory(state)
_exit()
# Connect to all the servers
if not quiet:
click.echo(err=True)
click.echo('--> Connecting to hosts...', err=True)
connect_all(state)
# Just getting a fact?
#
if command == 'fact':
if not quiet:
click.echo(err=True)
click.echo('--> Gathering facts...', err=True)
state.print_fact_info = True
fact_data = {}
for i, command in enumerate(operations):
name, args = command
fact_key = name
if args:
fact_key = '{0}{1}'.format(name, tuple(args))
try:
fact_data[fact_key] = get_facts(
state,
name,
args=args,
apply_failed_hosts=False,
)
except PyinfraError:
pass
print_facts(fact_data)
_exit()
# Prepare the deploy!
#
# Execute a raw command with server.shell
if command == 'exec':
# Print the output of the command
state.print_output = True
add_op(
state,
server.shell,
' '.join(operations),
_allow_cli_mode=True,
)
# Deploy files(s)
elif command == 'deploy':
if not quiet:
click.echo(err=True)
click.echo('--> Preparing operations...', err=True)
# Number of "steps" to make = number of files * number of hosts
for i, filename in enumerate(operations):
logger.info('Loading: {0}'.format(click.style(filename,
bold=True)))
state.current_op_file = i
load_deploy_file(state, filename)
# Operation w/optional args
elif command == 'op':
if not quiet:
click.echo(err=True)
click.echo('--> Preparing operation...', err=True)
op, args = operations
args, kwargs = args
kwargs['_allow_cli_mode'] = True
def print_host_ready(host):
logger.info('{0}{1} {2}'.format(
host.print_prefix,
click.style('Ready:', 'green'),
click.style(original_operations[0], bold=True),
))
kwargs['_after_host_callback'] = print_host_ready
add_op(state, op, *args, **kwargs)
# Always show meta output
if not quiet:
click.echo(err=True)
click.echo('--> Proposed changes:', err=True)
print_meta(state)
# If --debug-facts or --debug-operations, print and exit
if debug_facts or debug_operations:
if debug_facts:
print_state_facts(state)
if debug_operations:
print_state_operations(state)
_exit()
# Run the operations we generated with the deploy file
if dry:
_exit()
if not quiet:
click.echo(err=True)
if not quiet:
click.echo('--> Beginning operation run...', err=True)
run_ops(state, serial=serial, no_wait=no_wait)
if not quiet:
click.echo('--> Results:', err=True)
print_results(state)
_exit()
def make_inventory(
inventory_filename,
deploy_dir=None,
ssh_port=None,
ssh_user=None,
ssh_key=None,
ssh_key_password=None,
ssh_password=None,
winrm_username=None,
winrm_password=None,
winrm_port=None,
):
'''
Builds a ``pyinfra.api.Inventory`` from the filesystem. If the file does not exist
and doesn't contain a / attempts to use that as the only hostname.
'''
if ssh_port is not None:
ssh_port = int(ssh_port)
file_groupname = None
# If we're not a valid file we assume a list of comma separated hostnames
if not path.exists(inventory_filename):
groups = {
'all': inventory_filename.split(','),
}
else:
groups = _get_groups_from_filename(inventory_filename)
# Used to set all the hosts to an additional group - that of the filename
# ie inventories/dev.py means all the hosts are in the dev group, if not present
file_groupname = path.basename(inventory_filename).rsplit('.')[0]
all_data = {}
if 'all' in groups:
all_hosts = groups.pop('all')
if isinstance(all_hosts, tuple):
all_hosts, all_data = all_hosts
# Build all out of the existing hosts if not defined
else:
all_hosts = []
for hosts in groups.values():
# Groups can be a list of hosts or tuple of (hosts, data)
hosts = hosts[0] if isinstance(hosts, tuple) else hosts
for host in hosts:
# Hosts can be a hostname or tuple of (hostname, data)
hostname = host[0] if isinstance(host, tuple) else host
if hostname not in all_hosts:
all_hosts.append(hostname)
groups['all'] = (all_hosts, all_data)
# Apply the filename group if not already defined
if file_groupname and file_groupname not in groups:
groups[file_groupname] = all_hosts
# In pyinfra an inventory is a combination of (hostnames + data). However, in CLI
# mode we want to be define this in separate files (inventory / group data). The
# issue is we want inventory access within the group data files - but at this point
# we're not ready to make an Inventory. So here we just create a fake one, and
# attach it to pseudo_inventory while we import the data files.
logger.debug('Creating fake inventory...')
fake_groups = {
# In API mode groups *must* be tuples of (hostnames, data)
name: group if isinstance(group, tuple) else (group, {})
for name, group in six.iteritems(groups)
}
fake_inventory = Inventory((all_hosts, all_data), **fake_groups)
pseudo_inventory.set(fake_inventory)
# Get all group data (group_data/*.py)
group_data = _get_group_data(deploy_dir)
# Reset the pseudo inventory
pseudo_inventory.reset()
# For each group load up any data
for name, hosts in six.iteritems(groups):
data = {}
if isinstance(hosts, tuple):
hosts, data = hosts
if name in group_data:
data.update(group_data.pop(name))
# Attach to group object
groups[name] = (hosts, data)
# Loop back through any leftover group data and create an empty (for now)
# group - this is because inventory @connectors can attach arbitrary groups
# to hosts, so we need to support that.
for name, data in six.iteritems(group_data):
groups[name] = ([], data)
return Inventory(groups.pop('all'),
ssh_user=ssh_user,
ssh_key=ssh_key,
ssh_key_password=ssh_key_password,
ssh_port=ssh_port,
ssh_password=ssh_password,
winrm_username=winrm_username,
winrm_password=winrm_password,
winrm_port=winrm_port,
**groups), file_groupname and file_groupname.lower()
def make_inventory(inventory_filename,
deploy_dir=None,
limit=None,
ssh_user=None,
ssh_key=None,
ssh_key_password=None,
ssh_port=None,
ssh_password=None):
'''
Builds a ``pyinfra.api.Inventory`` from the filesystem. If the file does not exist
and doesn't contain a / attempts to use that as the only hostname.
'''
if ssh_port is not None:
ssh_port = int(ssh_port)
file_groupname = None
try:
attrs = exec_file(inventory_filename, return_locals=True)
groups = {
key: value
for key, value in six.iteritems(attrs) if key.isupper()
}
# Used to set all the hosts to an additional group - that of the filename
# ie inventories/dev.py means all the hosts are in the dev group, if not present
file_groupname = path.basename(inventory_filename).split(
'.')[0].upper()
except IOError as e:
# If a /, definitely not a hostname
if '/' in inventory_filename:
raise CliError('{0}: {1}'.format(e.strerror, inventory_filename))
# Otherwise we assume the inventory is actually a hostname or list of hostnames
groups = {'ALL': inventory_filename.split(',')}
all_data = {}
if 'ALL' in groups:
all_hosts = groups.pop('ALL')
if isinstance(all_hosts, tuple):
all_hosts, all_data = all_hosts
# Build ALL out of the existing hosts if not defined
else:
all_hosts = []
for hosts in groups.values():
# Groups can be a list of hosts or tuple of (hosts, data)
hosts = hosts[0] if isinstance(hosts, tuple) else hosts
for host in hosts:
# Hosts can be a hostname or tuple of (hostname, data)
hostname = host[0] if isinstance(host, tuple) else host
if hostname not in all_hosts:
all_hosts.append(hostname)
groups['ALL'] = (all_hosts, all_data)
# Apply the filename group if not already defined
if file_groupname and file_groupname not in groups:
groups[file_groupname] = all_hosts
# In pyinfra an inventory is a combination of (hostnames + data). However, in CLI
# mode we want to be define this in separate files (inventory / group data). The
# issue is we want inventory access within the group data files - but at this point
# we're not ready to make an Inventory. So here we just create a fake one, and attach
# it to pseudo_inventory while we import the data files.
fake_groups = {
# In API mode groups *must* be tuples of (hostnames, data)
name: group if isinstance(group, tuple) else (group, {})
for name, group in six.iteritems(groups)
}
fake_inventory = Inventory((all_hosts, all_data), **fake_groups)
pseudo_inventory.set(fake_inventory)
# For each group load up any data
for name, hosts in six.iteritems(groups):
data = {}
if isinstance(hosts, tuple):
hosts, data = hosts
data_filename = path.join(deploy_dir, 'group_data',
'{0}.py'.format(name.lower()))
logger.debug('Looking for group data: {0}'.format(data_filename))
if path.exists(data_filename):
# Read the files locals into a dict
attrs = exec_file(data_filename, return_locals=True)
data.update({
key: value
for key, value in six.iteritems(attrs)
if isinstance(value, ALLOWED_DATA_TYPES)
and not key.startswith('_') and key.islower()
})
# Attach to group object
groups[name] = (hosts, data)
# Reset the pseudo inventory
pseudo_inventory.reset()
# Apply any limit to all_hosts
if limit:
# Limits can be groups
limit_groupname = limit.upper()
if limit_groupname in groups:
all_hosts = [
host[0] if isinstance(host, tuple) else host
for host in groups[limit_groupname][0]
]
# Or hostnames w/*wildcards
else:
all_hosts = [
host for host in all_hosts
if (isinstance(host, tuple) and fnmatch(host[0], limit)) or
(isinstance(host, six.string_types) and fnmatch(host, limit))
]
# Reassign the ALL group w/limit
groups['ALL'] = (all_hosts, all_data)
return Inventory(groups.pop('ALL'),
ssh_user=ssh_user,
ssh_key=ssh_key,
ssh_key_password=ssh_key_password,
ssh_port=ssh_port,
ssh_password=ssh_password,
**groups), file_groupname and file_groupname.lower()
def _main(
inventory,
commands,
verbosity,
user,
port,
key,
key_password,
password,
sudo,
sudo_user,
su_user,
parallel,
fail_percent,
dry,
limit,
no_wait,
serial,
debug,
debug_data,
debug_state,
facts=None,
operations=None,
):
print()
print('### {0}'.format(click.style('Welcome to pyinfra', bold=True)))
print()
# Setup logging
log_level = logging.DEBUG if debug else logging.INFO
setup_logging(log_level)
deploy_dir = getcwd()
potential_deploy_dirs = []
# This is the most common case: we have a deploy file so use it's
# pathname - we only look at the first file as we can't have multiple
# deploy directories.
if commands[0].endswith('.py'):
deploy_file_dir, _ = path.split(commands[0])
above_deploy_file_dir, _ = path.split(deploy_file_dir)
deploy_dir = deploy_file_dir
potential_deploy_dirs.extend((
deploy_file_dir,
above_deploy_file_dir,
))
# If we have a valid inventory, look in it's path and it's parent for
# group_data or config.py to indicate deploy_dir (--fact, --run).
if inventory.endswith('.py') and path.isfile(inventory):
inventory_dir, _ = path.split(inventory)
above_inventory_dir, _ = path.split(inventory_dir)
potential_deploy_dirs.extend((
inventory_dir,
above_inventory_dir,
))
for potential_deploy_dir in potential_deploy_dirs:
logger.debug('Checking potential directory: {0}'.format(
potential_deploy_dir, ))
if any((
path.isdir(path.join(potential_deploy_dir, 'group_data')),
path.isfile(path.join(potential_deploy_dir, 'config.py')),
)):
logger.debug(
'Setting directory to: {0}'.format(potential_deploy_dir))
deploy_dir = potential_deploy_dir
break
# List facts
if commands[0] == 'fact':
command = 'fact'
fact_names = commands[1:]
facts = []
for name in fact_names:
args = None
if ':' in name:
name, args = name.split(':', 1)
args = args.split(',')
if not is_fact(name):
raise CliError('No fact: {0}'.format(name))
facts.append((name, args))
commands = facts
# Execute a raw command with server.shell
elif commands[0] == 'exec':
command = 'exec'
commands = commands[1:]
# Deploy files(s)
elif all(cmd.endswith('.py') for cmd in commands):
command = 'deploy'
commands = commands[0:]
# Check each file exists
for file in commands:
if not path.exists(file):
raise CliError('No deploy file: {0}'.format(file))
# Operation w/optional args
elif len(commands) == 2:
command = 'op'
commands = get_operation_and_args(
commands[0],
commands[1],
)
else:
raise CliError('''Invalid commands: {0}
Command usage:
pyinfra INVENTORY deploy_web.py [deploy_db.py]...
pyinfra INVENTORY server.user pyinfra,home=/home/pyinfra
pyinfra INVENTORY exec -- echo "hello world"
pyinfra INVENTORY fact os [users]...'''.format(commands))
print('--> Loading config...')
# Load up any config.py from the filesystem
config = load_config(deploy_dir)
# Load any hooks/config from the deploy file
if command == 'deploy':
load_deploy_config(commands[0], config)
# Arg based config overrides
if sudo:
config.SUDO = True
if sudo_user:
config.SUDO_USER = sudo_user
if su_user:
config.SU_USER = su_user
if parallel:
config.PARALLEL = parallel
if fail_percent is not None:
config.FAIL_PERCENT = fail_percent
print('--> Loading inventory...')
# Load up the inventory from the filesystem
inventory, inventory_group = make_inventory(
inventory,
deploy_dir=deploy_dir,
limit=limit,
ssh_user=user,
ssh_key=key,
ssh_key_password=key_password,
ssh_password=password,
ssh_port=port,
)
# If --debug-data dump & exit
if debug_data:
print_inventory(inventory)
_exit()
# Attach to pseudo inventory
pseudo_inventory.set(inventory)
# Create/set the state
state = State(inventory, config)
state.is_cli = True
state.print_lines = True
state.deploy_dir = deploy_dir
# Setup printing on the new state
print_output = verbosity > 0
print_fact_output = verbosity > 1
state.print_output = print_output # -v
state.print_fact_info = print_output # -v
state.print_fact_output = print_fact_output # -vv
# Attach to pseudo state
pseudo_state.set(state)
# Setup the data to be passed to config hooks
hook_data = FallbackAttrData(
state.inventory.get_override_data(),
state.inventory.get_group_data(inventory_group),
state.inventory.get_data(),
)
# Run the before_connect hook if provided
run_hook(state, 'before_connect', hook_data)
# Connect to all the servers
print('--> Connecting to hosts...')
with progress_spinner(state.inventory) as progress:
connect_all(state, progress=progress)
# Run the before_connect hook if provided
run_hook(state, 'before_facts', hook_data)
# Just getting a fact?
#
if command == 'fact':
print()
print('--> Gathering facts...')
# Print facts as we get them
state.print_fact_info = True
# Print fact output with -v
state.print_fact_output = print_output
fact_data = {}
with progress_spinner(commands) as progress:
for i, (name, args) in enumerate(commands):
fact_data[name] = get_facts(
state,
name,
args=args,
)
progress()
print_facts(fact_data)
_exit()
# Prepare the deploy!
#
# Execute a raw command with server.shell
if command == 'exec':
# Print the output of the command
state.print_output = True
add_op(
state,
server.shell,
' '.join(commands),
)
# Deploy files(s)
elif command == 'deploy':
print()
print('--> Preparing operations...')
# Number of "steps" to make = number of files * number of hosts
prepare_steps = len(commands) * len(state.inventory)
with progress_spinner(prepare_steps) as progress:
for filename in commands:
load_deploy_file(state, filename, progress=progress)
progress()
# Operation w/optional args
elif command == 'op':
print()
print('--> Preparing operation...')
op, args = commands
add_op(state, op, *args[0], **args[1])
# Always show meta output
print()
print('--> Proposed changes:')
print_meta(state, inventory)
# If --debug-state, dump state (ops, op order, op meta) now & exit
if debug_state:
dump_state(state)
_exit()
# Run the operations we generated with the deploy file
if dry:
_exit()
print()
# Run the before_deploy hook if provided
run_hook(state, 'before_deploy', hook_data)
print('--> Beginning operation run...')
# Number of "steps" to make = number of operations * number of hosts
operation_steps = len(state.op_order) * len(state.inventory)
with progress_spinner(operation_steps) as progress:
run_ops(
state,
serial=serial,
no_wait=no_wait,
progress=progress,
)
# Run the after_deploy hook if provided
run_hook(state, 'after_deploy', hook_data)
print('--> Results:')
print_results(state, inventory)
_exit()
deploy_dir = inventory_path
# Load up the inventory from the filesystem
inventory, inventory_group = make_inventory(
arguments['inventory'],
deploy_dir=deploy_dir,
limit=arguments['limit'],
ssh_user=arguments['user'],
ssh_key=arguments['key'],
ssh_key_password=arguments['key_password'],
ssh_port=arguments['port'],
ssh_password=arguments['password']
)
# Attach to pseudo inventory
pseudo_inventory.set(inventory)
# Load up any config.py from the filesystem
config = load_config(deploy_dir)
# Arg based overrides
if arguments['sudo']:
config.SUDO = True
if arguments['sudo_user']:
config.SUDO_USER = arguments['sudo_user']
if arguments['su_user']:
config.SU_USER = arguments['su_user']
if arguments['parallel']:
config.PARALLEL = arguments['parallel']
def _main(
inventory,
operations,
verbosity,
user,
port,
key,
key_password,
password,
sudo,
sudo_user,
su_user,
parallel,
fail_percent,
dry,
limit,
no_wait,
serial,
debug,
debug_data,
debug_facts,
debug_operations,
facts=None,
print_operations=None,
):
print()
print('### {0}'.format(click.style('Welcome to pyinfra', bold=True)))
print()
# Setup logging
log_level = logging.DEBUG if debug else logging.INFO
setup_logging(log_level)
# Bootstrap any virtualenv
init_virtualenv()
deploy_dir = getcwd()
potential_deploy_dirs = []
# This is the most common case: we have a deploy file so use it's
# pathname - we only look at the first file as we can't have multiple
# deploy directories.
if operations[0].endswith('.py'):
deploy_file_dir, _ = path.split(operations[0])
above_deploy_file_dir, _ = path.split(deploy_file_dir)
deploy_dir = deploy_file_dir
potential_deploy_dirs.extend((
deploy_file_dir,
above_deploy_file_dir,
))
# If we have a valid inventory, look in it's path and it's parent for
# group_data or config.py to indicate deploy_dir (--fact, --run).
if inventory.endswith('.py') and path.isfile(inventory):
inventory_dir, _ = path.split(inventory)
above_inventory_dir, _ = path.split(inventory_dir)
potential_deploy_dirs.extend((
inventory_dir,
above_inventory_dir,
))
for potential_deploy_dir in potential_deploy_dirs:
logger.debug('Checking potential directory: {0}'.format(
potential_deploy_dir, ))
if any((
path.isdir(path.join(potential_deploy_dir, 'group_data')),
path.isfile(path.join(potential_deploy_dir, 'config.py')),
)):
logger.debug(
'Setting directory to: {0}'.format(potential_deploy_dir))
deploy_dir = potential_deploy_dir
break
# List facts
if operations[0] == 'fact':
command = 'fact'
fact_names = operations[1:]
facts = []
for name in fact_names:
args = None
if ':' in name:
name, args = name.split(':', 1)
args = args.split(',')
if not is_fact(name):
raise CliError('No fact: {0}'.format(name))
facts.append((name, args))
operations = facts
# Execute a raw command with server.shell
elif operations[0] == 'exec':
command = 'exec'
operations = operations[1:]
# Deploy files(s)
elif all(cmd.endswith('.py') for cmd in operations):
command = 'deploy'
operations = operations[0:]
# Check each file exists
for file in operations:
if not path.exists(file):
raise CliError('No deploy file: {0}'.format(file))
# Operation w/optional args (<module>.<op> ARG1 ARG2 ...)
elif len(operations[0].split('.')) == 2:
command = 'op'
operations = get_operation_and_args(operations)
else:
raise CliError('''Invalid operations: {0}
Operation usage:
pyinfra INVENTORY deploy_web.py [deploy_db.py]...
pyinfra INVENTORY server.user pyinfra home=/home/pyinfra
pyinfra INVENTORY exec -- echo "hello world"
pyinfra INVENTORY fact os [users]...'''.format(operations))
# Create an empty/unitialised state object
state = State()
pseudo_state.set(state)
# Setup printing on the new state
print_output = verbosity > 0
print_fact_output = verbosity > 1
state.print_output = print_output # -v
state.print_fact_info = print_output # -v
state.print_fact_output = print_fact_output # -vv
print('--> Loading config...')
# Load up any config.py from the filesystem
config = load_config(deploy_dir)
# Load any hooks/config from the deploy file
if command == 'deploy':
load_deploy_config(operations[0], config)
# Arg based config overrides
if sudo:
config.SUDO = True
if sudo_user:
config.SUDO_USER = sudo_user
if su_user:
config.SU_USER = su_user
if parallel:
config.PARALLEL = parallel
if fail_percent is not None:
config.FAIL_PERCENT = fail_percent
print('--> Loading inventory...')
# Load up the inventory from the filesystem
inventory, inventory_group = make_inventory(
inventory,
deploy_dir=deploy_dir,
ssh_port=port,
ssh_user=user,
ssh_key=key,
ssh_key_password=key_password,
ssh_password=password,
)
# Apply any --limit to the inventory
limit_hosts = None
if limit:
try:
limit_hosts = inventory.get_group(limit)
except NoGroupError:
limits = limit.split(',')
limit_hosts = [
host for host in inventory if any(
fnmatch(host.name, limit) for limit in limits)
]
# Attach to pseudo inventory
pseudo_inventory.set(inventory)
# Initialise the state, passing any initial --limit
state.init(inventory, config, initial_limit=limit_hosts)
# If --debug-data dump & exit
if debug_data:
print_inventory(state)
_exit()
# Set the deploy directory
state.deploy_dir = deploy_dir
# Setup the data to be passed to config hooks
hook_data = FallbackDict(
state.inventory.get_override_data(),
state.inventory.get_group_data(inventory_group),
state.inventory.get_data(),
)
# Run the before_connect hook if provided
run_hook(state, 'before_connect', hook_data)
# Connect to all the servers
print('--> Connecting to hosts...')
connect_all(state)
# Run the before_connect hook if provided
run_hook(state, 'before_facts', hook_data)
# Just getting a fact?
#
if command == 'fact':
print()
print('--> Gathering facts...')
# Print facts as we get them
state.print_fact_info = True
# Print fact output with -v
state.print_fact_output = print_output
fact_data = {}
for i, command in enumerate(operations):
name, args = command
fact_data[name] = get_facts(
state,
name,
args=args,
)
print_facts(fact_data)
_exit()
# Prepare the deploy!
#
# Execute a raw command with server.shell
if command == 'exec':
# Print the output of the command
state.print_output = True
add_op(
state,
server.shell,
' '.join(operations),
)
# Deploy files(s)
elif command == 'deploy':
print()
print('--> Preparing operations...')
# Number of "steps" to make = number of files * number of hosts
for i, filename in enumerate(operations):
logger.info('Loading: {0}'.format(click.style(filename,
bold=True)))
state.current_op_file = i
load_deploy_file(state, filename)
# Operation w/optional args
elif command == 'op':
print()
print('--> Preparing operation...')
op, args = operations
add_op(state, op, *args[0], **args[1])
# Always show meta output
print()
print('--> Proposed changes:')
print_meta(state)
# If --debug-facts or --debug-operations, print and exit
if debug_facts or debug_operations:
if debug_facts:
print_state_facts(state)
if debug_operations:
print_state_operations(state)
_exit()
# Run the operations we generated with the deploy file
if dry:
_exit()
print()
# Run the before_deploy hook if provided
run_hook(state, 'before_deploy', hook_data)
print('--> Beginning operation run...')
run_ops(state, serial=serial, no_wait=no_wait)
# Run the after_deploy hook if provided
run_hook(state, 'after_deploy', hook_data)
print('--> Results:')
print_results(state)
# Triggers any executor disconnect requirements
disconnect_all(state)
_exit()
def make_inventory(
inventory_filename,
deploy_dir=None,
limit=None,
ssh_user=None,
ssh_key=None,
ssh_key_password=None,
ssh_port=None,
ssh_password=None,
):
'''
Builds a ``pyinfra.api.Inventory`` from the filesystem. If the file does not exist
and doesn't contain a / attempts to use that as the only hostname.
'''
if ssh_port is not None:
ssh_port = int(ssh_port)
file_groupname = None
try:
attrs = exec_file(inventory_filename, return_locals=True)
groups = {
key: value
for key, value in six.iteritems(attrs)
if is_inventory_group(key, value)
}
# Used to set all the hosts to an additional group - that of the filename
# ie inventories/dev.py means all the hosts are in the dev group, if not present
file_groupname = path.basename(inventory_filename).split('.')[0]
except IOError:
# Otherwise we assume the inventory is actually a hostname or list of hostnames
groups = {
'all': inventory_filename.split(','),
}
all_data = {}
if 'all' in groups:
all_hosts = groups.pop('all')
if isinstance(all_hosts, tuple):
all_hosts, all_data = all_hosts
# Build all out of the existing hosts if not defined
else:
all_hosts = []
for hosts in groups.values():
# Groups can be a list of hosts or tuple of (hosts, data)
hosts = hosts[0] if isinstance(hosts, tuple) else hosts
for host in hosts:
# Hosts can be a hostname or tuple of (hostname, data)
hostname = host[0] if isinstance(host, tuple) else host
if hostname not in all_hosts:
all_hosts.append(hostname)
groups['all'] = (all_hosts, all_data)
# Apply the filename group if not already defined
if file_groupname and file_groupname not in groups:
groups[file_groupname] = all_hosts
# In pyinfra an inventory is a combination of (hostnames + data). However, in CLI
# mode we want to be define this in separate files (inventory / group data). The
# issue is we want inventory access within the group data files - but at this point
# we're not ready to make an Inventory. So here we just create a fake one, and
# attach it to pseudo_inventory while we import the data files.
logger.debug('Creating fake inventory...')
fake_groups = {
# In API mode groups *must* be tuples of (hostnames, data)
name: group if isinstance(group, tuple) else (group, {})
for name, group in six.iteritems(groups)
}
fake_inventory = Inventory((all_hosts, all_data), **fake_groups)
pseudo_inventory.set(fake_inventory)
# Get all group data (group_data/*.py)
group_data = get_group_data(deploy_dir)
# Reset the pseudo inventory
pseudo_inventory.reset()
# For each group load up any data
for name, hosts in six.iteritems(groups):
data = {}
if isinstance(hosts, tuple):
hosts, data = hosts
if name in group_data:
data.update(group_data.pop(name))
# Attach to group object
groups[name] = (hosts, data)
# Loop back through any leftover group data and create an empty (for now)
# group - this is because inventory @connectors can attach arbitrary groups
# to hosts, so we need to support that.
for name, data in six.iteritems(group_data):
groups[name] = ([], data)
# Apply any limit to all_hosts
if limit:
# Limits can be groups
limit_groupname = limit
if limit_groupname in groups:
all_hosts = [
host[0] if isinstance(host, tuple) else host
for host in groups[limit_groupname][0]
]
# Or hostnames w/*wildcards
else:
limits = limit.split(',')
all_hosts = [
host for host in all_hosts if (isinstance(host, tuple) and any(
fnmatch(host[0], limit) for limit in limits)) or (
isinstance(host, six.string_types) and any(
fnmatch(host, limit) for limit in limits))
]
# Reassign the all group w/limit
groups['all'] = (all_hosts, all_data)
return Inventory(groups.pop('all'),
ssh_user=ssh_user,
ssh_key=ssh_key,
ssh_key_password=ssh_key_password,
ssh_port=ssh_port,
ssh_password=ssh_password,
**groups), file_groupname and file_groupname.lower()
def test_pseudo_inventory_iter(self):
inventory = Inventory(('host', 'anotherhost'))
pseudo_inventory.set(inventory)
assert pseudo_inventory.isset() is True
assert list(iter(pseudo_inventory)) == list(iter(inventory))
def test_pseudo_inventory_len(self):
inventory = Inventory(('host', 'anotherhost'))
pseudo_inventory.set(inventory)
assert pseudo_inventory.isset() is True
assert len(pseudo_inventory) == len(inventory)
