def test_full_op_fail(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) add_op(state, server.shell, 'echo "hi"') with patch('pyinfra.api.connectors.ssh.run_shell_command') as fake_run_command: fake_channel = FakeChannel(1) fake_run_command.return_value = ( False, FakeBuffer('', fake_channel), ) with self.assertRaises(PyinfraError) as e: run_ops(state) assert e.exception.args[0] == 'No hosts remaining!' somehost = inventory.get_host('somehost') # Ensure the op was not flagged as success assert state.results[somehost]['success_ops'] == 0 # And was flagged asn an error assert state.results[somehost]['error_ops'] == 1
def test_run_ops_no_wait(self): state = State(make_inventory(), make_config()) connect_all(state) add_op(state, server.shell, 'echo "hello world"') run_ops(state, no_wait=True)
def test_rsync_op(self): inventory = make_inventory(hosts=('somehost', )) state = State(inventory, Config()) connect_all(state) with patch('pyinfra.api.connectors.ssh.check_can_rsync'): add_op(state, files.rsync, 'src', 'dest', sudo=True, sudo_user='******') assert len(state.get_op_order()) == 1 with patch('pyinfra.api.connectors.ssh.run_local_process' ) as fake_run_local_process: fake_run_local_process.return_value = 0, [] run_ops(state) fake_run_local_process.assert_called_with( ('rsync -ax --delete --rsh ' "'ssh -o BatchMode=yes -o StrictHostKeyChecking=no '" " --rsync-path 'sudo -u root rsync' src vagrant@somehost:dest"), print_output=False, print_prefix=inventory.get_host('somehost').print_prefix, )
def test_ignore_errors_op_fail(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) add_op(state, server.shell, 'echo "hi"', ignore_errors=True) with patch("pyinfra.connectors.ssh.run_shell_command" ) as fake_run_command: fake_channel = FakeChannel(1) fake_run_command.return_value = ( False, FakeBuffer("", fake_channel), ) # This should run OK run_ops(state) somehost = inventory.get_host("somehost") # Ensure the op was added to results assert state.results[somehost]["ops"] == 1 assert state.results[somehost]["ignored_error_ops"] == 1 # But not as a success assert state.results[somehost]["success_ops"] == 0
def test_rsync_op(self): inventory = make_inventory(hosts=("somehost", )) state = State(inventory, Config()) connect_all(state) with patch("pyinfra.connectors.ssh.check_can_rsync"): add_op(state, files.rsync, "src", "dest", sudo=True, sudo_user="******") assert len(state.get_op_order()) == 1 with patch("pyinfra.connectors.ssh.run_local_process" ) as fake_run_local_process: fake_run_local_process.return_value = 0, [] run_ops(state) fake_run_local_process.assert_called_with( ("rsync -ax --delete --rsh " "'ssh -o BatchMode=yes '" " --rsync-path 'sudo -u root rsync' src vagrant@somehost:dest"), print_output=False, print_prefix=inventory.get_host("somehost").print_prefix, )
def test_ignore_errors_op_fail(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) add_op(state, server.shell, 'echo "hi"', ignore_errors=True) with patch('pyinfra.api.connectors.ssh.run_shell_command' ) as fake_run_command: fake_channel = FakeChannel(1) fake_run_command.return_value = ( False, FakeBuffer('', fake_channel), ) # This should run OK run_ops(state) somehost = inventory.get_host('somehost') # Ensure the op was added to results self.assertEqual(state.results[somehost]['ops'], 1) self.assertEqual(state.results[somehost]['error_ops'], 1) # But not as a success self.assertEqual(state.results[somehost]['success_ops'], 0)
def test_rsync_op_failure(self): inventory = make_inventory(hosts=('somehost',)) state = State(inventory, Config()) connect_all(state) with patch('pyinfra.api.connectors.ssh.find_executable', lambda x: None): with self.assertRaises(OperationError) as context: add_op(state, files.rsync, 'src', 'dest') assert context.exception.args[0] == 'The `rsync` binary is not available on this system.'
def test_add_op_rejects_cli(self): pyinfra.is_cli = True with self.assertRaises(PyinfraError) as context: add_op(None, server.shell) pyinfra.is_cli = False assert context.exception.args[0] == ( '`add_op` should not be called when pyinfra is executing in CLI mode! ' '(line 362 in tests/test_api/test_api_operations.py)')
def test_basic_op(self): state = State(make_inventory(), make_config()) connect_all(state) add_op(state, files.file, '/var/log/pyinfra.log', user='******', group='pyinfra', mode='644', sudo=True) run_ops(state)
def test_add_op_rejects_cli(self): pyinfra.is_cli = True with self.assertRaises(PyinfraError) as context: add_op(None, server.shell) call_line = getframeinfo(currentframe()).lineno - 1 pyinfra.is_cli = False assert context.exception.args[0] == ( '`add_op` should not be called when pyinfra is executing in CLI mode! ' '(line {0} in tests/test_api/test_api_operations.py)'.format( call_line))
def add_action(self, action, desc: str, args: list = [], kwargs: dict = {}, response=None, apply=False) -> Dict: add_op( # self.state, yum.packages, self.state, action, {desc}, *args, **kwargs ) # TODO fullfil response after apply if apply: self.apply() return response
def test_op_line_numbers(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) # Add op to both hosts add_op(state, server.shell, 'echo "hi"') # Add op to just the second host - using the pseudo modules such that # it replicates a deploy file. pseudo_state.set(state) pseudo_host.set(inventory['anotherhost']) first_pseudo_hash = server.user('anotherhost_user').hash first_pseudo_call_line = getframeinfo(currentframe()).lineno - 1 # Add op to just the first host - using the pseudo modules such that # it replicates a deploy file. pseudo_state.set(state) pseudo_host.set(inventory['somehost']) second_pseudo_hash = server.user('somehost_user').hash second_pseudo_call_line = getframeinfo(currentframe()).lineno - 1 pseudo_state.reset() pseudo_host.reset() # Ensure there are two ops op_order = state.get_op_order() self.assertEqual(len(op_order), 3) # And that the two ops above were called in the expected order self.assertEqual(op_order[1], first_pseudo_hash) self.assertEqual(op_order[2], second_pseudo_hash) # And that they have the expected line numbers self.assertEqual( state.op_line_numbers_to_hash.get((0, first_pseudo_call_line)), first_pseudo_hash, ) self.assertEqual( state.op_line_numbers_to_hash.get((0, second_pseudo_call_line)), second_pseudo_hash, ) # Ensure somehost has two ops and anotherhost only has the one self.assertEqual(len(state.ops[inventory.get_host('somehost')]), 2) self.assertEqual(len(state.ops[inventory.get_host('anotherhost')]), 2)
def test_op_cannot_change_execution_kwargs(self): inventory = make_inventory() state = State(inventory, Config()) class NoSetDefaultDict(defaultdict): def setdefault(self, key, _): return self[key] state.op_meta = NoSetDefaultDict(lambda: {'serial': True}) connect_all(state) with self.assertRaises(OperationValueError) as context: add_op(state, files.file, '/var/log/pyinfra.log', serial=False) assert context.exception.args[0] == 'Cannot have different values for `serial`.'
def test_limited_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) # Add op to both hosts add_op(state, server.shell, 'echo "hi"') # Add op to just the first host add_limited_op(state, server.user, inventory['somehost'], 'somehost_user') # Ensure there are two ops self.assertEqual(len(state.op_order), 2) # Ensure somehost has two ops and anotherhost only has the one self.assertEqual(len(state.ops['somehost']), 2) self.assertEqual(len(state.ops['anotherhost']), 1)
def test_file_download_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) with patch('pyinfra.operations.files.os_path.isfile', lambda *args, **kwargs: True): add_op( state, files.get, name='First op name', src='/home/vagrant/file.txt', dest='files/file.txt', ) op_order = state.get_op_order() assert len(op_order) == 1 first_op_hash = op_order[0] assert state.op_meta[first_op_hash]['names'] == {'First op name'} somehost = inventory.get_host('somehost') anotherhost = inventory.get_host('anotherhost') # Ensure first op has the right (upload) command assert state.ops[somehost][first_op_hash]['commands'] == [ FileDownloadCommand('/home/vagrant/file.txt', 'files/file.txt'), ] with patch('pyinfra.api.util.open', mock_open(read_data='test!'), create=True): run_ops(state) assert state.results[somehost]['success_ops'] == 1 assert state.results[somehost]['ops'] == 1 assert state.results[anotherhost]['success_ops'] == 1 assert state.results[anotherhost]['ops'] == 1 assert state.results[somehost]['error_ops'] == 0 assert state.results[anotherhost]['error_ops'] == 0
def test_file_download_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) with patch("pyinfra.operations.files.os.path.isfile", lambda *args, **kwargs: True): add_op( state, files.get, name="First op name", src="/home/vagrant/file.txt", dest="files/file.txt", ) op_order = state.get_op_order() assert len(op_order) == 1 first_op_hash = op_order[0] assert state.op_meta[first_op_hash]["names"] == {"First op name"} somehost = inventory.get_host("somehost") anotherhost = inventory.get_host("anotherhost") # Ensure first op has the right (upload) command assert state.ops[somehost][first_op_hash]["commands"] == [ FileDownloadCommand("/home/vagrant/file.txt", "files/file.txt"), ] with patch("pyinfra.api.util.open", mock_open(read_data="test!"), create=True): run_ops(state) assert state.results[somehost]["success_ops"] == 1 assert state.results[somehost]["ops"] == 1 assert state.results[anotherhost]["success_ops"] == 1 assert state.results[anotherhost]["ops"] == 1 assert state.results[somehost]["error_ops"] == 0 assert state.results[anotherhost]["error_ops"] == 0
def test_run_once_serial_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) # Add a run once op add_op(state, server.shell, 'echo "hi"', run_once=True, serial=True) # Ensure it's added to op_order self.assertEqual(len(state.op_order), 1) # Ensure between the two hosts we only run the one op self.assertEqual( len(state.ops['somehost']) + len(state.ops['anotherhost']), 1) # Check run works run_ops(state) self.assertEqual((state.results['somehost']['success_ops'] + state.results['anotherhost']['success_ops']), 1)
def test_function_call_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) is_called = [] def mocked_function(*args, **kwargs): is_called.append(True) return None # Add op to both hosts add_op(state, python.call, mocked_function) # Ensure there is one op assert len(state.get_op_order()) == 1 run_ops(state) assert is_called
def test_op_hosts_limit(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) # Add op to both hosts add_op(state, server.shell, 'echo "hi"') # Add op to just the first host add_op( state, server.user, 'somehost_user', hosts=inventory['somehost'], ) # Ensure there are two ops assert len(state.get_op_order()) == 2 # Ensure somehost has two ops and anotherhost only has the one assert len(state.ops[inventory.get_host('somehost')]) == 2 assert len(state.ops[inventory.get_host('anotherhost')]) == 1
def test_op_state_hosts_limit(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) # Add op to both hosts add_op(state, server.shell, 'echo "hi"') # Add op to just the first host with state.hosts('test_group'): add_op( state, server.user, 'somehost_user', ) # Now, also limited but set hosts to the non-limited hosts, which # should mean this operation applies to no hosts. add_op( state, server.user, 'somehost_user', hosts=inventory.get_host('anotherhost'), ) # Ensure there are three ops assert len(state.get_op_order()) == 3 # Ensure somehost has two ops and anotherhost only has the one assert len(state.ops[inventory.get_host('somehost')]) == 2 assert len(state.ops[inventory.get_host('anotherhost')]) == 1
def test_run_once_serial_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) # Add a run once op add_op(state, server.shell, 'echo "hi"', run_once=True, serial=True) # Ensure it's added to op_order assert len(state.get_op_order()) == 1 somehost = inventory.get_host('somehost') anotherhost = inventory.get_host('anotherhost') # Ensure between the two hosts we only run the one op assert len(state.ops[somehost]) + len(state.ops[anotherhost]) == 1 # Check run works run_ops(state) assert (state.results[somehost]['success_ops'] + state.results[anotherhost]['success_ops']) == 1
def test_full_op_fail(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) add_op(state, server.shell, 'echo "hi"') with patch('pyinfra.api.operations.run_shell_command' ) as fake_run_command: fake_channel = FakeChannel(1) fake_run_command.return_value = (fake_channel, FakeBuffer('', fake_channel), FakeBuffer('', fake_channel)) with self.assertRaises(PyinfraError) as e: run_ops(state) self.assertEqual(e.exception.args[0], 'No hosts remaining!') # Ensure the op was not flagged as success self.assertEqual(state.results['somehost']['success_ops'], 0) # And was flagged asn an error self.assertEqual(state.results['somehost']['error_ops'], 1)
def test_file_op(self): state = State(make_inventory(), make_config()) connect_all(state) # Test normal with patch('pyinfra.modules.files.open', mock_open(read_data='test!'), create=True): add_op(state, files.put, 'files/file.txt', '/home/vagrant/file.txt') # And with sudo with patch('pyinfra.modules.files.open', mock_open(read_data='test!'), create=True): add_op(state, files.put, 'files/file.txt', '/home/vagrant/file.txt', sudo=True, sudo_user='******') run_ops(state)
def test_api_op_line_numbers(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) another_host = inventory.get_host('anotherhost') def add_another_op(): return add_op(state, server.shell, 'echo second-op')[another_host].hash first_op_hash = add_op(state, server.shell, 'echo first-op')[another_host].hash second_op_hash = add_another_op() # note `add_op` will be called on an earlier line op_order = state.get_op_order() assert len(op_order) == 2 assert op_order[0] == first_op_hash assert op_order[1] == second_op_hash
def test_pipelining_active_works(self): state = State(make_inventory(), Config()) connect_all(state) state.pipelining = True add_op(state, server.shell, 'echo "hi"')
def test_file_op(self): state = State(make_inventory(), Config()) connect_all(state) with patch('pyinfra.modules.files.path.isfile', lambda *args, **kwargs: True): # Test normal add_op(state, files.put, {'First op name'}, 'files/file.txt', '/home/vagrant/file.txt') # And with sudo add_op(state, files.put, 'files/file.txt', '/home/vagrant/file.txt', sudo=True, sudo_user='******') # And with su add_op(state, files.put, 'files/file.txt', '/home/vagrant/file.txt', sudo=True, su_user='******') # Ensure we have all ops self.assertEqual(len(state.op_order), 3) first_op_hash = state.op_order[0] # Ensure first op is the right one self.assertEqual(state.op_meta[first_op_hash]['names'], {'First op name'}) # Ensure first op has the right (upload) command self.assertEqual(state.ops['somehost'][first_op_hash]['commands'], [('files/file.txt', '/home/vagrant/file.txt')]) # Ensure second op has sudo/sudo_user self.assertEqual(state.op_meta[state.op_order[1]]['sudo'], True) self.assertEqual(state.op_meta[state.op_order[1]]['sudo_user'], 'pyinfra') # Ensure third has su_user self.assertEqual(state.op_meta[state.op_order[2]]['su_user'], 'pyinfra') # Check run ops works with patch('pyinfra.api.util.open', mock_open(read_data='test!'), create=True): run_ops(state) # Ensure ops completed OK self.assertEqual(state.results['somehost']['success_ops'], 3) self.assertEqual(state.results['somehost']['ops'], 3) self.assertEqual(state.results['anotherhost']['success_ops'], 3) self.assertEqual(state.results['anotherhost']['ops'], 3) # And w/o errors self.assertEqual(state.results['somehost']['error_ops'], 0) self.assertEqual(state.results['anotherhost']['error_ops'], 0)
def test_op(self): state = State(make_inventory(), Config()) # Enable printing on this test to catch any exceptions in the formatting state.print_output = True state.print_fact_info = True state.print_fact_output = True state.print_lines = True connect_all(state) add_op(state, files.file, '/var/log/pyinfra.log', user='******', group='pyinfra', mode='644', sudo=True, sudo_user='******', su_user='******', ignore_errors=True, env={ 'TEST': 'what', }) # Ensure we have an op self.assertEqual(len(state.op_order), 1) first_op_hash = state.op_order[0] # Ensure the op name self.assertEqual(state.op_meta[first_op_hash]['names'], {'Files/File'}) # Ensure the commands self.assertEqual(state.ops['somehost'][first_op_hash]['commands'], [ 'touch /var/log/pyinfra.log', 'chmod 644 /var/log/pyinfra.log', 'chown pyinfra:pyinfra /var/log/pyinfra.log' ]) # Ensure the meta meta = state.op_meta[first_op_hash] self.assertEqual(meta['sudo'], True) self.assertEqual(meta['sudo_user'], 'test_sudo') self.assertEqual(meta['su_user'], 'test_su') self.assertEqual(meta['ignore_errors'], True) # Ensure run ops works run_ops(state) # Ensure ops completed OK self.assertEqual(state.results['somehost']['success_ops'], 1) self.assertEqual(state.results['somehost']['ops'], 1) self.assertEqual(state.results['anotherhost']['success_ops'], 1) self.assertEqual(state.results['anotherhost']['ops'], 1) # And w/o errors self.assertEqual(state.results['somehost']['error_ops'], 0) self.assertEqual(state.results['anotherhost']['error_ops'], 0) # And with the different modes run_ops(state, serial=True) run_ops(state, no_wait=True)
def _main( inventory, operations, verbosity, user, port, key, key_password, password, winrm_username, winrm_password, winrm_port, shell_executable, sudo, sudo_user, use_sudo_password, su_user, parallel, fail_percent, dry, limit, no_wait, serial, quiet, debug, debug_data, debug_facts, debug_operations, facts=None, print_operations=None, support=None, ): if not debug and not sys.warnoptions: warnings.simplefilter('ignore') # Setup logging log_level = logging.INFO if debug: log_level = logging.DEBUG elif quiet: log_level = logging.WARNING setup_logging(log_level) # Bootstrap any virtualenv init_virtualenv() deploy_dir = getcwd() potential_deploy_dirs = [] # This is the most common case: we have a deploy file so use it's # pathname - we only look at the first file as we can't have multiple # deploy directories. if operations[0].endswith('.py'): deploy_file_dir, _ = path.split(operations[0]) above_deploy_file_dir, _ = path.split(deploy_file_dir) deploy_dir = deploy_file_dir potential_deploy_dirs.extend(( deploy_file_dir, above_deploy_file_dir, )) # If we have a valid inventory, look in it's path and it's parent for # group_data or config.py to indicate deploy_dir (--fact, --run). if inventory.endswith('.py') and path.isfile(inventory): inventory_dir, _ = path.split(inventory) above_inventory_dir, _ = path.split(inventory_dir) potential_deploy_dirs.extend(( inventory_dir, above_inventory_dir, )) for potential_deploy_dir in potential_deploy_dirs: logger.debug('Checking potential directory: {0}'.format( potential_deploy_dir, )) if any(( path.isdir(path.join(potential_deploy_dir, 'group_data')), path.isfile(path.join(potential_deploy_dir, 'config.py')), )): logger.debug( 'Setting directory to: {0}'.format(potential_deploy_dir)) deploy_dir = potential_deploy_dir break # Create an empty/unitialised state object state = State() # Set the deploy directory state.deploy_dir = deploy_dir pseudo_state.set(state) if verbosity > 0: state.print_fact_info = True state.print_noop_info = True if verbosity > 1: state.print_input = state.print_fact_input = True if verbosity > 2: state.print_output = state.print_fact_output = True if not quiet: click.echo('--> Loading config...', err=True) # Load up any config.py from the filesystem config = load_config(deploy_dir) # Make a copy before we overwrite original_operations = operations # Debug (print) inventory + group data if operations[0] == 'debug-inventory': command = 'debug-inventory' # Get all non-arg facts elif operations[0] == 'all-facts': command = 'fact' fact_names = [] for fact_name in get_fact_names(): fact_class = get_fact_class(fact_name) if (not issubclass(fact_class, ShortFactBase) and not callable(fact_class.command)): fact_names.append(fact_name) operations = [(name, None) for name in fact_names] # Get one or more facts elif operations[0] == 'fact': command = 'fact' fact_names = operations[1:] facts = [] for name in fact_names: args = None if ':' in name: name, args = name.split(':', 1) args = args.split(',') if not is_fact(name): raise CliError('No fact: {0}'.format(name)) facts.append((name, args)) operations = facts # Execute a raw command with server.shell elif operations[0] == 'exec': command = 'exec' operations = operations[1:] # Execute one or more deploy files elif all(cmd.endswith('.py') for cmd in operations): command = 'deploy' operations = operations[0:] for file in operations: if not path.exists(file): raise CliError('No deploy file: {0}'.format(file)) # Operation w/optional args (<module>.<op> ARG1 ARG2 ...) elif len(operations[0].split('.')) == 2: command = 'op' operations = get_operation_and_args(operations) else: raise CliError('''Invalid operations: {0} Operation usage: pyinfra INVENTORY deploy_web.py [deploy_db.py]... pyinfra INVENTORY server.user pyinfra home=/home/pyinfra pyinfra INVENTORY exec -- echo "hello world" pyinfra INVENTORY fact os [users]...'''.format(operations)) # Load any hooks/config from the deploy file if command == 'deploy': load_deploy_config(operations[0], config) # Arg based config overrides if sudo: config.SUDO = True if sudo_user: config.SUDO_USER = sudo_user if use_sudo_password: config.USE_SUDO_PASSWORD = use_sudo_password if su_user: config.SU_USER = su_user if parallel: config.PARALLEL = parallel if shell_executable: config.SHELL = shell_executable if fail_percent is not None: config.FAIL_PERCENT = fail_percent if not quiet: click.echo('--> Loading inventory...', err=True) # Load up the inventory from the filesystem inventory, inventory_group = make_inventory( inventory, deploy_dir=deploy_dir, ssh_port=port, ssh_user=user, ssh_key=key, ssh_key_password=key_password, ssh_password=password, winrm_username=winrm_username, winrm_password=winrm_password, winrm_port=winrm_port, ) # Attach to pseudo inventory pseudo_inventory.set(inventory) # Now that we have inventory, apply --limit config override initial_limit = None if limit: all_limit_hosts = [] for limiter in limit: try: limit_hosts = inventory.get_group(limiter) except NoGroupError: limits = limiter.split(',') if len(limits) > 1: logger.warning(( 'Specifying comma separated --limit values is deprecated, ' 'please use multiple --limit options.')) limit_hosts = [ host for host in inventory if any( fnmatch(host.name, match) for match in limits) ] all_limit_hosts.extend(limit_hosts) initial_limit = list(set(all_limit_hosts)) # Initialise the state state.init(inventory, config, initial_limit=initial_limit) # If --debug-data dump & exit if command == 'debug-inventory' or debug_data: if debug_data: logger.warning( ('--debug-data is deprecated, ' 'please use `pyinfra INVENTORY debug-inventory` instead.')) print_inventory(state) _exit() # Connect to all the servers if not quiet: click.echo(err=True) click.echo('--> Connecting to hosts...', err=True) connect_all(state) # Just getting a fact? # if command == 'fact': if not quiet: click.echo(err=True) click.echo('--> Gathering facts...', err=True) state.print_fact_info = True fact_data = {} for i, command in enumerate(operations): name, args = command fact_key = name if args: fact_key = '{0}{1}'.format(name, tuple(args)) try: fact_data[fact_key] = get_facts( state, name, args=args, apply_failed_hosts=False, ) except PyinfraError: pass print_facts(fact_data) _exit() # Prepare the deploy! # # Execute a raw command with server.shell if command == 'exec': # Print the output of the command state.print_output = True add_op( state, server.shell, ' '.join(operations), _allow_cli_mode=True, ) # Deploy files(s) elif command == 'deploy': if not quiet: click.echo(err=True) click.echo('--> Preparing operations...', err=True) # Number of "steps" to make = number of files * number of hosts for i, filename in enumerate(operations): logger.info('Loading: {0}'.format(click.style(filename, bold=True))) state.current_op_file = i load_deploy_file(state, filename) # Operation w/optional args elif command == 'op': if not quiet: click.echo(err=True) click.echo('--> Preparing operation...', err=True) op, args = operations args, kwargs = args kwargs['_allow_cli_mode'] = True def print_host_ready(host): logger.info('{0}{1} {2}'.format( host.print_prefix, click.style('Ready:', 'green'), click.style(original_operations[0], bold=True), )) kwargs['_after_host_callback'] = print_host_ready add_op(state, op, *args, **kwargs) # Always show meta output if not quiet: click.echo(err=True) click.echo('--> Proposed changes:', err=True) print_meta(state) # If --debug-facts or --debug-operations, print and exit if debug_facts or debug_operations: if debug_facts: print_state_facts(state) if debug_operations: print_state_operations(state) _exit() # Run the operations we generated with the deploy file if dry: _exit() if not quiet: click.echo(err=True) if not quiet: click.echo('--> Beginning operation run...', err=True) run_ops(state, serial=serial, no_wait=no_wait) if not quiet: click.echo('--> Results:', err=True) print_results(state) _exit()
def _main( inventory, commands, verbosity, user, port, key, key_password, password, sudo, sudo_user, su_user, parallel, fail_percent, dry, limit, no_wait, serial, debug, debug_data, debug_state, facts=None, operations=None, ): print() print('### {0}'.format(click.style('Welcome to pyinfra', bold=True))) print() # Setup logging log_level = logging.DEBUG if debug else logging.INFO setup_logging(log_level) deploy_dir = getcwd() potential_deploy_dirs = [] # This is the most common case: we have a deploy file so use it's # pathname - we only look at the first file as we can't have multiple # deploy directories. if commands[0].endswith('.py'): deploy_file_dir, _ = path.split(commands[0]) above_deploy_file_dir, _ = path.split(deploy_file_dir) deploy_dir = deploy_file_dir potential_deploy_dirs.extend(( deploy_file_dir, above_deploy_file_dir, )) # If we have a valid inventory, look in it's path and it's parent for # group_data or config.py to indicate deploy_dir (--fact, --run). if inventory.endswith('.py') and path.isfile(inventory): inventory_dir, _ = path.split(inventory) above_inventory_dir, _ = path.split(inventory_dir) potential_deploy_dirs.extend(( inventory_dir, above_inventory_dir, )) for potential_deploy_dir in potential_deploy_dirs: logger.debug('Checking potential directory: {0}'.format( potential_deploy_dir, )) if any(( path.isdir(path.join(potential_deploy_dir, 'group_data')), path.isfile(path.join(potential_deploy_dir, 'config.py')), )): logger.debug( 'Setting directory to: {0}'.format(potential_deploy_dir)) deploy_dir = potential_deploy_dir break # List facts if commands[0] == 'fact': command = 'fact' fact_names = commands[1:] facts = [] for name in fact_names: args = None if ':' in name: name, args = name.split(':', 1) args = args.split(',') if not is_fact(name): raise CliError('No fact: {0}'.format(name)) facts.append((name, args)) commands = facts # Execute a raw command with server.shell elif commands[0] == 'exec': command = 'exec' commands = commands[1:] # Deploy files(s) elif all(cmd.endswith('.py') for cmd in commands): command = 'deploy' commands = commands[0:] # Check each file exists for file in commands: if not path.exists(file): raise CliError('No deploy file: {0}'.format(file)) # Operation w/optional args elif len(commands) == 2: command = 'op' commands = get_operation_and_args( commands[0], commands[1], ) else: raise CliError('''Invalid commands: {0} Command usage: pyinfra INVENTORY deploy_web.py [deploy_db.py]... pyinfra INVENTORY server.user pyinfra,home=/home/pyinfra pyinfra INVENTORY exec -- echo "hello world" pyinfra INVENTORY fact os [users]...'''.format(commands)) print('--> Loading config...') # Load up any config.py from the filesystem config = load_config(deploy_dir) # Load any hooks/config from the deploy file if command == 'deploy': load_deploy_config(commands[0], config) # Arg based config overrides if sudo: config.SUDO = True if sudo_user: config.SUDO_USER = sudo_user if su_user: config.SU_USER = su_user if parallel: config.PARALLEL = parallel if fail_percent is not None: config.FAIL_PERCENT = fail_percent print('--> Loading inventory...') # Load up the inventory from the filesystem inventory, inventory_group = make_inventory( inventory, deploy_dir=deploy_dir, limit=limit, ssh_user=user, ssh_key=key, ssh_key_password=key_password, ssh_password=password, ssh_port=port, ) # If --debug-data dump & exit if debug_data: print_inventory(inventory) _exit() # Attach to pseudo inventory pseudo_inventory.set(inventory) # Create/set the state state = State(inventory, config) state.is_cli = True state.print_lines = True state.deploy_dir = deploy_dir # Setup printing on the new state print_output = verbosity > 0 print_fact_output = verbosity > 1 state.print_output = print_output # -v state.print_fact_info = print_output # -v state.print_fact_output = print_fact_output # -vv # Attach to pseudo state pseudo_state.set(state) # Setup the data to be passed to config hooks hook_data = FallbackAttrData( state.inventory.get_override_data(), state.inventory.get_group_data(inventory_group), state.inventory.get_data(), ) # Run the before_connect hook if provided run_hook(state, 'before_connect', hook_data) # Connect to all the servers print('--> Connecting to hosts...') with progress_spinner(state.inventory) as progress: connect_all(state, progress=progress) # Run the before_connect hook if provided run_hook(state, 'before_facts', hook_data) # Just getting a fact? # if command == 'fact': print() print('--> Gathering facts...') # Print facts as we get them state.print_fact_info = True # Print fact output with -v state.print_fact_output = print_output fact_data = {} with progress_spinner(commands) as progress: for i, (name, args) in enumerate(commands): fact_data[name] = get_facts( state, name, args=args, ) progress() print_facts(fact_data) _exit() # Prepare the deploy! # # Execute a raw command with server.shell if command == 'exec': # Print the output of the command state.print_output = True add_op( state, server.shell, ' '.join(commands), ) # Deploy files(s) elif command == 'deploy': print() print('--> Preparing operations...') # Number of "steps" to make = number of files * number of hosts prepare_steps = len(commands) * len(state.inventory) with progress_spinner(prepare_steps) as progress: for filename in commands: load_deploy_file(state, filename, progress=progress) progress() # Operation w/optional args elif command == 'op': print() print('--> Preparing operation...') op, args = commands add_op(state, op, *args[0], **args[1]) # Always show meta output print() print('--> Proposed changes:') print_meta(state, inventory) # If --debug-state, dump state (ops, op order, op meta) now & exit if debug_state: dump_state(state) _exit() # Run the operations we generated with the deploy file if dry: _exit() print() # Run the before_deploy hook if provided run_hook(state, 'before_deploy', hook_data) print('--> Beginning operation run...') # Number of "steps" to make = number of operations * number of hosts operation_steps = len(state.op_order) * len(state.inventory) with progress_spinner(operation_steps) as progress: run_ops( state, serial=serial, no_wait=no_wait, progress=progress, ) # Run the after_deploy hook if provided run_hook(state, 'after_deploy', hook_data) print('--> Results:') print_results(state, inventory) _exit()
def test_file_upload_op(self): inventory = make_inventory() state = State(inventory, Config()) connect_all(state) with patch('pyinfra.operations.files.path.isfile', lambda *args, **kwargs: True): # Test normal add_op( state, files.put, {'First op name'}, 'files/file.txt', '/home/vagrant/file.txt', ) # And with sudo add_op( state, files.put, 'files/file.txt', '/home/vagrant/file.txt', sudo=True, sudo_user='******', ) # And with su add_op( state, files.put, 'files/file.txt', '/home/vagrant/file.txt', sudo=True, su_user='******', ) op_order = state.get_op_order() # Ensure we have all ops assert len(op_order) == 3 first_op_hash = op_order[0] # Ensure first op is the right one assert state.op_meta[first_op_hash]['names'] == {'First op name'} somehost = inventory.get_host('somehost') anotherhost = inventory.get_host('anotherhost') # Ensure first op has the right (upload) command assert state.ops[somehost][first_op_hash]['commands'] == [ ('upload', 'files/file.txt', '/home/vagrant/file.txt'), ] # Ensure second op has sudo/sudo_user assert state.op_meta[op_order[1]]['sudo'] is True assert state.op_meta[op_order[1]]['sudo_user'] == 'pyinfra' # Ensure third has su_user assert state.op_meta[op_order[2]]['su_user'] == 'pyinfra' # Check run ops works with patch('pyinfra.api.util.open', mock_open(read_data='test!'), create=True): run_ops(state) # Ensure ops completed OK assert state.results[somehost]['success_ops'] == 3 assert state.results[somehost]['ops'] == 3 assert state.results[anotherhost]['success_ops'] == 3 assert state.results[anotherhost]['ops'] == 3 # And w/o errors assert state.results[somehost]['error_ops'] == 0 assert state.results[anotherhost]['error_ops'] == 0