def test_nested_op_api(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
somehost = inventory.get_host("somehost")
ctx_state.set(state)
ctx_host.set(somehost)
pyinfra.is_cli = True
try:
outer_result = server.shell(commands="echo outer")
assert outer_result.combined_output_lines is None
def callback():
inner_result = server.shell(commands="echo inner")
assert inner_result.combined_output_lines is not None
python.call(function=callback)
assert len(state.get_op_order()) == 2
run_ops(state)
assert len(state.get_op_order()) == 3
assert state.results[somehost]["success_ops"] == 3
assert outer_result.combined_output_lines is not None
disconnect_all(state)
finally:
pyinfra.is_cli = False
def test_op_state_hosts_limit(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Add op to both hosts
add_op(state, server.shell, 'echo "hi"')
# Add op to just the first host
with state.hosts('test_group'):
add_op(
state, server.user,
'somehost_user',
)
# Now, also limited but set hosts to the non-limited hosts, which
# should mean this operation applies to no hosts.
add_op(
state, server.user,
'somehost_user',
hosts=inventory.get_host('anotherhost'),
)
# Ensure there are three ops
assert len(state.get_op_order()) == 3
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host('somehost')]) == 2
assert len(state.ops[inventory.get_host('anotherhost')]) == 1
def test_rsync_op(self):
inventory = make_inventory(hosts=("somehost", ))
state = State(inventory, Config())
connect_all(state)
with patch("pyinfra.connectors.ssh.check_can_rsync"):
add_op(state,
files.rsync,
"src",
"dest",
sudo=True,
sudo_user="******")
assert len(state.get_op_order()) == 1
with patch("pyinfra.connectors.ssh.run_local_process"
) as fake_run_local_process:
fake_run_local_process.return_value = 0, []
run_ops(state)
fake_run_local_process.assert_called_with(
("rsync -ax --delete --rsh "
"'ssh -o BatchMode=yes '"
" --rsync-path 'sudo -u root rsync' src vagrant@somehost:dest"),
print_output=False,
print_prefix=inventory.get_host("somehost").print_prefix,
)
def test_rsync_op(self):
inventory = make_inventory(hosts=('somehost', ))
state = State(inventory, Config())
connect_all(state)
with patch('pyinfra.api.connectors.ssh.check_can_rsync'):
add_op(state,
files.rsync,
'src',
'dest',
sudo=True,
sudo_user='******')
assert len(state.get_op_order()) == 1
with patch('pyinfra.api.connectors.ssh.run_local_process'
) as fake_run_local_process:
fake_run_local_process.return_value = 0, []
run_ops(state)
fake_run_local_process.assert_called_with(
('rsync -ax --delete --rsh '
"'ssh -o BatchMode=yes -o StrictHostKeyChecking=no '"
" --rsync-path 'sudo -u root rsync' src vagrant@somehost:dest"),
print_output=False,
print_prefix=inventory.get_host('somehost').print_prefix,
)
def test_nested_deploy(self):
inventory = make_inventory()
somehost = inventory.get_host("somehost")
state = State(inventory, Config())
# Enable printing on this test to catch any exceptions in the formatting
state.print_output = True
state.print_input = True
state.print_fact_info = True
state.print_noop_info = True
connect_all(state)
@deploy
def test_nested_deploy():
server.shell(commands=["echo nested command"])
@deploy
def test_deploy():
server.shell(commands=["echo first command"])
test_nested_deploy()
server.shell(commands=["echo second command"])
add_deploy(state, test_deploy)
op_order = state.get_op_order()
# Ensure we have an op
assert len(op_order) == 3
first_op_hash = op_order[0]
assert state.op_meta[first_op_hash]["names"] == {
"test_deploy | Server/Shell"
}
assert state.ops[somehost][first_op_hash]["commands"] == [
StringCommand("echo first command"),
]
second_op_hash = op_order[1]
assert state.op_meta[second_op_hash]["names"] == {
"test_deploy | test_nested_deploy | Server/Shell",
}
assert state.ops[somehost][second_op_hash]["commands"] == [
StringCommand("echo nested command"),
]
third_op_hash = op_order[2]
assert state.op_meta[third_op_hash]["names"] == {
"test_deploy | Server/Shell"
}
assert state.ops[somehost][third_op_hash]["commands"] == [
StringCommand("echo second command"),
]
def test_cli_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
pyinfra.is_cli = True
pseudo_state.set(state)
# Add op to both hosts
for name in ('anotherhost', 'somehost'):
pseudo_host.set(inventory.get_host(name))
server.shell(
'echo hi') # note this is called twice but on *the same line*
# Add op to just the second host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_host.set(inventory.get_host('anotherhost'))
first_pseudo_hash = server.user('anotherhost_user').hash
first_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
# Add op to just the first host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_host.set(inventory.get_host('somehost'))
second_pseudo_hash = server.user('somehost_user').hash
second_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
pseudo_state.reset()
pseudo_host.reset()
pyinfra.is_cli = False
# Ensure there are two ops
op_order = state.get_op_order()
assert len(op_order) == 3
# And that the two ops above were called in the expected order
assert op_order[1] == first_pseudo_hash
assert op_order[2] == second_pseudo_hash
# And that they have the expected line numbers
assert state.op_line_numbers_to_hash.get(
(0, first_pseudo_call_line)) == first_pseudo_hash
assert state.op_line_numbers_to_hash.get(
(0, second_pseudo_call_line)) == second_pseudo_hash
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host('somehost')]) == 2
assert len(state.ops[inventory.get_host('anotherhost')]) == 2
def test_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Add op to both hosts
add_op(state, server.shell, 'echo "hi"')
# Add op to just the second host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_state.set(state)
pseudo_host.set(inventory['anotherhost'])
first_pseudo_hash = server.user('anotherhost_user').hash
first_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
# Add op to just the first host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_state.set(state)
pseudo_host.set(inventory['somehost'])
second_pseudo_hash = server.user('somehost_user').hash
second_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
pseudo_state.reset()
pseudo_host.reset()
# Ensure there are two ops
op_order = state.get_op_order()
self.assertEqual(len(op_order), 3)
# And that the two ops above were called in the expected order
self.assertEqual(op_order[1], first_pseudo_hash)
self.assertEqual(op_order[2], second_pseudo_hash)
# And that they have the expected line numbers
self.assertEqual(
state.op_line_numbers_to_hash.get((0, first_pseudo_call_line)),
first_pseudo_hash,
)
self.assertEqual(
state.op_line_numbers_to_hash.get((0, second_pseudo_call_line)),
second_pseudo_hash,
)
# Ensure somehost has two ops and anotherhost only has the one
self.assertEqual(len(state.ops[inventory.get_host('somehost')]), 2)
self.assertEqual(len(state.ops[inventory.get_host('anotherhost')]), 2)
def test_api_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
another_host = inventory.get_host('anotherhost')
def add_another_op():
return add_op(state, server.shell, 'echo second-op')[another_host].hash
first_op_hash = add_op(state, server.shell, 'echo first-op')[another_host].hash
second_op_hash = add_another_op() # note `add_op` will be called on an earlier line
op_order = state.get_op_order()
assert len(op_order) == 2
assert op_order[0] == first_op_hash
assert op_order[1] == second_op_hash
def _do_test_deploy(self):
correct_op_name_and_host_names = [
('First main operation', True), # true for all hosts
('Second main operation', ('somehost', )),
('tests/test_deploy/a_task.py | First task operation',
('anotherhost', )),
('tests/test_deploy/a_task.py | Second task operation',
('anotherhost', )),
('tests/test_deploy/a_task.py | First task operation', True),
('tests/test_deploy/a_task.py | Second task operation', True),
('Loop-0 main operation', True),
('Loop-1 main operation', True),
('Third main operation', True),
]
hosts = ['somehost', 'anotherhost', 'someotherhost']
shuffle(hosts)
inventory = make_inventory(hosts=hosts)
state = State(inventory, Config())
state.deploy_dir = path.join('tests', 'test_deploy')
connect_all(state)
pseudo_state.set(state)
pyinfra.is_cli = True
load_deploy_file(state, path.join(state.deploy_dir, 'deploy.py'))
pyinfra.is_cli = False
op_order = state.get_op_order()
for i, (correct_op_name, correct_host_names) in enumerate(
correct_op_name_and_host_names, ):
op_hash = op_order[i]
op_meta = state.op_meta[op_hash]
self.assertEqual(list(op_meta['names'])[0], correct_op_name)
for host in inventory:
op_hashes = state.meta[host]['op_hashes']
if correct_host_names is True or host.name in correct_host_names:
self.assertIn(op_hash, op_hashes)
else:
self.assertNotIn(op_hash, op_hashes)
def test_file_download_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
with patch("pyinfra.operations.files.os.path.isfile",
lambda *args, **kwargs: True):
add_op(
state,
files.get,
name="First op name",
src="/home/vagrant/file.txt",
dest="files/file.txt",
)
op_order = state.get_op_order()
assert len(op_order) == 1
first_op_hash = op_order[0]
assert state.op_meta[first_op_hash]["names"] == {"First op name"}
somehost = inventory.get_host("somehost")
anotherhost = inventory.get_host("anotherhost")
# Ensure first op has the right (upload) command
assert state.ops[somehost][first_op_hash]["commands"] == [
FileDownloadCommand("/home/vagrant/file.txt", "files/file.txt"),
]
with patch("pyinfra.api.util.open",
mock_open(read_data="test!"),
create=True):
run_ops(state)
assert state.results[somehost]["success_ops"] == 1
assert state.results[somehost]["ops"] == 1
assert state.results[anotherhost]["success_ops"] == 1
assert state.results[anotherhost]["ops"] == 1
assert state.results[somehost]["error_ops"] == 0
assert state.results[anotherhost]["error_ops"] == 0
def test_file_download_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
with patch('pyinfra.operations.files.os_path.isfile',
lambda *args, **kwargs: True):
add_op(
state,
files.get,
name='First op name',
src='/home/vagrant/file.txt',
dest='files/file.txt',
)
op_order = state.get_op_order()
assert len(op_order) == 1
first_op_hash = op_order[0]
assert state.op_meta[first_op_hash]['names'] == {'First op name'}
somehost = inventory.get_host('somehost')
anotherhost = inventory.get_host('anotherhost')
# Ensure first op has the right (upload) command
assert state.ops[somehost][first_op_hash]['commands'] == [
FileDownloadCommand('/home/vagrant/file.txt', 'files/file.txt'),
]
with patch('pyinfra.api.util.open',
mock_open(read_data='test!'),
create=True):
run_ops(state)
assert state.results[somehost]['success_ops'] == 1
assert state.results[somehost]['ops'] == 1
assert state.results[anotherhost]['success_ops'] == 1
assert state.results[anotherhost]['ops'] == 1
assert state.results[somehost]['error_ops'] == 0
assert state.results[anotherhost]['error_ops'] == 0
def test_cli_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
state.current_deploy_filename = __file__
pyinfra.is_cli = True
ctx_state.set(state)
# Add op to both hosts
for name in ("anotherhost", "somehost"):
ctx_host.set(inventory.get_host(name))
server.shell(
"echo hi") # note this is called twice but on *the same line*
# Add op to just the second host - using the context modules such that
# it replicates a deploy file.
ctx_host.set(inventory.get_host("anotherhost"))
first_context_hash = server.user("anotherhost_user").hash
# Add op to just the first host - using the context modules such that
# it replicates a deploy file.
ctx_host.set(inventory.get_host("somehost"))
second_context_hash = server.user("somehost_user").hash
ctx_state.reset()
ctx_host.reset()
pyinfra.is_cli = False
# Ensure there are two ops
op_order = state.get_op_order()
assert len(op_order) == 3
# And that the two ops above were called in the expected order
assert op_order[1] == first_context_hash
assert op_order[2] == second_context_hash
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host("somehost")]) == 2
assert len(state.ops[inventory.get_host("anotherhost")]) == 2
def test_function_call_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
is_called = []
def mocked_function(*args, **kwargs):
is_called.append(True)
return None
# Add op to both hosts
add_op(state, python.call, mocked_function)
# Ensure there is one op
assert len(state.get_op_order()) == 1
run_ops(state)
assert is_called
def test_op_hosts_limit(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Add op to both hosts
add_op(state, server.shell, 'echo "hi"')
# Add op to just the first host
add_op(
state, server.user,
'somehost_user',
hosts=inventory['somehost'],
)
# Ensure there are two ops
assert len(state.get_op_order()) == 2
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host('somehost')]) == 2
assert len(state.ops[inventory.get_host('anotherhost')]) == 1
def test_run_once_serial_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Add a run once op
add_op(state, server.shell, 'echo "hi"', run_once=True, serial=True)
# Ensure it's added to op_order
assert len(state.get_op_order()) == 1
somehost = inventory.get_host('somehost')
anotherhost = inventory.get_host('anotherhost')
# Ensure between the two hosts we only run the one op
assert len(state.ops[somehost]) + len(state.ops[anotherhost]) == 1
# Check run works
run_ops(state)
assert (state.results[somehost]['success_ops'] +
state.results[anotherhost]['success_ops']) == 1
def test_op(self):
inventory = make_inventory()
somehost = inventory.get_host("somehost")
anotherhost = inventory.get_host("anotherhost")
state = State(inventory, Config())
state.add_callback_handler(BaseStateCallback())
# Enable printing on this test to catch any exceptions in the formatting
state.print_output = True
state.print_input = True
state.print_fact_info = True
state.print_noop_info = True
connect_all(state)
add_op(
state,
files.file,
"/var/log/pyinfra.log",
user="******",
group="pyinfra",
mode="644",
create_remote_dir=False,
sudo=True,
sudo_user="******",
su_user="******",
ignore_errors=True,
env={
"TEST": "what",
},
)
op_order = state.get_op_order()
# Ensure we have an op
assert len(op_order) == 1
first_op_hash = op_order[0]
# Ensure the op name
assert state.op_meta[first_op_hash]["names"] == {"Files/File"}
# Ensure the commands
assert state.ops[somehost][first_op_hash]["commands"] == [
StringCommand("touch /var/log/pyinfra.log"),
StringCommand("chmod 644 /var/log/pyinfra.log"),
StringCommand("chown pyinfra:pyinfra /var/log/pyinfra.log"),
]
# Ensure the global kwargs (same for both hosts)
somehost_global_kwargs = state.ops[somehost][first_op_hash][
"global_kwargs"]
assert somehost_global_kwargs["sudo"] is True
assert somehost_global_kwargs["sudo_user"] == "test_sudo"
assert somehost_global_kwargs["su_user"] == "test_su"
assert somehost_global_kwargs["ignore_errors"] is True
anotherhost_global_kwargs = state.ops[anotherhost][first_op_hash][
"global_kwargs"]
assert anotherhost_global_kwargs["sudo"] is True
assert anotherhost_global_kwargs["sudo_user"] == "test_sudo"
assert anotherhost_global_kwargs["su_user"] == "test_su"
assert anotherhost_global_kwargs["ignore_errors"] is True
# Ensure run ops works
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]["success_ops"] == 1
assert state.results[somehost]["ops"] == 1
assert state.results[anotherhost]["success_ops"] == 1
assert state.results[anotherhost]["ops"] == 1
# And w/o errors
assert state.results[somehost]["error_ops"] == 0
assert state.results[anotherhost]["error_ops"] == 0
# And with the different modes
run_ops(state, serial=True)
run_ops(state, no_wait=True)
disconnect_all(state)
def test_op(self):
inventory = make_inventory()
somehost = inventory.get_host('somehost')
anotherhost = inventory.get_host('anotherhost')
state = State(inventory, Config())
# Enable printing on this test to catch any exceptions in the formatting
state.print_output = True
state.print_fact_info = True
state.print_fact_output = True
connect_all(state)
add_op(
state, files.file,
'/var/log/pyinfra.log',
user='******',
group='pyinfra',
mode='644',
sudo=True,
sudo_user='******',
su_user='******',
ignore_errors=True,
env={
'TEST': 'what',
},
)
op_order = state.get_op_order()
# Ensure we have an op
assert len(op_order) == 1
first_op_hash = op_order[0]
# Ensure the op name
assert state.op_meta[first_op_hash]['names'] == {'Files/File'}
# Ensure the commands
assert state.ops[somehost][first_op_hash]['commands'] == [
'touch /var/log/pyinfra.log',
'chmod 644 /var/log/pyinfra.log',
'chown pyinfra:pyinfra /var/log/pyinfra.log',
]
# Ensure the meta
meta = state.op_meta[first_op_hash]
assert meta['sudo'] is True
assert meta['sudo_user'] == 'test_sudo'
assert meta['su_user'] == 'test_su'
assert meta['ignore_errors'] is True
# Ensure run ops works
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]['success_ops'] == 1
assert state.results[somehost]['ops'] == 1
assert state.results[anotherhost]['success_ops'] == 1
assert state.results[anotherhost]['ops'] == 1
# And w/o errors
assert state.results[somehost]['error_ops'] == 0
assert state.results[anotherhost]['error_ops'] == 0
# And with the different modes
run_ops(state, serial=True)
run_ops(state, no_wait=True)
def test_deploy(self):
inventory = make_inventory()
somehost = inventory.get_host("somehost")
anotherhost = inventory.get_host("anotherhost")
state = State(inventory, Config())
# Enable printing on this test to catch any exceptions in the formatting
state.print_output = True
state.print_input = True
state.print_fact_info = True
state.print_noop_info = True
connect_all(state)
@deploy
def test_deploy(state=None, host=None):
server.shell(commands=["echo first command"])
server.shell(commands=["echo second command"])
add_deploy(state, test_deploy)
op_order = state.get_op_order()
# Ensure we have an op
assert len(op_order) == 2
first_op_hash = op_order[0]
assert state.op_meta[first_op_hash]["names"] == {
"test_deploy | Server/Shell"
}
assert state.ops[somehost][first_op_hash]["commands"] == [
StringCommand("echo first command"),
]
assert state.ops[anotherhost][first_op_hash]["commands"] == [
StringCommand("echo first command"),
]
second_op_hash = op_order[1]
assert state.op_meta[second_op_hash]["names"] == {
"test_deploy | Server/Shell"
}
assert state.ops[somehost][second_op_hash]["commands"] == [
StringCommand("echo second command"),
]
assert state.ops[anotherhost][second_op_hash]["commands"] == [
StringCommand("echo second command"),
]
# Ensure run ops works
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]["success_ops"] == 2
assert state.results[somehost]["ops"] == 2
assert state.results[anotherhost]["success_ops"] == 2
assert state.results[anotherhost]["ops"] == 2
# And w/o errors
assert state.results[somehost]["error_ops"] == 0
assert state.results[anotherhost]["error_ops"] == 0
# And with the different modes
run_ops(state, serial=True)
run_ops(state, no_wait=True)
disconnect_all(state)
def test_file_upload_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
with patch('pyinfra.operations.files.path.isfile', lambda *args, **kwargs: True):
# Test normal
add_op(
state, files.put,
{'First op name'},
'files/file.txt',
'/home/vagrant/file.txt',
)
# And with sudo
add_op(
state, files.put,
'files/file.txt',
'/home/vagrant/file.txt',
sudo=True,
sudo_user='******',
)
# And with su
add_op(
state, files.put,
'files/file.txt',
'/home/vagrant/file.txt',
sudo=True,
su_user='******',
)
op_order = state.get_op_order()
# Ensure we have all ops
assert len(op_order) == 3
first_op_hash = op_order[0]
# Ensure first op is the right one
assert state.op_meta[first_op_hash]['names'] == {'First op name'}
somehost = inventory.get_host('somehost')
anotherhost = inventory.get_host('anotherhost')
# Ensure first op has the right (upload) command
assert state.ops[somehost][first_op_hash]['commands'] == [
('upload', 'files/file.txt', '/home/vagrant/file.txt'),
]
# Ensure second op has sudo/sudo_user
assert state.op_meta[op_order[1]]['sudo'] is True
assert state.op_meta[op_order[1]]['sudo_user'] == 'pyinfra'
# Ensure third has su_user
assert state.op_meta[op_order[2]]['su_user'] == 'pyinfra'
# Check run ops works
with patch('pyinfra.api.util.open', mock_open(read_data='test!'), create=True):
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]['success_ops'] == 3
assert state.results[somehost]['ops'] == 3
assert state.results[anotherhost]['success_ops'] == 3
assert state.results[anotherhost]['ops'] == 3
# And w/o errors
assert state.results[somehost]['error_ops'] == 0
assert state.results[anotherhost]['error_ops'] == 0
def test_file_upload_op(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Test normal
add_op(
state,
files.put,
name="First op name",
src="files/file.txt",
dest="/home/vagrant/file.txt",
)
# And with sudo
add_op(
state,
files.put,
src="files/file.txt",
dest="/home/vagrant/file.txt",
sudo=True,
sudo_user="******",
)
# And with su
add_op(
state,
files.put,
src="files/file.txt",
dest="/home/vagrant/file.txt",
sudo=True,
su_user="******",
)
op_order = state.get_op_order()
# Ensure we have all ops
assert len(op_order) == 3
first_op_hash = op_order[0]
second_op_hash = op_order[1]
# Ensure first op is the right one
assert state.op_meta[first_op_hash]["names"] == {"First op name"}
somehost = inventory.get_host("somehost")
anotherhost = inventory.get_host("anotherhost")
# Ensure first op has the right (upload) command
assert state.ops[somehost][first_op_hash]["commands"] == [
StringCommand("mkdir -p /home/vagrant"),
FileUploadCommand("files/file.txt", "/home/vagrant/file.txt"),
]
# Ensure second op has sudo/sudo_user
assert state.ops[somehost][second_op_hash]["global_kwargs"][
"sudo"] is True
assert state.ops[somehost][second_op_hash]["global_kwargs"][
"sudo_user"] == "pyinfra"
# Ensure third has su_user
assert state.ops[somehost][
op_order[2]]["global_kwargs"]["su_user"] == "pyinfra"
# Check run ops works
run_ops(state)
# Ensure ops completed OK
assert state.results[somehost]["success_ops"] == 3
assert state.results[somehost]["ops"] == 3
assert state.results[anotherhost]["success_ops"] == 3
assert state.results[anotherhost]["ops"] == 3
# And w/o errors
assert state.results[somehost]["error_ops"] == 0
assert state.results[anotherhost]["error_ops"] == 0