def install_node_exporter(state, host):
if not host.data.node_exporter_version:
raise DeployError(
'No node_exporter_version set for this host, refusing to install node_exporter!',
)
server.user(
name='Create the node_exporter user (Called prometheus by default)',
user='******',
shell='/sbin/nologin',
state=state,
host=host,
)
files.directory(
name='Ensure the node_exporter install directory exists',
path='{{ host.data.node_exporter_install_dir }}',
user=host.data.node_exporter_user,
group=host.data.node_exporter_user,
state=state,
host=host,
)
# Work out the filename
host.data.node_exporter_version_name = (
'node_exporter-{0}.linux-'
'amd64' if host.fact.arch == 'x86_64' else host.fact.arch).format(
host.data.node_exporter_version)
host.data.node_exporter_temp_filename = state.get_temp_filename(
'node_exporter-{0}'.format(host.data.node_exporter_version), )
download_node_exporter = files.download(
name='Download node_exporter',
src=('{{ host.data.node_exporter_download_base_url }}/'
'v{{ host.data.node_exporter_version }}/'
'{{ host.data.node_exporter_version_name }}.tar.gz'),
dest='{{ host.data.node_exporter_temp_filename }}',
state=state,
host=host,
)
# If we downloaded node_exporter, extract it!
if download_node_exporter.changed:
server.shell(
name='Extract node_exporter',
commands='tar -xzf {{ host.data.node_exporter_temp_filename }}'
' -C {{ host.data.node_exporter_install_dir }}',
state=state,
host=host,
)
files.link(
name='Symlink node_exporter to /usr/bin',
path='{{ host.data.node_exporter_bin_dir }}/node_exporter', # link
target='{{ host.data.node_exporter_install_dir }}/'
'{{ host.data.node_exporter_version_name }}/node_exporter',
state=state,
host=host,
)
def test_op_call_rejects_no_cli(self):
with self.assertRaises(PyinfraError) as context:
server.shell()
assert context.exception.args[0] == (
'API operation called without state/host: '
'server.shell (line 373 in tests/test_api/test_api_operations.py)')
def my_nested_deploy(state, host):
server.shell(
name='First nested deploy operation',
commands='echo first nested_deploy_op',
state=state,
host=host,
)
def delete(state=None, host=None):
supported_schema_versions = [
v1beta3.DhcpData,
]
validate_schema_version(host.data.dhcp, supported_schema_versions)
filename = "dhcp-disable.sh.j2"
file_path = './dhcp-disable.sh'
files.template(
name='Render configuration script',
src=deploy_dir / 'templates' / filename,
dest=file_path,
mode='700',
state=state,
host=host,
)
server.shell(
name="Execute configuration script",
commands=[file_path],
state=state,
host=host,
)
def install_exporter(
state,
host,
ex_url,
ex_install_dir=None,
ex_user='******',
ex_bin_dir='/usr/local/bin',
):
if ex_install_dir is None:
ex_install_dir = '/usr/local'
ex_name, ex_bin_name = _get_names(ex_url)
server.user(
name='Create the node_exporter user (Called prometheus by default)',
user=ex_user,
shell='/sbin/nologin',
state=state,
host=host,
)
files.directory(
name='Ensure the node_exporter install directory exists',
path='{}/{}'.format(ex_install_dir, ex_name),
user=host.data.node_exporter_user,
group=host.data.node_exporter_user,
state=state,
host=host,
)
ex_temp_filename = state.get_temp_filename(ex_url, )
download_exporter = files.download(
name='Download exporter',
src=ex_url,
dest=ex_temp_filename,
state=state,
host=host,
)
# If we downloaded exporter, extract it!
if download_exporter.changed:
server.shell(
name='Extract exporter',
commands='tar -xzf {} -C {}/'.format(ex_temp_filename,
ex_install_dir),
state=state,
host=host,
)
files.link(
name='Symlink exporter to /usr/local/bin',
path='{}/{}'.format(ex_bin_dir, ex_name), # link
target='{}/{}/{}'.format(ex_install_dir, ex_name, ex_bin_name),
state=state,
host=host,
)
def test_op_call_rejects_no_cli(self):
with self.assertRaises(PyinfraError) as context:
server.shell()
call_line = getframeinfo(currentframe()).lineno - 1
assert context.exception.args[0] == (
'API operation called without state/host: '
'server.shell (line {0} in tests/test_api/test_api_operations.py)'.
format(call_line))
def configure(state=None, host=None):
supported_schema_versions = [
v1beta3.HttpData,
]
validate_schema_version(host.data.http, supported_schema_versions)
apt.packages(
name='Install package',
packages=['apache2'],
sudo=True,
state=state,
host=host,
)
files.directory(
name=f'Ensure HTTP root dir {host.data.http.root_dir}',
path=str(host.data.http.root_dir),
present=True,
recursive=True,
sudo=True,
state=state,
host=host,
)
apache_conf = files.template(
name='Render config file',
src=str(deploy_dir / 'templates' / 'apache2-directory.conf.j2'),
dest=str(Path('/etc') / 'apache2' / 'conf-available' / 'root.conf'),
mode='744',
user='******',
group='root',
sudo=True,
http=host.data.http,
state=state,
host=host,
)
server.shell(
name='Enable root.conf',
commands=['a2enconf root'],
sudo=True,
state=state,
host=host,
)
systemd.service(
name='Restart apache2',
service='apache2',
running=True,
restarted=apache_conf.changed,
sudo=True,
state=state,
host=host,
)
def test_deploy(state=None, host=None):
server.shell(
commands=['echo first command'],
state=state,
host=host,
)
server.shell(
commands=['echo second command'],
state=state,
host=host,
)
def my_deploy():
server.shell(
name="First deploy operation",
commands="echo first_deploy_op",
)
my_nested_deploy()
server.shell(
name="Second deploy operation",
commands="echo second_deploy_op",
)
def test_op_call_rejects_in_op(self):
state = FakeState()
pyinfra.is_cli = True
pseudo_state.set(state)
with self.assertRaises(PyinfraError) as context:
server.shell()
pyinfra.is_cli = False
pseudo_state.reset()
assert context.exception.args[0] == (
'Nested operation called without state/host: '
'server.shell (line 387 in tests/test_api/test_api_operations.py)')
def test_cli_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
pyinfra.is_cli = True
pseudo_state.set(state)
# Add op to both hosts
for name in ('anotherhost', 'somehost'):
pseudo_host.set(inventory.get_host(name))
server.shell(
'echo hi') # note this is called twice but on *the same line*
# Add op to just the second host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_host.set(inventory.get_host('anotherhost'))
first_pseudo_hash = server.user('anotherhost_user').hash
first_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
# Add op to just the first host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_host.set(inventory.get_host('somehost'))
second_pseudo_hash = server.user('somehost_user').hash
second_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
pseudo_state.reset()
pseudo_host.reset()
pyinfra.is_cli = False
# Ensure there are two ops
op_order = state.get_op_order()
assert len(op_order) == 3
# And that the two ops above were called in the expected order
assert op_order[1] == first_pseudo_hash
assert op_order[2] == second_pseudo_hash
# And that they have the expected line numbers
assert state.op_line_numbers_to_hash.get(
(0, first_pseudo_call_line)) == first_pseudo_hash
assert state.op_line_numbers_to_hash.get(
(0, second_pseudo_call_line)) == second_pseudo_hash
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host('somehost')]) == 2
assert len(state.ops[inventory.get_host('anotherhost')]) == 2
def test_nested_op_api(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
somehost = inventory.get_host("somehost")
ctx_state.set(state)
ctx_host.set(somehost)
pyinfra.is_cli = True
try:
outer_result = server.shell(commands="echo outer")
assert outer_result.combined_output_lines is None
def callback():
inner_result = server.shell(commands="echo inner")
assert inner_result.combined_output_lines is not None
python.call(function=callback)
assert len(state.get_op_order()) == 2
run_ops(state)
assert len(state.get_op_order()) == 3
assert state.results[somehost]["success_ops"] == 3
assert outer_result.combined_output_lines is not None
disconnect_all(state)
finally:
pyinfra.is_cli = False
def test_op_call_rejects_in_op(self):
state = FakeState()
pyinfra.is_cli = True
pseudo_state.set(state)
with self.assertRaises(PyinfraError) as context:
server.shell()
call_line = getframeinfo(currentframe()).lineno - 1
pyinfra.is_cli = False
pseudo_state.reset()
assert context.exception.args[0] == (
'Nested operation called without state/host: '
'server.shell (line {0} in tests/test_api/test_api_operations.py)'.
format(call_line))
def test_cli_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
state.current_deploy_filename = __file__
pyinfra.is_cli = True
ctx_state.set(state)
# Add op to both hosts
for name in ("anotherhost", "somehost"):
ctx_host.set(inventory.get_host(name))
server.shell(
"echo hi") # note this is called twice but on *the same line*
# Add op to just the second host - using the context modules such that
# it replicates a deploy file.
ctx_host.set(inventory.get_host("anotherhost"))
first_context_hash = server.user("anotherhost_user").hash
# Add op to just the first host - using the context modules such that
# it replicates a deploy file.
ctx_host.set(inventory.get_host("somehost"))
second_context_hash = server.user("somehost_user").hash
ctx_state.reset()
ctx_host.reset()
pyinfra.is_cli = False
# Ensure there are two ops
op_order = state.get_op_order()
assert len(op_order) == 3
# And that the two ops above were called in the expected order
assert op_order[1] == first_context_hash
assert op_order[2] == second_context_hash
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host("somehost")]) == 2
assert len(state.ops[inventory.get_host("anotherhost")]) == 2
def test_op_call_direct_falls(self):
inventory = make_inventory()
somehost = inventory.get_host('somehost')
state = State(inventory, Config())
# Enable printing on this test to catch any exceptions in the formatting
state.print_output = True
state.print_input = True
state.print_fact_info = True
state.print_noop_info = True
connect_all(state)
with self.assertRaises(PyinfraError) as context:
server.shell(state=state, host=somehost, commands='echo hi')
assert context.exception.args[0] == (
'Operation order number not provided in API mode - '
'you must use `add_op` to add operations.'
)
def _manage_worker_node_keys(concourse_config: ConcourseWorkerConfig,
sudo=True,
host=None,
state=None):
if concourse_config.tsa_public_key:
tsa_key_file = tempfile.NamedTemporaryFile(delete=False)
tsa_key_file.write(concourse_config.tsa_public_key.encode("utf8"))
files.put(
name="Write TSA public key file",
dest=concourse_config.tsa_public_key_path,
src=tsa_key_file.name,
user=concourse_config.user,
mode="600",
state=state,
host=host,
sudo=sudo,
)
if concourse_config.worker_private_key:
worker_key_file = tempfile.NamedTemporaryFile(delete=False)
worker_key_file.write(
concourse_config.worker_private_key.encode("utf8"))
files.put(
name="Write worker private key file",
dest=concourse_config.worker_private_key_path,
src=worker_key_file.name,
user=concourse_config.user,
mode="600",
state=state,
host=host,
sudo=sudo,
)
elif not host.fact.file(concourse_config.worker_private_key_path):
server.shell(
name="Generate a worker private key",
commands=[
f"{concourse_config.deploy_directory}/bin/concourse generate-key -t ssh -f {concourse_config.worker_private_key_path}" # noqa: E501
],
state=state,
host=host,
sudo=sudo,
)
def nested_op():
sleep(randint(1, 10) * 0.01)
server.shell(
name="First nested operation",
commands="echo first_nested_operation",
)
if host.name == "anotherhost":
sleep(randint(1, 10) * 0.01)
server.shell(
name="Second nested anotherhost operation",
commands="echo first_nested_operation",
)
if host.name == "somehost":
server.shell(
name="Second nested somehost operation",
commands="echo first_nested_operation",
)
from pyinfra.operations import pacman
from pyinfra.operations import server
pacman.packages(name="Install fonts",
packages=[
"adobe-source-code-pro-fonts",
"adobe-source-han-sans-jp-fonts", "noto-fonts",
"noto-fonts-cjk"
],
update=True,
upgrade=True,
sudo=True)
server.shell(name="Install fonts (AUR)",
commands=["trizen -Syu --noconfirm otf-source-han-code-jp"])
apt.repo(
{'Install VirtualBox repo'},
'deb https://download.virtualbox.org/virtualbox/debian {} contrib'.format(code_name),
)
# install kernel headers
# Note: host.fact.os_version is the same as `uname -r` (ex: '4.15.0-72-generic')
apt.packages(
{
'Install VirtualBox version {} and '
'kernel headers for {}'.format(virtualbox_version, host.fact.os_version),
},
[
'virtualbox-{}'.format(virtualbox_version),
'linux-headers-{}'.format(host.fact.os_version),
],
update=True,
)
server.shell(
{'Run vboxconfig which will stop/start VirtualBox services and build kernel modules'},
'/sbin/vboxconfig',
)
python.call(
{'Verify VirtualBox version'},
verify_virtualbox_version,
version=virtualbox_version,
)
from os import path
from pyinfra import local, state
from pyinfra.operations import server
server.shell(
name='First task operation',
commands='echo first_task_operation',
)
with state.preserve_loop_order([1, 2]) as loop_items:
for item in loop_items():
server.shell(
name='Task order loop {0}'.format(item),
commands='echo loop_{0}'.format(item),
)
server.shell(
name='2nd Task order loop {0}'.format(item),
commands='echo loop_{0}'.format(item),
)
# Import a file *relative* to this one (./another_task.py)
local.include(path.join('.', 'another_task.py'))
def test_deploy():
server.shell(commands=["echo first command"])
test_nested_deploy()
server.shell(commands=["echo second command"])
# ensure all hosts are added to each /etc/hosts file
inv = inventory.get_group('@vagrant')
for item in inv:
update_hosts_file(item.name, item.fact.ipv4_addresses['eth0'])
if host.name == '@vagrant/two':
server.hostname(
name='Set the hostname for two',
hostname='two.example.com',
)
if host.name == '@vagrant/one':
server.hostname(
name='Set the hostname for one',
hostname='one.example.com',
)
server.shell(
name='Generate vagrant ssh key',
commands=('sudo -u vagrant ssh-keygen -t rsa -C [email protected] '
'-b 4096 -N "" -q -f /home/vagrant/.ssh/id_rsa'),
)
files.get(
name='Download id_rsa.pub from one',
src='/home/vagrant/.ssh/id_rsa.pub',
dest='/tmp/one_vagrant_id_rsa.pub',
)
def test_deploy(state=None, host=None):
server.shell(commands=["echo first command"])
server.shell(commands=["echo second command"])
def test_nested_deploy():
server.shell(commands=["echo nested command"])
from pyinfra.operations import server
server.shell(name="Install ahoviewer",
commands=["trizen -Syu --noconfirm ahoviewer-git"])
def my_nested_deploy():
server.shell(
name="First nested deploy operation",
commands="echo first nested_deploy_op",
)
def my_deploy():
server.shell(
name="First deploy operation",
commands="echo first_deploy_op",
)
my_nested_deploy()
server.shell(
name="Second deploy operation",
commands="echo second_deploy_op",
)
server.shell(
name="First main operation",
commands="echo first_main_op",
)
# Create some conditional branches
if host.name == "somehost":
server.shell(
name="Second main operation",
commands="echo second_main_op",
)
elif host.name == "anotherhost":
local.include(path.join("tasks", "a_task.py"))
# Include the whole file again, but for all hosts
local.include(path.join("tasks", "a_task.py"))
# Execute the @deploy function
from pyinfra.operations import server
server.shell(name="Install Polybar",
commands=["trizen -Syu --noconfirm polybar"])
{'Create uberadmin group'},
'uberadmin',
)
# multiple groups
for group in ['wheel', 'lusers']:
server.group(
{'Create the group {}'.format(group)},
group,
)
# To see output need to run pyinfra with '-v'
server.script(
{'Hello'},
'files/hello.bash',
)
# To see output need to run pyinfra with '-v'
some_var = 'blah blah blah '
server.script_template(
{'Hello from script'},
'templates/hello2.bash.j2',
some_var=some_var,
)
# To see output need to run pyinfra with '-v'
server.shell(
{'Say Hello'},
'echo Hello',
)
