def install_node_exporter(state, host):
if not host.data.node_exporter_version:
raise DeployError(
'No node_exporter_version set for this host, refusing to install node_exporter!',
)
server.user(
name='Create the node_exporter user (Called prometheus by default)',
user='******',
shell='/sbin/nologin',
state=state,
host=host,
)
files.directory(
name='Ensure the node_exporter install directory exists',
path='{{ host.data.node_exporter_install_dir }}',
user=host.data.node_exporter_user,
group=host.data.node_exporter_user,
state=state,
host=host,
)
# Work out the filename
host.data.node_exporter_version_name = (
'node_exporter-{0}.linux-'
'amd64' if host.fact.arch == 'x86_64' else host.fact.arch).format(
host.data.node_exporter_version)
host.data.node_exporter_temp_filename = state.get_temp_filename(
'node_exporter-{0}'.format(host.data.node_exporter_version), )
download_node_exporter = files.download(
name='Download node_exporter',
src=('{{ host.data.node_exporter_download_base_url }}/'
'v{{ host.data.node_exporter_version }}/'
'{{ host.data.node_exporter_version_name }}.tar.gz'),
dest='{{ host.data.node_exporter_temp_filename }}',
state=state,
host=host,
)
# If we downloaded node_exporter, extract it!
if download_node_exporter.changed:
server.shell(
name='Extract node_exporter',
commands='tar -xzf {{ host.data.node_exporter_temp_filename }}'
' -C {{ host.data.node_exporter_install_dir }}',
state=state,
host=host,
)
files.link(
name='Symlink node_exporter to /usr/bin',
path='{{ host.data.node_exporter_bin_dir }}/node_exporter', # link
target='{{ host.data.node_exporter_install_dir }}/'
'{{ host.data.node_exporter_version_name }}/node_exporter',
state=state,
host=host,
)
def install_exporter(
state,
host,
ex_url,
ex_install_dir=None,
ex_user='******',
ex_bin_dir='/usr/local/bin',
):
if ex_install_dir is None:
ex_install_dir = '/usr/local'
ex_name, ex_bin_name = _get_names(ex_url)
server.user(
name='Create the node_exporter user (Called prometheus by default)',
user=ex_user,
shell='/sbin/nologin',
state=state,
host=host,
)
files.directory(
name='Ensure the node_exporter install directory exists',
path='{}/{}'.format(ex_install_dir, ex_name),
user=host.data.node_exporter_user,
group=host.data.node_exporter_user,
state=state,
host=host,
)
ex_temp_filename = state.get_temp_filename(ex_url, )
download_exporter = files.download(
name='Download exporter',
src=ex_url,
dest=ex_temp_filename,
state=state,
host=host,
)
# If we downloaded exporter, extract it!
if download_exporter.changed:
server.shell(
name='Extract exporter',
commands='tar -xzf {} -C {}/'.format(ex_temp_filename,
ex_install_dir),
state=state,
host=host,
)
files.link(
name='Symlink exporter to /usr/local/bin',
path='{}/{}'.format(ex_bin_dir, ex_name), # link
target='{}/{}/{}'.format(ex_install_dir, ex_name, ex_bin_name),
state=state,
host=host,
)
def test_no_invalid_op_call(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
pseudo_state.set(state)
state.in_op = True
with self.assertRaises(PyinfraError):
server.user('someuser')
state.in_op = False
state.in_deploy = True
with self.assertRaises(PyinfraError):
server.user('someuser')
def test_cli_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
pyinfra.is_cli = True
pseudo_state.set(state)
# Add op to both hosts
for name in ('anotherhost', 'somehost'):
pseudo_host.set(inventory.get_host(name))
server.shell(
'echo hi') # note this is called twice but on *the same line*
# Add op to just the second host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_host.set(inventory.get_host('anotherhost'))
first_pseudo_hash = server.user('anotherhost_user').hash
first_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
# Add op to just the first host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_host.set(inventory.get_host('somehost'))
second_pseudo_hash = server.user('somehost_user').hash
second_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
pseudo_state.reset()
pseudo_host.reset()
pyinfra.is_cli = False
# Ensure there are two ops
op_order = state.get_op_order()
assert len(op_order) == 3
# And that the two ops above were called in the expected order
assert op_order[1] == first_pseudo_hash
assert op_order[2] == second_pseudo_hash
# And that they have the expected line numbers
assert state.op_line_numbers_to_hash.get(
(0, first_pseudo_call_line)) == first_pseudo_hash
assert state.op_line_numbers_to_hash.get(
(0, second_pseudo_call_line)) == second_pseudo_hash
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host('somehost')]) == 2
assert len(state.ops[inventory.get_host('anotherhost')]) == 2
def test_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
# Add op to both hosts
add_op(state, server.shell, 'echo "hi"')
# Add op to just the second host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_state.set(state)
pseudo_host.set(inventory['anotherhost'])
first_pseudo_hash = server.user('anotherhost_user').hash
first_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
# Add op to just the first host - using the pseudo modules such that
# it replicates a deploy file.
pseudo_state.set(state)
pseudo_host.set(inventory['somehost'])
second_pseudo_hash = server.user('somehost_user').hash
second_pseudo_call_line = getframeinfo(currentframe()).lineno - 1
pseudo_state.reset()
pseudo_host.reset()
# Ensure there are two ops
op_order = state.get_op_order()
self.assertEqual(len(op_order), 3)
# And that the two ops above were called in the expected order
self.assertEqual(op_order[1], first_pseudo_hash)
self.assertEqual(op_order[2], second_pseudo_hash)
# And that they have the expected line numbers
self.assertEqual(
state.op_line_numbers_to_hash.get((0, first_pseudo_call_line)),
first_pseudo_hash,
)
self.assertEqual(
state.op_line_numbers_to_hash.get((0, second_pseudo_call_line)),
second_pseudo_hash,
)
# Ensure somehost has two ops and anotherhost only has the one
self.assertEqual(len(state.ops[inventory.get_host('somehost')]), 2)
self.assertEqual(len(state.ops[inventory.get_host('anotherhost')]), 2)
def test_cli_op_line_numbers(self):
inventory = make_inventory()
state = State(inventory, Config())
connect_all(state)
state.current_deploy_filename = __file__
pyinfra.is_cli = True
ctx_state.set(state)
# Add op to both hosts
for name in ("anotherhost", "somehost"):
ctx_host.set(inventory.get_host(name))
server.shell(
"echo hi") # note this is called twice but on *the same line*
# Add op to just the second host - using the context modules such that
# it replicates a deploy file.
ctx_host.set(inventory.get_host("anotherhost"))
first_context_hash = server.user("anotherhost_user").hash
# Add op to just the first host - using the context modules such that
# it replicates a deploy file.
ctx_host.set(inventory.get_host("somehost"))
second_context_hash = server.user("somehost_user").hash
ctx_state.reset()
ctx_host.reset()
pyinfra.is_cli = False
# Ensure there are two ops
op_order = state.get_op_order()
assert len(op_order) == 3
# And that the two ops above were called in the expected order
assert op_order[1] == first_context_hash
assert op_order[2] == second_context_hash
# Ensure somehost has two ops and anotherhost only has the one
assert len(state.ops[inventory.get_host("somehost")]) == 2
assert len(state.ops[inventory.get_host("anotherhost")]) == 2
from pyinfra import config, host
from pyinfra.operations import server
config.SUDO = True
if host.name == "@vagrant/two":
key_file = open("/tmp/one_vagrant_id_rsa.pub", "r")
key = key_file.read().strip()
server.user(
name="Add the vagrant public key from one on to two",
user="******",
public_keys=[key],
)
server.sysctl(
{'Change the fs.file-max value'},
'fs.file-max',
'100000',
persist=True,
)
if host.fact.linux_name in ['CentOS', 'RedHat']:
server.modprobe(
{'Silly example for modprobe'},
'floppy',
)
server.user(
{'Ensure user is removed'},
'kevin',
present=False,
)
# multiple users
for user in ['kevin', 'bob']:
server.user(
{'Ensure user {} is removed'.format(user)},
user,
present=False,
)
server.group(
{'Create uberadmin group'},
'uberadmin',
)
def install_hashicorp_products(hashicorp_products: List[HashicorpProduct],
state=None,
host=None):
apt.packages(
name="Ensure unzip is installed",
packages=["unzip"],
update=True,
state=state,
host=host,
)
for product in hashicorp_products:
server.user(
name=f"Create system user for {product.name}",
user=product.name,
system=True,
shell="/bin/false", # noqa: S604
state=state,
host=host,
)
if linux_family(host.fact.linux_name).lower == "debian":
cpu_arch = host.fact.debian_cpu_arch
elif linux_family(host.fact.linux_name).lower == "redhat":
cpu_arch = host.fact.redhat_cpu_arch
else:
cpu_arch = "amd64"
file_download = f"{product.name}_{product.version}_linux_{cpu_arch}.zip"
file_hashes = (
httpx.get(
"https://releases.hashicorp.com/{product_name}/{product_version}/{product_name}_{product_version}_SHA256SUMS"
.format( # noqa: E501
product_name=product.name,
product_version=product.version)).read().decode(
"utf8").strip("\n").split("\n"))
file_hash_map = {
file_hash.split()[1]: file_hash.split()[0]
for file_hash in file_hashes
}
download_destination = f"/tmp/{product.name}.zip" # noqa: S108
target_directory = product.install_directory or "/usr/local/bin/"
download_binary = files.download(
name=f"Download {product.name} archive",
src=
f"https://releases.hashicorp.com/{product.name}/{product.version}/{file_download}", # noqa: WPS221,E501
dest=download_destination,
sha256sum=file_hash_map[file_download],
state=state,
host=host,
)
server.shell(
name=f"Unzip {product.name}",
commands=[
f"unzip -o {download_destination} -d {target_directory}"
],
state=state,
host=host,
)
files.file(
name=f"Ensure {product.name} binary is executable",
path=Path(target_directory).joinpath(product.name),
assume_present=download_binary.changed,
user=product.name,
group=product.name,
mode="755",
state=state,
host=host,
)
files.directory(
name=f"Ensure configuration directory for {product.name}",
path=product.configuration_directory
or product.configuration_file.parent,
present=True,
user=product.name,
group=product.name,
recursive=True,
state=state,
host=host,
)
if hasattr(product, "data_directory"): # noqa: WPS421
files.directory(
name=f"Create data directory for {product.name}",
path=product.data_directory,
present=True,
user=product.name,
group=product.name,
recursive=True,
state=state,
host=host,
)
server.sysctl(
name='Change the fs.file-max value',
key='fs.file-max',
value='100000',
persist=True,
)
if host.fact.linux_name in ['CentOS', 'RedHat']:
server.modprobe(
name='Silly example for modprobe',
module='floppy',
)
server.user(
name='Ensure user is removed',
user='******',
present=False,
)
# multiple users
for user in ['kevin', 'bob']:
server.user(
name='Ensure user {} is removed'.format(user),
user=user,
present=False,
)
server.group(
name='Create uberadmin group',
group='uberadmin',
)
from pyinfra import config
from pyinfra.operations import files, server
config.SUDO = True
config.FAIL_PERCENT = 0
server.user(
name="Create the pyinfra user",
user="******",
)
files.file(
name="Create a file as the pyinfra user using sudo",
path="/home/pyinfra/sudo_testfile",
sudo_user="******",
)
files.file(
name="Create a file as the pyinfra user using su",
path="/home/pyinfra/su_testfile",
su_user="******",
)
from pyinfra.operations import files, server
SUDO = True
FAIL_PERCENT = 0
server.user(
name='Create the pyinfra user',
user='******',
)
files.file(
name='Create a file as the pyinfra user using sudo',
path='/home/pyinfra/sudo_testfile',
sudo_user='******',
)
files.file(
name='Create a file as the pyinfra user using su',
path='/home/pyinfra/su_testfile',
su_user='******',
)
def install_concourse(concourse_config: ConcourseBaseConfig,
state=None,
host=None):
# Create a Concourse system user
server.user(
name="Create the Concourse system user",
user=concourse_config.user,
present=True,
home=concourse_config.deploy_directory,
ensure_home=False,
shell="/bin/false", # noqa: S604
system=True,
state=state,
host=host,
)
installation_directory = (
f"{concourse_config.deploy_directory}-{concourse_config.version}")
if not host.fact.directory(installation_directory):
# Download latest Concourse release from GitHub
concourse_archive = f"https://github.com/concourse/concourse/releases/download/v{concourse_config.version}/concourse-{concourse_config.version}-linux-amd64.tgz" # noqa: E501
concourse_archive_hash = f"https://github.com/concourse/concourse/releases/download/v{concourse_config.version}/concourse-{concourse_config.version}-linux-amd64.tgz.sha1" # noqa: E501
concourse_archive_path = (
f"/tmp/concourse-{concourse_config.version}.tgz" # noqa: S108
)
files.download(
name="Download the Concourse release archive",
src=concourse_archive,
dest=concourse_archive_path,
sha1sum=httpx.get(concourse_archive_hash).read().decode(
"utf8").split()[0],
state=state,
host=host,
)
# Unpack Concourse to /opt/concourse
server.shell(
name="Extract the Concourse release archive.",
commands=[
f"tar -xvzf {concourse_archive_path}",
f"mv concourse {installation_directory}",
],
state=state,
host=host,
)
# Verify ownership of Concourse directory
files.directory(
name="Set ownership of Concourse directory",
path=installation_directory,
user=concourse_config.user,
state=state,
host=host,
)
# Link Concourse installation to target directory
active_installation_path = files.link(
name="Link Concourse installation to target directory",
path=concourse_config.deploy_directory,
target=f"{installation_directory}",
user=concourse_config.user,
symbolic=True,
present=True,
state=state,
host=host,
)
return active_installation_path.changed
path="/somelink",
target="/elsewhere",
)
files.link(
path="/somelink",
present=False,
)
files.link(
path="/somelink",
target="/elsewhere",
)
# Add/remove/add same user
server.user(user="******", )
server.user(
user="******",
present=False,
)
server.user(user="******", )
# Add/remove/add same group
server.group(group="somegroup", )
server.group(
group="somegroup",
present=False,
)
from pyinfra import host
from pyinfra.operations import files, init, server
SUDO = True
if host.fact.linux_name in ['Ubuntu']:
server.user(
{'Ensure myweb user exists'},
'myweb',
shell='/bin/bash',
)
files.directory(
{'Ensure /web exists'},
'/web',
user='******',
group='myweb',
)
files.template(
{'Create script to run inside the service'},
'templates/myweb.sh.j2',
'/usr/local/bin/myweb.sh',
mode='755',
user='******',
group='myweb',
)
files.template(
{'Create service file'},
server.sysctl(
name="Change the fs.file-max value",
key="fs.file-max",
value="100000",
persist=True,
)
if host.get_fact(LinuxName) in ["CentOS", "RedHat"]:
server.modprobe(
name="Silly example for modprobe",
module="floppy",
)
server.user(
name="Ensure user is removed",
user="******",
present=False,
)
# multiple users
for user in ["kevin", "bob"]:
server.user(
name="Ensure user {} is removed".format(user),
user=user,
present=False,
)
server.group(
name="Create uberadmin group",
group="uberadmin",
)
from pyinfra import host
from pyinfra.operations import server
SUDO = True
if host.name == '@vagrant/two':
key_file = open('/tmp/one_vagrant_id_rsa.pub', 'r')
key = key_file.read().strip()
server.user(
{'Add the vagrant public key from one on to two'},
'vagrant',
public_keys=[key],
)
from pyinfra import config, host
from pyinfra.facts.server import LinuxName
from pyinfra.operations import files, init, server
config.SUDO = True
if host.get_fact(LinuxName) in ["Ubuntu"]:
server.user(
name="Ensure myweb user exists",
user="******",
shell="/bin/bash",
)
files.directory(
name="Ensure /web exists",
path="/web",
user="******",
group="myweb",
)
files.template(
name="Create script to run inside the service",
src="templates/myweb.sh.j2",
dest="/usr/local/bin/myweb.sh",
mode="755",
user="******",
group="myweb",
)
files.template(
def install_prometheus(state, host):
if not host.data.prometheus_version:
raise DeployError(
'No prometheus_version set for this host, refusing to install prometheus!',
)
server.user(
name='Create the prometheus user',
user='******',
shell='/sbin/nologin',
state=state,
host=host,
)
files.directory(
name='Ensure the prometheus data directory exists',
path='{{ host.data.prometheus_data_dir }}',
user=host.data.prometheus_user,
group=host.data.prometheus_user,
state=state,
host=host,
)
files.directory(
name='Ensure the prometheus install directory exists',
path='{{ host.data.prometheus_install_dir }}',
user=host.data.prometheus_user,
group=host.data.prometheus_user,
state=state,
host=host,
)
# Work out the filename
host.data.prometheus_version_name = ('prometheus-{0}.linux-'
'amd64' if host.fact.arch == 'x86_64'
else host.fact.arch).format(
host.data.prometheus_version)
host.data.prometheus_temp_filename = state.get_temp_filename(
'prometheus-{0}'.format(host.data.prometheus_version), )
download_prometheus = files.download(
name='Download prometheus',
src=('{{ host.data.prometheus_download_base_url }}/'
'v{{ host.data.prometheus_version }}/'
'{{ host.data.prometheus_version_name }}.tar.gz'),
dest='{{ host.data.prometheus_temp_filename }}',
state=state,
host=host,
)
# If we downloaded prometheus, extract it!
if download_prometheus.changed:
server.shell(
name='Extract prometheus',
commands='tar -xzf {{ host.data.prometheus_temp_filename }}'
' -C {{ host.data.prometheus_install_dir }}',
state=state,
host=host,
)
files.link(
name='Symlink prometheus to /usr/bin',
path='{{ host.data.prometheus_bin_dir }}/prometheus', # link
target=
'{{ host.data.prometheus_install_dir }}/{{ host.data.prometheus_version_name }}/prometheus',
state=state,
host=host,
)
from pyinfra import host
from pyinfra.operations import files, init, server
SUDO = True
if host.fact.linux_name in ['Ubuntu']:
server.user(
name='Ensure myweb user exists',
user='******',
shell='/bin/bash',
)
files.directory(
name='Ensure /web exists',
path='/web',
user='******',
group='myweb',
)
files.template(
name='Create script to run inside the service',
src='templates/myweb.sh.j2',
dest='/usr/local/bin/myweb.sh',
mode='755',
user='******',
group='myweb',
)
files.template(
name='Create service file',
def install_caddy(caddy_config: CaddyConfig, state=None, host=None):
if caddy_config.plugins:
caddy_user = "******"
server.user(
name="Create system user for Caddy",
user=caddy_user,
system=True,
ensure_home=False,
state=state,
host=host,
)
caddy_install = files.download(
name="Download custom build of Caddy",
dest="/usr/local/bin/caddy",
src=caddy_config.custom_download_url(),
mode=DEFAULT_DIRECTORY_MODE,
state=state,
host=host,
)
files.directory(
name="Create Caddy configuration directory",
path="/etc/caddy/",
user=caddy_user,
group=caddy_user,
present=True,
recursive=True,
state=state,
host=host,
)
files.directory(
name="Create Caddy configuration directory",
path=caddy_config.data_directory,
user=caddy_user,
group=caddy_user,
present=True,
recursive=True,
state=state,
host=host,
)
files.template(
name="Create SystemD service definition for Caddy",
dest="/usr/lib/systemd/system/caddy.service",
src=Path(__file__).parent.joinpath("templates/caddy.service.j2"),
state=state,
host=host,
)
else:
apt.key(
name="Add Caddy repository GPG key",
src="https://dl.cloudsmith.io/public/caddy/stable/gpg.key",
state=state,
host=host,
)
apt.repo(
name="Set up Caddy APT repository",
src="deb https://dl.cloudsmith.io/public/caddy/stable/deb/debian any-version main", # noqa: E501
present=True,
filename="caddy.list",
state=state,
host=host,
)
caddy_install = apt.packages(
name="Install Caddy from APT",
packages=["caddy"],
present=True,
latest=True,
update=True,
state=state,
host=host,
)
if caddy_config.log_file:
files.directory(
name="Crate Caddy log directory",
path=caddy_config.log_file.parent,
user=caddy_user,
present=True,
state=state,
host=host,
)
return caddy_install.changed
from pyinfra import host
from pyinfra.operations import server
SUDO = True
if host.name == '@vagrant/two':
key_file = open('/tmp/one_vagrant_id_rsa.pub', 'r')
key = key_file.read().strip()
server.user(
name='Add the vagrant public key from one on to two',
user='******',
public_keys=[key],
)
