def configure_prometheus(state, host, enable_service=True):
# Setup prometheus init
generate_service = files.template(
state,
host,
{'Upload the prometheus systemd unit file'},
get_template_path('prometheus.service.j2'),
'/etc/systemd/system/prometheus.service',
)
# Configure prometheus
files.template(
state,
host,
{'Upload the prometheus config file'},
get_template_path('prometheus.yml.j2'),
'/etc/default/prometheus.yml',
)
# Start (/enable) the prometheus service
op_name = 'Ensure prometheus service is running'
if enable_service:
op_name = '{0} and enabled'.format(op_name)
init.systemd(
state,
host,
{op_name},
'prometheus',
restarted=True,
enabled=enable_service,
daemon_reload=generate_service.changed,
)
def configure_etcd(state, host, enable_service=True):
# Setup etcd init
generate_service = files.template(
state,
host,
{'Upload the etcd systemd unit file'},
get_template_path('etcd.service.j2'),
'/etc/systemd/system/etcd.service',
)
# Configure etcd
files.template(
state,
host,
{'Upload the etcd env file'},
get_template_path('etcd.conf.j2'),
'{{ host.data.etcd_env_file }}',
# Cluster (peers)
cluster_node_urls=get_cluster_node_urls(state.inventory),
get_urls=get_urls,
)
# Start (/enable) the etcd service
op_name = 'Ensure etcd service is running'
if enable_service:
op_name = '{0} and enabled'.format(op_name)
init.systemd(
state,
host,
{op_name},
'etcd',
enabled=enable_service,
daemon_reload=generate_service.changed,
)
def install_controller_services(state, host):
apt.packages(
state,
host,
{'Install packages'},
[
'apache2',
'mariadb-server',
'rabbitmq-server',
'memcached',
'python-memcache',
'python-pymysql',
],
)
# MariaDB
mariadb_configure = files.template(
state,
host,
{'Generate MariaDB config'},
get_template_path('mysql.cnf.j2'),
'/etc/mysql/mariadb.conf.d/99-openstack.cnf',
)
init.service(
state,
host,
{'Restart MariadB'},
'mysql',
restarted=mariadb_configure.changed,
)
# RabbitMQ
server.shell(
state,
host,
{'Setup RabbitMQ user'},
(
'rabbitmqctl add_user openstack {{ host.data.rabbitmq_password }} || true',
'rabbitmqctl set_permissions openstack ".*" ".*" ".*"',
),
)
# Memcached
memcached_configure = files.template(
state,
host,
{'Generate memcached config'},
get_template_path('memcached.conf.j2'),
'/etc/memcached.conf',
)
init.service(
state,
host,
{'Restart memcached'},
'memcached',
restarted=memcached_configure.changed,
)
def install_glance_service(state, host):
install_glance = apt.packages(
state,
host,
{'Install glance'},
['glance'],
)
if install_glance.changed:
create_database(state, host, 'glance')
create_service_user(state, host, 'glance', 'image')
create_service_endpoints(state, host, 'image', ':9292')
generate_glance_api_config = files.template(
state,
host,
{'Generate glance-api config'},
get_template_path('glance-api.conf.j2'),
'/etc/glance/glance-api.conf',
)
generate_glance_registry_config = files.template(
state,
host,
{'Generate glance-registry config'},
get_template_path('glance-registry.conf.j2'),
'/etc/glance/glance-registry.conf',
)
server.shell(
state,
host,
{'Sync the glance database'},
'glance-manage db_sync',
)
should_restart_glance = (generate_glance_api_config.changed
or generate_glance_registry_config.changed)
init.service(
state,
host,
{'Restart glance-registry'},
'glance-registry',
restarted=should_restart_glance,
)
init.service(
state,
host,
{'Restart glance-api'},
'glance-api',
restarted=should_restart_glance,
)
def configure_node_exporter(state, host, enable_service=True, extra_args=None):
op_name = 'Ensure node_exporter service is running'
if enable_service:
op_name = '{0} and enabled'.format(op_name)
if host.fact.linux_distribution['major'] >= 16:
# Setup node_exporter init
generate_service = files.template(
state,
host,
{'Upload the node_exporter systemd unit file'},
get_template_path('node_exporter.service.j2'),
'/etc/systemd/system/node_exporter.service',
extra_args=extra_args,
)
init.systemd(
state,
host,
{op_name},
'node_exporter',
running=True,
restarted=generate_service.changed,
daemon_reload=generate_service.changed,
enabled=enable_service,
)
elif host.fact.linux_distribution['major'] == 14:
generate_service = files.template(
state,
host,
{'Upload the node_exporter init.d file'},
get_template_path('init.d.j2'),
'/etc/init.d/node_exporter',
mode=755,
ex_name='node_exporter',
ex_bin_dir=host.data.node_exporter_bin_dir,
ex_user=host.data.node_exporter_user,
extra_args=extra_args,
)
# Start (/enable) the prometheus service
init.d(
state,
host,
{op_name},
'node_exporter',
running=True,
restarted=generate_service.changed,
reloaded=generate_service.changed,
enabled=enable_service,
)
def install_neutron_node(state, host, nova=False):
files.directory(
{'Create neutron config directory'},
'/etc/neutron',
)
files.template(
state,
host,
{'Generate neutron config'},
get_template_path('neutron-node.conf.j2'),
'/etc/neutron/neutron.conf',
nova=nova,
)
def install_neutron_service(state, host, nova=False, placement=False):
create_database(state, host, 'neutron')
neutron_install = apt.packages(
state,
host,
{'Install neutron network controller packages'},
[
'neutron-server',
'neutron-plugin-ml2',
],
)
if neutron_install.changed:
create_service_user(state, host, 'neutron', 'network')
create_service_endpoints(state, host, 'network', ':9696')
neutron_configure = files.template(
state,
host,
{'Generate neutron config'},
get_template_path('neutron-controller.conf.j2'),
'/etc/neutron/neutron.conf',
nova=nova,
)
ml2_plugin_configure = files.template(
state,
host,
{'Generate neutron ml2 plugin config'},
get_template_path('ml2_conf.ini.j2'),
'/etc/neutron/plugins/ml2/ml2_conf.ini',
)
server.shell(
{'Sync the neutron database'},
'''
neutron-db-manage --config-file /etc/neutron/neutron.conf \
--config-file /etc/neutron/plugins/ml2/ml2_conf.ini \
upgrade head
''',
)
init.service(
state,
host,
{'Restart neutron-server'},
'neutron-server',
restarted=neutron_configure.changed or ml2_plugin_configure.changed,
)
def install_horizon_service(state, host):
apt.packages(
{'Install openstack-dashboard'},
['openstack-dashboard'],
)
generate_horizon_config = files.template(
state,
host,
{'Generate horizon config'},
get_template_path('local_settings.py.j2'),
'/etc/openstack-dashboard/local_settings.py',
)
server.shell(
state,
host,
{'Give www-data access to the secret key'},
'chown www-data /var/lib/openstack-dashboard/secret_key',
)
init.service(
state,
host,
{'Restart apache2'},
'apache2',
restarted=generate_horizon_config.changed,
)
def install_nova_node(state,
host,
neutron=False,
placement=False,
ceilometer=False):
apt.packages(
state,
host,
{'Install nova-compute'},
['nova-compute'],
)
files.put(
{'Upload interface.j2'},
get_template_path('interfaces.j2'),
'/etc/nova/interfaces.j2',
)
nova_configure = files.template(
state,
host,
{'Generate nova config'},
get_template_path('nova-node.conf.j2'),
'/etc/nova/nova.conf',
neutron=neutron,
placement=placement,
ceilometer=ceilometer,
)
nova_compute_configure = files.template(
state,
host,
{'Generate nova-compute config'},
get_template_path('nova-compute.conf.j2'),
'/etc/nova/nova-compute.conf',
)
init.service(
state,
host,
{'Restart nova-compute'},
'nova-compute',
restarted=nova_configure.changed or nova_compute_configure.changed,
)
def wireguard(state: Optional[State] = None,
host: Optional[Host] = None) -> None:
"""Install wireguard."""
if host.fact.os == "Linux":
if host.fact.linux_distribution["release_meta"]["ID"] in [
"debian", "ubuntu"
]:
apt.packages(packages=["wireguard"], state=state, host=host)
else:
python.raise_exception(exception_class=NotImplementedError,
state=state,
host=host)
else:
python.raise_exception(exception_class=NotImplementedError,
state=state,
host=host)
files.template(
template_filename="templates/wg0.conf.j2",
remote_filename="/etc/wireguard/wg0.conf",
create_remote_dir=True,
state=state,
host=host,
wg_private_key=environ["WG_PRIVATE_KEY"],
wg_port=environ["WG_PORT"],
wg_interface=environ["WG_INTERFACE"],
wg_peer_1_public_key=environ["WG_PEER_1_PUBLIC_KEY"],
wg_peer_2_public_key=environ["WG_PEER_2_PUBLIC_KEY"],
)
server.sysctl(key="net.ipv4.ip_forward", value=1, state=state, host=host)
server.sysctl(key="net.ipv6.conf.all.forwarding",
value=1,
state=state,
host=host)
server.shell(commands=["wg-quick up wg0"],
success_exit_codes=[0, 1],
state=state,
host=host)
def install_ceilometer_service(state, host):
ceilometer_install = apt.packages(
state,
host,
{'Install ceilometer controller packages'},
[
'ceilometer-collector',
'ceilometer-agent-central',
'ceilometer-agent-notification',
'python-ceilometerclient',
],
)
if ceilometer_install.changed:
create_service_user(state, host, 'ceilometer', 'metering')
create_service_user(state, host, 'gnocchi', 'metric')
create_service_endpoints(state, host, 'metric', ':8041')
ceilometer_configure = files.template(
state,
host,
{'Generate ceilometer config'},
get_template_path('ceilometer-controller.conf.j2'),
'/etc/ceilometer/ceilometer.conf',
)
# server.shell(
# state, host,
# {'Create ceilometer resoucres in gnocchi'},
# 'ceilometer-upgrade --skip-metering-database',
# )
for service in (
'ceilometer-agent-central',
'ceilometer-agent-notification',
'ceilometer-collector',
):
init.service(
state,
host,
{'Restart {0}'.format(service)},
service,
restarted=ceilometer_configure,
)
def install_ceilometer_agent(state, host):
apt.packages(
{'Install ceilometer-agent-compute'},
'ceilometer-agent-compute',
)
ceilometer_configure = files.template(
state,
host,
{'Generate ceilometer config'},
get_template_path('ceilometer-node.conf.j2'),
'/etc/ceilometer/ceilometer.conf',
)
init.service(
state,
host,
{'Restart ceilometer-agent-compute'},
'ceilometer-agent-compute',
restarted=ceilometer_configure.changed,
)
def install_chrony_node(state, host):
apt.packages(
state,
host,
{'Install chrony'},
['chrony'],
)
generate_chrony_config = files.template(
state,
host,
{'Generate chrony config'},
get_template_path('chrony-node.conf.j2'),
'/etc/chrony/chrony.conf',
)
init.service(
state,
host,
{'Restart chrony'},
'chrony',
restarted=generate_chrony_config.changed,
)
def _install_agent(state, host, agent_name, template_name, config_path):
install = apt.packages(
state,
host,
{'Install {0} package'.format(agent_name)},
[agent_name],
)
configure = files.template(
state,
host,
{'Generate neutron {0} agent config'.format(agent_name)},
get_template_path(template_name),
config_path,
)
init.service(
state,
host,
{'Restart {0}'.format(agent_name)},
agent_name,
restarted=install.changed or configure.changed,
)
def configure_node_exporter(state, host, enable_service=True):
# Setup node_exporter init
generate_service = files.template(
state,
host,
{'Upload the node_exporter systemd unit file'},
get_template_path('node_exporter.service.j2'),
'/etc/systemd/system/node_exporter.service',
)
# Start (/enable) the node_exporter service
op_name = 'Ensure node_exporter service is running'
if enable_service:
op_name = '{0} and enabled'.format(op_name)
init.systemd(
state,
host,
{op_name},
'node_exporter',
enabled=enable_service,
daemon_reload=generate_service.changed,
)
files.download(
{'Download `{}`'.format(uefi_file)},
'http://archive.ubuntu.com/ubuntu/dists/trusty/main/'
'uefi/grub2-amd64/current/grubnetx64.efi.signed',
uefi_full_path,
)
grub_dir = '{}/grub'.format(tftp_dir)
files.directory(
{'Ensure the `{}` exists'.format(grub_dir)},
grub_dir,
)
files.template(
{'Create a templated file'},
'templates/grub.cfg.j2',
'{}/grub.cfg'.format(grub_dir),
)
# configure dnsmasq
files.template(
{'Create dnsmasq configuration file'},
'templates/dnsmasq.conf.j2',
'/etc/dnsmasq.conf',
pxe_server=pxe_server,
dns_server=dns_server,
interface=interface,
dhcp_start=dhcp_start,
dhcp_end=dhcp_end,
tftp_dir=tftp_dir,
)
def install_nova_service(state, host, neutron=False, placement=False):
create_database(state, host, 'nova')
create_database(state, host, 'nova_api', name='nova')
create_database(state, host, 'nova_cell0', name='nova')
packages = [
'nova-api',
'nova-conductor',
'nova-consoleauth',
'nova-novncproxy',
'nova-scheduler',
]
if placement:
packages.append('nova-placement-api')
nova_install = apt.packages(
state,
host,
{'Install nova compute controller packages'},
packages,
)
if nova_install.changed:
create_service_user(state, host, 'nova', 'compute')
create_service_endpoints(state, host, 'compute',
':8774/v2.1/%\(tenant_id\)s')
if packages:
create_service_user(state, host, 'placement', 'placement')
create_service_endpoints(state, host, 'placement', ':8778')
nova_configure = files.template(
state,
host,
{'Generate nova config'},
get_template_path('nova-controller.conf.j2'),
'/etc/nova/nova.conf',
neutron=neutron,
placement=placement,
)
server.shell(
state,
host,
{'Sync the nova api database'},
'nova-manage api_db sync',
)
if nova_install.changed:
server.shell(
state,
host,
{'Setup nova cells and sync db'},
(
'nova-manage cell_v2 map_cell0',
'nova-manage cell_v2 create_cell --name=cell1 --verbose',
),
)
server.shell(
state,
host,
{'Sync the nova database'},
'nova-manage db sync',
)
for service in (
'nova-api',
'nova-consoleauth',
'nova-scheduler',
'nova-conductor',
'nova-novncproxy',
):
init.service(
state,
host,
{'Restart {0}'.format(service)},
service,
restarted=nova_configure.changed,
)
'/home/vagrant/file.txt',
mode=777,
)
# and sync directories
files.sync(
'files',
'/home/vagrant/example_files',
user='******',
group='pyinfra',
delete=True,
sudo=True,
)
# Generate files from local jinja2 templates
files.template(
'templates/template.txt.j2',
'/home/vagrant/template.txt',
)
# Execute arbitrary shell commands
server.shell([
'echo "Shell command"',
'echo "My hostname is {{ host.fact.hostname }}"',
])
# and scripts
server.script(
'files/test.sh',
)
# Manage init systems
init.service(
'cron',
{'Sync a local directory with remote'},
'files/tempdir',
'/tmp/tempdir',
)
if host.fact.file('/etc/os-release'):
files.get(
{'Download a file from a remote'},
'/etc/os-release',
'/tmp/whocares',
)
foo_variable = 'This is some foo variable contents'
files.template(
{'Create a templated file'},
'templates/foo.j2',
'/tmp/foo',
foo_variable=foo_variable,
)
files.link(
{'Create link /etc/issue2 that points to /etc/issue'},
'/etc/issue2',
'/etc/issue',
)
# Note: The directory /tmp/secret will get created with the default umask.
files.file(
{'Create /tmp/secret/file'},
'/tmp/secret/file',
mode='600',
user='******',
from pyinfra import host, inventory
from pyinfra.modules import files, puppet
SUDO = True
USE_SUDO_LOGIN = True
if host in inventory.get_group('master_servers'):
files.template(
{'Create a puppet manifest'},
'templates/environments/production/manifests/httpd.pp.j2',
'/etc/puppetlabs/code/environments/production/manifests/httpd.pp',
)
if host in inventory.get_group('agent_servers'):
# Either 'USE_SUDO_LOGIN=True' or 'USE_SU_LOGIN=True' for
# puppet.agent() as `puppet` is added to the path in
# the .bash_profile.
# We also expect a return code of:
# 0=no changes or 2=changes applied
puppet.agent(
{'Run the puppet agent'},
success_exit_codes=[0, 2],
)
def install_keystone_service(state, host):
create_database(state, host, 'keystone')
keystone_install = apt.packages(
state, host,
{'Install keystone'},
['keystone'],
)
files.template(
state, host,
{'Generate keystone config'},
get_template_path('keystone.conf.j2'),
'/etc/keystone/keystone.conf',
)
server.shell(
state, host,
{'Sync the keystone database'},
'keystone-manage db_sync',
)
# Bootstrap keystone: only do this if newly installed
if keystone_install.changed:
server.shell(
state, host, '''
keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone
keystone-manage credential_setup --keystone-user keystone --keystone-group keystone
keystone-manage bootstrap \
--bootstrap-password {{ host.data.admin_password }} \
--bootstrap-admin-url http://{{ host.data.controller_host }}:35357/v3/ \
--bootstrap-internal-url http://{{ host.data.controller_host }}:35357/v3/ \
--bootstrap-public-url http://{{ host.data.controller_host }}:5000/v3/ \
--bootstrap-region-id RegionOne
''')
update_apache_config = files.line(
state, host,
{'Set ServerName in apache2 config'},
'/etc/apache2/apache2.conf',
'ServerName.*',
replace='ServerName {{ host.data.ssh_hostname }}',
)
init.service(
state, host,
{'Restart apache2'},
'apache2',
restarted=update_apache_config.changed,
)
if keystone_install.changed:
server.shell(
state, host,
{'Create initial projects/users/roles'},
(
'openstack project create --domain default service',
'openstack project create --domain default user',
'openstack user create --domain default --password hamble user-1',
'openstack role create user',
'openstack role add --project user --user user-1 user',
),
env=make_admin_env(host),
)
for i in range(2):
server.shell(
{'Loop-{0} main operation'.format(i)},
'echo loop_{0}_main_operation'.format(i),
)
files.file(
{'Third main operation'},
'files/a_file',
'/a_file',
)
with state.preserve_loop_order([1, 2]) as loop_items:
for item in loop_items():
server.shell(
{'Order loop {0}'.format(item)},
'echo loop_{0}'.format(item),
)
server.shell(
{'2nd Order loop {0}'.format(item)},
'echo loop_{0}'.format(item),
)
if host.name == 'somehost':
files.template(
{'Final limited operation'},
'templates/a_template.j2',
'/a_template',
is_template=True,
)
)
files.put(
{'Install hockeypuck service'},
project_root + '/debian/hockeypuck.service',
'/etc/systemd/system/hockeypuck.service',
mode='644',
user='******',
group='root',
sudo=True,
)
files.template(
{'Configure hockeypuck'},
'hockeypuck.conf',
'/etc/hockeypuck/hockeypuck.conf',
mode='644',
sudo=True,
peers=host.data.peers,
)
apt.packages(
{'Install postgresql'},
['postgresql'],
latest=True,
sudo=True,
)
postgresql.role(
{'Create hockeypuck database role'},
'hockeypuck',
login=True,
# setup pxe infra
if host.fact.linux_name == 'Ubuntu':
apt.packages(
{'Install packages'},
['dnsmasq', 'nfs-kernel-server', 'syslinux', 'pxelinux'],
update=True,
)
# configure dnsmasq
files.template(
{'Create dnsmasq configuration file'},
'templates/dnsmasq.conf.j2',
'/etc/dnsmasq.conf',
pxe_server=pxe_server,
interface=interface,
dhcp_start=dhcp_start,
dhcp_end=dhcp_end,
)
# create necessary directories
dirs = [
'/netboot/tftp',
'/netboot/nfs',
'/netboot/tftp/pxelinux.cfg',
'/mnt',
'/netboot/nfs/ubuntu1804',
'/netboot/tftp/ubuntu1804',
]
for dir in dirs:
from pyinfra import host
from pyinfra.modules import files, server
# Executing as the SSH user (vagrant):
#
# Generate files from local jinja2 templates
files.template(
{'Generate/upload templates/template.txt.j2'},
'templates/template.txt.j2',
'/home/vagrant/template.txt',
)
server.shell(
{'Execute some shell commands'},
[
'echo "Shell command"',
'echo "My hostname is {{ host.fact.hostname }}"',
],
)
# and scripts
server.script(
{'Run the files/test.sh script'},
'files/test.sh',
)
# Copy local files to remote host
files.put(
{'Upload files/file.txt'},
'files/file.txt',
'/home/vagrant/file.txt',
)
# TODO: should reboot after SELINUX is disabled (how to check/easy way to reboot)
# TODO: how to determine when reboot is complete
# TODO: run sestatus
if host in masters:
install = yum.packages(
{'Install puppet server'},
['puppetserver'],
)
config = files.template(
{'Manage the puppet master configuration'},
'templates/master_puppet.conf.j2',
'/etc/puppetlabs/puppet/puppet.conf',
)
# TODO: tune always shows as changed
# I think it should only show as changed if there really was a difference.
# Might have to add a suffix to the sed -i option, then move file only if
# there is a diff. Maybe?
tune = files.line(
{'Tune the puppet server jvm to only use 1gb'},
'/etc/sysconfig/puppetserver',
r'^JAVA_ARGS=.*$',
replace='JAVA_ARGS=\\"-Xms1g -Xmx1g -Djruby.logger.class=com.puppetlabs.'
'jruby_utils.jruby.Slf4jLogger\\"',
)
{'Ensure myweb user exists'},
'myweb',
shell='/bin/bash',
)
files.directory(
{'Ensure /web exists'},
'/web',
user='******',
group='myweb',
)
files.template(
{'Create script to run inside the service'},
'templates/myweb.sh.j2',
'/usr/local/bin/myweb.sh',
mode='755',
user='******',
group='myweb',
)
files.template(
{'Create service file'},
'templates/myweb.service.j2',
'/etc/systemd/system/myweb.service',
mode='755',
user='******',
group='root',
)
files.template(
{'Create index.html'},
