class Voltoapp(Component):
apprepository = Attribute(str, configuration['apprepository'])
address = Attribute(Address, 'localhost:3000')
razzleapipath = Attribute(str, 'http://localhost:11080/Plone') # TODO haproxy port
def configure(self):
self.provide('voltoapp', self)
# TODO Try to avoid dependency cycle without setting dirty=True
# self.varnish = self.require_one('varnish:http', reverse=True, dirty=True)
self += Clone(
self.apprepository,
branch='master')
def verify(self):
# raise UpdateNeeded() # enforce rebuild of Volto app
self.assert_no_changes()
if not os.path.exists(self.workdir + '/build'):
raise UpdateNeeded()
def update(self):
# yarn production build
if not os.path.exists(self.workdir + "/node_modules"):
self.cmd("yarn")
port = self.address.connect.port
voltoportandrazzle = 'PORT={} RAZZLE_API_PATH={}'.format( \
port, self.razzleapipath)
self.cmd('{} yarn build'.format(voltoportandrazzle or ''))
self.log("Voltoapp rebuild with {}".format(voltoportandrazzle))
class Varnish(Component):
address = Attribute(Address, '127.0.0.1:11090')
control_port = Attribute(int, '11091')
daemon = ''
daemonargs = ''
def configure(self):
self.provide('varnish:http', self)
self.purgehosts = self.require('zope:http')
self.voltoapp = self.require_one('voltoapp')
self.haproxy = self.require_one('haproxy:frontend')
self += VirtualEnv('3.8')
self += Build(
'http://varnish-cache.org/_downloads/varnish-6.5.1.tgz',
checksum=
'sha256:11964c688f9852237c99c1e327d54dc487549ddb5f0f5aa7996e521333d7cdb5',
)
self += File('websiteplone.vcl', source='websiteplone.vcl')
self.daemon = 'sbin/varnishd'
self.daemonargs = self.expand(
'-F -f {{component.workdir}}/websiteplone.vcl '
'-T localhost:{{component.control_port}} '
'-a {{component.address.listen}} '
'-p thread_pool_min=10 '
'-p thread_pool_max=50 '
'-s malloc,250M '
'-n websitesomething')
self += PurgeCache()
class SSHKeyPair(Component):
"""Install SSH user and host keys.
User keys are read from the secrets file and written to
~/.ssh/id_rsa{,.pub} and/or ~/.ssh/id_ed25519{,.pub}."""
# RSA-keys
id_rsa = None
id_rsa_pub = None
# ed25510-keys
id_ed25519 = None
id_ed25519_pub = None
scan_hosts = Attribute(list, '')
provide_itself = Attribute('literal', True)
purge_unmanaged_keys = Attribute('literal', False)
def configure(self):
if self.provide_itself:
self.provide('sshkeypair', self)
self += Directory('~/.ssh', mode=0o700)
# RSA
if self.id_rsa:
self += File('~/.ssh/id_rsa',
content=self.id_rsa,
mode=0o600)
elif self.purge_unmanaged_keys:
self += Purge('~/.ssh/id_rsa')
if self.id_rsa_pub:
self += File('~/.ssh/id_rsa.pub',
content=self.id_rsa_pub)
# ED25519
if self.id_ed25519:
self += File('~/.ssh/id_ed25519',
content='{}\n'.format(self.id_ed25519),
mode=0o600)
elif self.purge_unmanaged_keys:
self += Purge('~/.ssh/id_ed25519')
if self.id_ed25519_pub:
self += File('~/.ssh/id_ed25519.pub',
content=self.id_ed25519_pub)
# ScanHost
for host in self.scan_hosts:
self += ScanHost(host)
class DNSProblem(Component):
attribute_with_problem = Attribute(Address,
default_conf_string="isnotahostname")
def configure(self):
self.require("application")
class Test(Component):
address = Attribute(Address, "default:8080")
def configure(self):
self += File("test", content="asdf {{component.address.listen}}")
self += Buildout(version="2.3.1", python="2.7", setuptools="17.1")
class PFAPostfix(Component):
address = Attribute(Address, 'localhost:25')
def configure(self):
self.address.listen.host_v6 = resolve_v6(self.address)
self.db = self.require_one('pfa::database')
self.keypair = self.require_one('keypair::mail')
self.provide('postfix', self.address)
self += File('/etc/postfix/myhostname',
content=self.address.connect.host)
self += File('/etc/postfix/main.d/40_local.cf',
source=self.resource('local.cf'))
self += File('postfixadmin_virtual_alias',
source=self.resource('postfixadmin_virtual_alias'))
self += File('postfixadmin_virtual_domains',
source=self.resource('postfixadmin_virtual_domains'))
self += File('postfixadmin_virtual_sender_login',
source=self.resource('postfixadmin_virtual_sender_login'))
self += File('postfixadmin_virtual_mailboxes',
source=self.resource('postfixadmin_virtual_mailboxes'))
def resource(self, filename):
return os.path.join(os.path.dirname(__file__), 'postfix', filename)
class BasePackages(Component):
packages = Attribute(
'literal', """[
'build-essential',
'emacs-nox',
'dnsutils',
'git',
'htop',
'httpie',
'jq',
'mc',
'mosh',
'python3-dev',
'python-is-python3',
'rsync',
'screen',
'unzip',
'zip',
]""")
def configure(self):
for name in self.packages:
self += Package(name)
# Allow accessing (mostly python) software installed by batou
self += File('/root', ensure='directory', mode=0o755)
self += User('wosc', home='/home/wosc')
self += GroupMember('sudo', user='******')
self += File('/etc/motd', is_template=False)
self += File('/etc/ssh/sshd_config.d/cyberduck.conf',
source='ssh.conf',
is_template=False)
class BaseInstance(Component):
workdir = '{{component.zope.workdir}}'
address = Attribute(Address, '127.0.0.1:11991')
script_id = "instance1"
def configure(self):
self.provide('zope:http', self)
class Test(Component):
address = Attribute(Address, 'default:8080')
def configure(self):
self += File('test', content='asdf {{component.address.listen}}')
self += Buildout(version='2.3.1',
python='2.7',
setuptools='17.1')
class ElasticSearch(Component):
# collective.elastic.ingest
# collective.elastic.plone
uri = Attribute(
str,
'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.0-linux-x86_64.tar.gz'
)
checksum = Attribute(
str,
'sha512:5c159bdf0d6e140a2bee5fbb1c379fbe23b0ea39b01d715564f02e4674b444b065a8abfda86440229c4b70defa175722c479b60009b7eef7b3de66e2339aacea'
)
def configure(self):
self.provide('elasticsearch', self)
download = Download(self.uri, checksum=self.checksum)
self += download
self += Extract(download.target, create_target_dir=False, strip=1)
class CronTab(Component):
crontab_template = os.path.join(os.path.dirname(__file__), 'resources',
'crontab')
mailto = Attribute(str, None)
purge = False
# Dict of additional environment variables
env = Attribute('literal', '{}')
def configure(self):
self.jobs = self.require(CronJob.key, host=self.host, strict=False)
if self.purge and self.jobs:
raise ConfigurationError(
'Found cron jobs, but expecting an empty crontab.')
elif not self.purge and not self.jobs:
raise ConfigurationError('No cron jobs found.', self)
self.jobs.sort(key=lambda job: job.command + ' ' + job.args)
self.crontab = File('crontab', source=self.crontab_template)
self += self.crontab
class CronTab(Component):
crontab_template = os.path.join(os.path.dirname(__file__), "resources",
"crontab")
mailto = Attribute(default=None)
purge = False
# Dict of additional environment variables
env = Attribute("literal", default_conf_string="{}")
def configure(self):
self.jobs = self.require(CronJob.key, host=self.host, strict=False)
if self.purge and self.jobs:
raise ConfigurationError(
"Found cron jobs, but expecting an empty crontab.")
elif not self.purge and not self.jobs:
raise ConfigurationError("No cron jobs found.", self)
self.jobs.sort(key=lambda job: job.command + " " + job.args)
self.crontab = File("crontab", source=self.crontab_template)
self += self.crontab
class Pm2(Component):
pm2prefix = Attribute(str, 'local.mydomain.ch-')
def configure(self):
# self.provide('pm2', self)
self.voltoapp = self.require_one('voltoapp')
self.varnish = self.require_one('varnish:http')
self.zopecommon = self.require_one('zopecommon')
self.elasticsearch = self.require_one('elasticsearch')
self += File('website.pm2.config.js', source='website.pm2.config.js')
self += RestartTasks('all')
class Zope(Component):
backupsdir = Attribute(str, '')
adminpw = Attribute(str, 'admin')
zeoaddress = Attribute(Address, '127.0.0.1:11981')
buildoutuser = Attribute(str, 'plone')
def configure(self):
self.provide('zopecommon', self)
self.common = self.require_one('common', host=self.host)
self.zope_instances = self.require('zope:http')
self.zope_instances.sort(key=lambda s: s.script_id)
self.backupsdir = self.backupsdir or self.expand(
'{{component.workdir}}/var/backup')
config = File('buildout.cfg',
source='buildout.cfg',
template_context=self)
buildout_general = File('buildout_general.cfg',
source='buildout_general.cfg',
template_context=self)
additional_config = [
buildout_general,
Directory('profiles', source='profiles')
]
self += Buildout(python='3.8',
version=self.common.zc_buildout,
setuptools=self.common.setuptools,
config=config,
additional_config=additional_config)
self += InstallPythonPackages()
# some ElasticSearch, Celery configuration
self += File('elasticsearch-mappings.json',
source='elasticsearch-mappings.json')
self += File('elasticsearch-preprocessings.json',
source='elasticsearch-preprocessings.json')
self += File('.env', source='_env', template_context=self)
class KeyPair(Component):
namevar = 'name'
crt = None
key = None
base_path = Attribute(str, '')
provide_itself = Attribute(bool, True)
def configure(self):
self.crt_file = File(
os.path.join(self.base_path, '{}.crt'.format(self.name)),
content=self.crt)
self += self.crt_file
self.key_file = File(
os.path.join(self.base_path, '{}.key'.format(self.name)),
content=self.key,
mode=0o600)
self += self.key_file
if self.provide_itself:
self.provide('keypair::{}'.format(self.name), self)
class PFA(Component):
release = '2.92'
checksum = 'sha1:21481f6eb8f10ba05fc6fcd1fe0fd468062956f2'
address = Attribute(Address, '127.0.0.1:9001')
admin_password = None
salt = 'ab8f1b639d31875b59fa047481c581fd'
config = os.path.join(os.path.dirname(__file__), 'postfixadmin',
'config.local.php')
def configure(self):
self.db = self.require_one('pfa::database')
self.postfix = self.require_one('postfix')
self.provide('pfa', self)
self.basedir = self.map('postfixadmin')
download = Download(
'http://downloads.sourceforge.net/project/postfixadmin/'
'postfixadmin/postfixadmin-{}/postfixadmin-{}.tar.gz'.format(
self.release, self.release),
target='postfixadmin-{}.tar.gz'.format(self.release),
checksum=self.checksum)
self += download
self += Extract(download.target, target='postfixadmin.orig')
self += SyncDirectory(self.basedir,
source=self.map(
'postfixadmin.orig/postfixadmin-{}'.format(
self.release)))
self += File(self.basedir + '/config.local.php', source=self.config)
self.fpm = FPM('postfixadmin', adress=self.address)
self += self.fpm
@property
def admin_password_encrypted(self):
# password generation ported from postfixadmin/setup.php
encrypt = hashlib.sha1()
encrypt.update("{}:{}".format(self.salt, self.admin_password))
return "{}:{}".format(self.salt, encrypt.hexdigest())
class CronTab(Component):
crontab_template = pkg_resources.resource_filename('batou_ext', 'crontab')
mailto = ''
install = Attribute('literal', default='True')
def configure(self):
per_user = collections.defaultdict(list)
for job in self.require(CronJob.key, host=self.host, strict=False):
per_user[job.user].append(job)
for user, jobs in per_user.items():
jobs.sort(key=lambda job: job.command + ' ' + job.args)
self += File('crontab.%s' % user,
source=self.crontab_template,
template_args={'jobs': jobs})
if self.install:
self += InstallCrontab(user, crontab=self._)
class User(Component):
namevar = 'user'
shell = '/bin/bash'
home = Attribute(default='/srv/{{component.user}}')
def verify(self):
try:
pwd.getpwnam(self.user)
except KeyError:
raise UpdateNeeded()
def update(self):
self.cmd(
self.expand('adduser '
'--home {{component.home}} '
'--shell {{component.shell}} '
'{{component.user}}'))
class Mode(FileComponent):
mode = Attribute(default=None)
def configure(self):
super().configure()
if isinstance(self.mode, str):
try:
self.mode = int(self.mode, 8)
except ValueError:
try:
self.mode = convert_mode(self.mode)
except Exception as e:
raise batou.ConversionError(self, 'mode', self.mode,
convert_mode, e)
elif isinstance(self.mode, int):
pass
else:
raise batou.ConfigurationError(
f'`mode` is required and `{self.mode!r}` is not a valid value.`'
)
def verify(self):
try:
self._select_stat_implementation()
except AttributeError:
# Happens on systems without lstat/lchmod implementation (like
# Linux) Not sure whether ignoring it is really the right thing.
return
assert os.path.lexists(self.path)
current = self._stat(self.path).st_mode
assert stat.S_IMODE(current) == self.mode
def update(self):
self._chmod(self.path, self.mode)
def _select_stat_implementation(self):
self._stat = os.stat
self._chmod = os.chmod
if os.path.islink(self.path):
self._stat = os.lstat
self._chmod = os.lchmod
class PHP(Program):
typ = 'fcgi-program'
command = '/usr/bin/php-cgi -d error_log=/dev/stderr'
params = None
socket = 'unix:///run/supervisor/%(program_name)s.sock'
socket_owner = Attribute(default='{{component.user}}:www-data')
socket_mode = '0770'
option_names = Program.option_names + [
'socket', 'socket_owner', 'socket_mode'
]
dependencies = ()
def configure(self):
if self.params:
for key, value in self.params.items():
self.command += ' -d %s=%s' % (key, value)
super().configure()
class Patch(Component):
namevar = 'path'
path = Attribute()
# inline
source = ''
target = ''
check_source_removed = False # useful when only removing comments
# separate file
file = ''
strip = '0'
def configure(self):
if self.source and self.file:
raise ValueError('Either source or file must be given')
if not self.target:
raise ValueError('Target text is required')
def verify(self):
if not os.path.exists(self.path):
return
file = open(self.path).read()
if self.check_source_removed and self.source in file:
raise UpdateNeeded()
elif self.target not in file:
raise UpdateNeeded()
def update(self):
if self.source:
with open(self.path) as f:
contents = f.read()
contents = re.sub(
self.source, self.target, contents, flags=re.MULTILINE)
with open(self.path, 'w') as f:
f.write(contents)
else:
self.cmd('patch -d/ -p%s < %s/%s' % (
self.strip, self.parent.defdir, self.file))
class HAProxy(Component):
subdomain = '.dev'
svc_subdomain = 'testing'
memcache_settings = [
'inter 5s fastinter 1s rise 2 fall 3',
'backup',
]
memcache_port = 11211
zeit_networks = [
"127.0.0.1", # localhost
"10.100.0.0/16", # Server HH
"10.30.0.0/21", # ZON HH
"10.30.8.0/21", # ZON Ber
"10.200.200.0/21", # OpenVPN
"10.210.0.0/21", # OpenVPN
"10.110.0.0/16", # Google k8s ("GKE staging", eigentlich 10.110.16.0/20)
"10.111.48.0/20", # Google k8s production (siehe terraform-ops/.../production/gke.tf)
"10.111.32.0/20", # Google k8s staging (siehe terraform-ops/.../staging/gke-ip-masq-agent.tf)
"194.77.156.0/23", # ZON HH public
"217.13.68.0/23", # Gaertner
"192.168.0.0/16", # Docker
"34.89.176.195/32", # Data Team GCP VM
]
# We hard-code this here, but it comes from zeit-letsencrypt-acme.sh recipe
# (`node['acme.sh']['haproxy']['pem_file']`)
ssl_cert = Attribute(
default='/etc/haproxy/letsencrypt/fullchain_with_key.pem')
def configure(self):
self.nameservers = []
for line in open('/etc/resolv.conf'):
if line.startswith('nameserver'):
parts = re.split(' +', line.strip())
self.nameservers.append(parts[1])
self.varnish_hosts = self.require('varnish:http')
self += File('haproxy.cfg')
class Component2(Component):
this_does_exist = Attribute("literal", None)
class Component1(Component):
do_what_is_needed = Attribute("literal", None)
class ZEO(Component):
port = Attribute(int, "9001")
features = ["test", "test2"]
class Instance2(BaseInstance):
address = Attribute(Address, '127.0.0.1:11992')
script_id = "instance2"
class Foo(Component):
a = Attribute('list', '')
b = Attribute('list', '1,2')
c = Attribute('list', '3')
d = Attribute('list', ' 3, 3,')
e = Attribute('list', [])
class ZEO(Component):
port = Attribute(int, default_conf_string="9001")
features = ["test", "test2"]
class Roundcube(Component):
"""Configure Roundcube with database connection.
Roundcube is installed with php/fastcgi. A basic configuration for the
frontend is created but it is up to the user to fine-tune that
configuration.
"""
release = '1.1.4'
checksum = 'sha256:9bfe88255d4ffc288f5776de1cead78352469b1766d5ebaebe6e28043affe181'
address = Attribute(Address, '127.0.0.1:9000')
skin = 'larry'
support_url = 'http://localhost'
smtp_user = '******'
smtp_pass = '******'
config = os.path.join(os.path.dirname(__file__), 'config.inc.php')
def configure(self):
self.db = self.require_one('roundcube::database')
postfix = self.require_one('postfix')
self.imap_host = postfix.connect.host
self.smtp_server = postfix.connect.host
self.smtp_port = postfix.connect.port
self.basedir = self.map('roundcube')
self.provide('roundcube', self)
self += Directory('download')
download = Download(
'http://downloads.sourceforge.net/project/roundcubemail/'
'roundcubemail/{}/roundcubemail-{}-complete.tar.gz'.format(
self.release, self.release),
target='download/roundcube-{}.tar.gz'.format(self.release),
checksum=self.checksum)
self += download
self += Extract(download.target, target='roundcube.orig')
self += SyncDirectory(
self.basedir,
source=self.map(
'roundcube.orig/roundcubemail-{}'.format(self.release)))
self.db_dsnw = '{}://{}:{}@{}/{}'.format(
self.db.dbms,
self.db.username,
self.db.password,
self.db.address.connect.host,
self.db.database)
self += File(
self.basedir + '/config/config.inc.php',
source=self.config)
self.fpm = FPM('roundcube', address=self.address)
self += self.fpm
self += RoundcubeInit(self)
class Supervisor(batou.lib.supervisor.Supervisor):
pidfile = Attribute(str, 'var/supervisord.pid', map=True)
