class Voltoapp(Component): apprepository = Attribute(str, configuration['apprepository']) address = Attribute(Address, 'localhost:3000') razzleapipath = Attribute(str, 'http://localhost:11080/Plone') # TODO haproxy port def configure(self): self.provide('voltoapp', self) # TODO Try to avoid dependency cycle without setting dirty=True # self.varnish = self.require_one('varnish:http', reverse=True, dirty=True) self += Clone( self.apprepository, branch='master') def verify(self): # raise UpdateNeeded() # enforce rebuild of Volto app self.assert_no_changes() if not os.path.exists(self.workdir + '/build'): raise UpdateNeeded() def update(self): # yarn production build if not os.path.exists(self.workdir + "/node_modules"): self.cmd("yarn") port = self.address.connect.port voltoportandrazzle = 'PORT={} RAZZLE_API_PATH={}'.format( \ port, self.razzleapipath) self.cmd('{} yarn build'.format(voltoportandrazzle or '')) self.log("Voltoapp rebuild with {}".format(voltoportandrazzle))
class Varnish(Component): address = Attribute(Address, '127.0.0.1:11090') control_port = Attribute(int, '11091') daemon = '' daemonargs = '' def configure(self): self.provide('varnish:http', self) self.purgehosts = self.require('zope:http') self.voltoapp = self.require_one('voltoapp') self.haproxy = self.require_one('haproxy:frontend') self += VirtualEnv('3.8') self += Build( 'http://varnish-cache.org/_downloads/varnish-6.5.1.tgz', checksum= 'sha256:11964c688f9852237c99c1e327d54dc487549ddb5f0f5aa7996e521333d7cdb5', ) self += File('websiteplone.vcl', source='websiteplone.vcl') self.daemon = 'sbin/varnishd' self.daemonargs = self.expand( '-F -f {{component.workdir}}/websiteplone.vcl ' '-T localhost:{{component.control_port}} ' '-a {{component.address.listen}} ' '-p thread_pool_min=10 ' '-p thread_pool_max=50 ' '-s malloc,250M ' '-n websitesomething') self += PurgeCache()
class SSHKeyPair(Component): """Install SSH user and host keys. User keys are read from the secrets file and written to ~/.ssh/id_rsa{,.pub} and/or ~/.ssh/id_ed25519{,.pub}.""" # RSA-keys id_rsa = None id_rsa_pub = None # ed25510-keys id_ed25519 = None id_ed25519_pub = None scan_hosts = Attribute(list, '') provide_itself = Attribute('literal', True) purge_unmanaged_keys = Attribute('literal', False) def configure(self): if self.provide_itself: self.provide('sshkeypair', self) self += Directory('~/.ssh', mode=0o700) # RSA if self.id_rsa: self += File('~/.ssh/id_rsa', content=self.id_rsa, mode=0o600) elif self.purge_unmanaged_keys: self += Purge('~/.ssh/id_rsa') if self.id_rsa_pub: self += File('~/.ssh/id_rsa.pub', content=self.id_rsa_pub) # ED25519 if self.id_ed25519: self += File('~/.ssh/id_ed25519', content='{}\n'.format(self.id_ed25519), mode=0o600) elif self.purge_unmanaged_keys: self += Purge('~/.ssh/id_ed25519') if self.id_ed25519_pub: self += File('~/.ssh/id_ed25519.pub', content=self.id_ed25519_pub) # ScanHost for host in self.scan_hosts: self += ScanHost(host)
class DNSProblem(Component): attribute_with_problem = Attribute(Address, default_conf_string="isnotahostname") def configure(self): self.require("application")
class Test(Component): address = Attribute(Address, "default:8080") def configure(self): self += File("test", content="asdf {{component.address.listen}}") self += Buildout(version="2.3.1", python="2.7", setuptools="17.1")
class PFAPostfix(Component): address = Attribute(Address, 'localhost:25') def configure(self): self.address.listen.host_v6 = resolve_v6(self.address) self.db = self.require_one('pfa::database') self.keypair = self.require_one('keypair::mail') self.provide('postfix', self.address) self += File('/etc/postfix/myhostname', content=self.address.connect.host) self += File('/etc/postfix/main.d/40_local.cf', source=self.resource('local.cf')) self += File('postfixadmin_virtual_alias', source=self.resource('postfixadmin_virtual_alias')) self += File('postfixadmin_virtual_domains', source=self.resource('postfixadmin_virtual_domains')) self += File('postfixadmin_virtual_sender_login', source=self.resource('postfixadmin_virtual_sender_login')) self += File('postfixadmin_virtual_mailboxes', source=self.resource('postfixadmin_virtual_mailboxes')) def resource(self, filename): return os.path.join(os.path.dirname(__file__), 'postfix', filename)
class BasePackages(Component): packages = Attribute( 'literal', """[ 'build-essential', 'emacs-nox', 'dnsutils', 'git', 'htop', 'httpie', 'jq', 'mc', 'mosh', 'python3-dev', 'python-is-python3', 'rsync', 'screen', 'unzip', 'zip', ]""") def configure(self): for name in self.packages: self += Package(name) # Allow accessing (mostly python) software installed by batou self += File('/root', ensure='directory', mode=0o755) self += User('wosc', home='/home/wosc') self += GroupMember('sudo', user='******') self += File('/etc/motd', is_template=False) self += File('/etc/ssh/sshd_config.d/cyberduck.conf', source='ssh.conf', is_template=False)
class BaseInstance(Component): workdir = '{{component.zope.workdir}}' address = Attribute(Address, '127.0.0.1:11991') script_id = "instance1" def configure(self): self.provide('zope:http', self)
class Test(Component): address = Attribute(Address, 'default:8080') def configure(self): self += File('test', content='asdf {{component.address.listen}}') self += Buildout(version='2.3.1', python='2.7', setuptools='17.1')
class ElasticSearch(Component): # collective.elastic.ingest # collective.elastic.plone uri = Attribute( str, 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.0-linux-x86_64.tar.gz' ) checksum = Attribute( str, 'sha512:5c159bdf0d6e140a2bee5fbb1c379fbe23b0ea39b01d715564f02e4674b444b065a8abfda86440229c4b70defa175722c479b60009b7eef7b3de66e2339aacea' ) def configure(self): self.provide('elasticsearch', self) download = Download(self.uri, checksum=self.checksum) self += download self += Extract(download.target, create_target_dir=False, strip=1)
class CronTab(Component): crontab_template = os.path.join(os.path.dirname(__file__), 'resources', 'crontab') mailto = Attribute(str, None) purge = False # Dict of additional environment variables env = Attribute('literal', '{}') def configure(self): self.jobs = self.require(CronJob.key, host=self.host, strict=False) if self.purge and self.jobs: raise ConfigurationError( 'Found cron jobs, but expecting an empty crontab.') elif not self.purge and not self.jobs: raise ConfigurationError('No cron jobs found.', self) self.jobs.sort(key=lambda job: job.command + ' ' + job.args) self.crontab = File('crontab', source=self.crontab_template) self += self.crontab
class CronTab(Component): crontab_template = os.path.join(os.path.dirname(__file__), "resources", "crontab") mailto = Attribute(default=None) purge = False # Dict of additional environment variables env = Attribute("literal", default_conf_string="{}") def configure(self): self.jobs = self.require(CronJob.key, host=self.host, strict=False) if self.purge and self.jobs: raise ConfigurationError( "Found cron jobs, but expecting an empty crontab.") elif not self.purge and not self.jobs: raise ConfigurationError("No cron jobs found.", self) self.jobs.sort(key=lambda job: job.command + " " + job.args) self.crontab = File("crontab", source=self.crontab_template) self += self.crontab
class Pm2(Component): pm2prefix = Attribute(str, 'local.mydomain.ch-') def configure(self): # self.provide('pm2', self) self.voltoapp = self.require_one('voltoapp') self.varnish = self.require_one('varnish:http') self.zopecommon = self.require_one('zopecommon') self.elasticsearch = self.require_one('elasticsearch') self += File('website.pm2.config.js', source='website.pm2.config.js') self += RestartTasks('all')
class Zope(Component): backupsdir = Attribute(str, '') adminpw = Attribute(str, 'admin') zeoaddress = Attribute(Address, '127.0.0.1:11981') buildoutuser = Attribute(str, 'plone') def configure(self): self.provide('zopecommon', self) self.common = self.require_one('common', host=self.host) self.zope_instances = self.require('zope:http') self.zope_instances.sort(key=lambda s: s.script_id) self.backupsdir = self.backupsdir or self.expand( '{{component.workdir}}/var/backup') config = File('buildout.cfg', source='buildout.cfg', template_context=self) buildout_general = File('buildout_general.cfg', source='buildout_general.cfg', template_context=self) additional_config = [ buildout_general, Directory('profiles', source='profiles') ] self += Buildout(python='3.8', version=self.common.zc_buildout, setuptools=self.common.setuptools, config=config, additional_config=additional_config) self += InstallPythonPackages() # some ElasticSearch, Celery configuration self += File('elasticsearch-mappings.json', source='elasticsearch-mappings.json') self += File('elasticsearch-preprocessings.json', source='elasticsearch-preprocessings.json') self += File('.env', source='_env', template_context=self)
class KeyPair(Component): namevar = 'name' crt = None key = None base_path = Attribute(str, '') provide_itself = Attribute(bool, True) def configure(self): self.crt_file = File( os.path.join(self.base_path, '{}.crt'.format(self.name)), content=self.crt) self += self.crt_file self.key_file = File( os.path.join(self.base_path, '{}.key'.format(self.name)), content=self.key, mode=0o600) self += self.key_file if self.provide_itself: self.provide('keypair::{}'.format(self.name), self)
class PFA(Component): release = '2.92' checksum = 'sha1:21481f6eb8f10ba05fc6fcd1fe0fd468062956f2' address = Attribute(Address, '127.0.0.1:9001') admin_password = None salt = 'ab8f1b639d31875b59fa047481c581fd' config = os.path.join(os.path.dirname(__file__), 'postfixadmin', 'config.local.php') def configure(self): self.db = self.require_one('pfa::database') self.postfix = self.require_one('postfix') self.provide('pfa', self) self.basedir = self.map('postfixadmin') download = Download( 'http://downloads.sourceforge.net/project/postfixadmin/' 'postfixadmin/postfixadmin-{}/postfixadmin-{}.tar.gz'.format( self.release, self.release), target='postfixadmin-{}.tar.gz'.format(self.release), checksum=self.checksum) self += download self += Extract(download.target, target='postfixadmin.orig') self += SyncDirectory(self.basedir, source=self.map( 'postfixadmin.orig/postfixadmin-{}'.format( self.release))) self += File(self.basedir + '/config.local.php', source=self.config) self.fpm = FPM('postfixadmin', adress=self.address) self += self.fpm @property def admin_password_encrypted(self): # password generation ported from postfixadmin/setup.php encrypt = hashlib.sha1() encrypt.update("{}:{}".format(self.salt, self.admin_password)) return "{}:{}".format(self.salt, encrypt.hexdigest())
class CronTab(Component): crontab_template = pkg_resources.resource_filename('batou_ext', 'crontab') mailto = '' install = Attribute('literal', default='True') def configure(self): per_user = collections.defaultdict(list) for job in self.require(CronJob.key, host=self.host, strict=False): per_user[job.user].append(job) for user, jobs in per_user.items(): jobs.sort(key=lambda job: job.command + ' ' + job.args) self += File('crontab.%s' % user, source=self.crontab_template, template_args={'jobs': jobs}) if self.install: self += InstallCrontab(user, crontab=self._)
class User(Component): namevar = 'user' shell = '/bin/bash' home = Attribute(default='/srv/{{component.user}}') def verify(self): try: pwd.getpwnam(self.user) except KeyError: raise UpdateNeeded() def update(self): self.cmd( self.expand('adduser ' '--home {{component.home}} ' '--shell {{component.shell}} ' '{{component.user}}'))
class Mode(FileComponent): mode = Attribute(default=None) def configure(self): super().configure() if isinstance(self.mode, str): try: self.mode = int(self.mode, 8) except ValueError: try: self.mode = convert_mode(self.mode) except Exception as e: raise batou.ConversionError(self, 'mode', self.mode, convert_mode, e) elif isinstance(self.mode, int): pass else: raise batou.ConfigurationError( f'`mode` is required and `{self.mode!r}` is not a valid value.`' ) def verify(self): try: self._select_stat_implementation() except AttributeError: # Happens on systems without lstat/lchmod implementation (like # Linux) Not sure whether ignoring it is really the right thing. return assert os.path.lexists(self.path) current = self._stat(self.path).st_mode assert stat.S_IMODE(current) == self.mode def update(self): self._chmod(self.path, self.mode) def _select_stat_implementation(self): self._stat = os.stat self._chmod = os.chmod if os.path.islink(self.path): self._stat = os.lstat self._chmod = os.lchmod
class PHP(Program): typ = 'fcgi-program' command = '/usr/bin/php-cgi -d error_log=/dev/stderr' params = None socket = 'unix:///run/supervisor/%(program_name)s.sock' socket_owner = Attribute(default='{{component.user}}:www-data') socket_mode = '0770' option_names = Program.option_names + [ 'socket', 'socket_owner', 'socket_mode' ] dependencies = () def configure(self): if self.params: for key, value in self.params.items(): self.command += ' -d %s=%s' % (key, value) super().configure()
class Patch(Component): namevar = 'path' path = Attribute() # inline source = '' target = '' check_source_removed = False # useful when only removing comments # separate file file = '' strip = '0' def configure(self): if self.source and self.file: raise ValueError('Either source or file must be given') if not self.target: raise ValueError('Target text is required') def verify(self): if not os.path.exists(self.path): return file = open(self.path).read() if self.check_source_removed and self.source in file: raise UpdateNeeded() elif self.target not in file: raise UpdateNeeded() def update(self): if self.source: with open(self.path) as f: contents = f.read() contents = re.sub( self.source, self.target, contents, flags=re.MULTILINE) with open(self.path, 'w') as f: f.write(contents) else: self.cmd('patch -d/ -p%s < %s/%s' % ( self.strip, self.parent.defdir, self.file))
class HAProxy(Component): subdomain = '.dev' svc_subdomain = 'testing' memcache_settings = [ 'inter 5s fastinter 1s rise 2 fall 3', 'backup', ] memcache_port = 11211 zeit_networks = [ "127.0.0.1", # localhost "10.100.0.0/16", # Server HH "10.30.0.0/21", # ZON HH "10.30.8.0/21", # ZON Ber "10.200.200.0/21", # OpenVPN "10.210.0.0/21", # OpenVPN "10.110.0.0/16", # Google k8s ("GKE staging", eigentlich 10.110.16.0/20) "10.111.48.0/20", # Google k8s production (siehe terraform-ops/.../production/gke.tf) "10.111.32.0/20", # Google k8s staging (siehe terraform-ops/.../staging/gke-ip-masq-agent.tf) "194.77.156.0/23", # ZON HH public "217.13.68.0/23", # Gaertner "192.168.0.0/16", # Docker "34.89.176.195/32", # Data Team GCP VM ] # We hard-code this here, but it comes from zeit-letsencrypt-acme.sh recipe # (`node['acme.sh']['haproxy']['pem_file']`) ssl_cert = Attribute( default='/etc/haproxy/letsencrypt/fullchain_with_key.pem') def configure(self): self.nameservers = [] for line in open('/etc/resolv.conf'): if line.startswith('nameserver'): parts = re.split(' +', line.strip()) self.nameservers.append(parts[1]) self.varnish_hosts = self.require('varnish:http') self += File('haproxy.cfg')
class Component2(Component): this_does_exist = Attribute("literal", None)
class Component1(Component): do_what_is_needed = Attribute("literal", None)
class ZEO(Component): port = Attribute(int, "9001") features = ["test", "test2"]
class Instance2(BaseInstance): address = Attribute(Address, '127.0.0.1:11992') script_id = "instance2"
class Foo(Component): a = Attribute('list', '') b = Attribute('list', '1,2') c = Attribute('list', '3') d = Attribute('list', ' 3, 3,') e = Attribute('list', [])
class ZEO(Component): port = Attribute(int, default_conf_string="9001") features = ["test", "test2"]
class Roundcube(Component): """Configure Roundcube with database connection. Roundcube is installed with php/fastcgi. A basic configuration for the frontend is created but it is up to the user to fine-tune that configuration. """ release = '1.1.4' checksum = 'sha256:9bfe88255d4ffc288f5776de1cead78352469b1766d5ebaebe6e28043affe181' address = Attribute(Address, '127.0.0.1:9000') skin = 'larry' support_url = 'http://localhost' smtp_user = '******' smtp_pass = '******' config = os.path.join(os.path.dirname(__file__), 'config.inc.php') def configure(self): self.db = self.require_one('roundcube::database') postfix = self.require_one('postfix') self.imap_host = postfix.connect.host self.smtp_server = postfix.connect.host self.smtp_port = postfix.connect.port self.basedir = self.map('roundcube') self.provide('roundcube', self) self += Directory('download') download = Download( 'http://downloads.sourceforge.net/project/roundcubemail/' 'roundcubemail/{}/roundcubemail-{}-complete.tar.gz'.format( self.release, self.release), target='download/roundcube-{}.tar.gz'.format(self.release), checksum=self.checksum) self += download self += Extract(download.target, target='roundcube.orig') self += SyncDirectory( self.basedir, source=self.map( 'roundcube.orig/roundcubemail-{}'.format(self.release))) self.db_dsnw = '{}://{}:{}@{}/{}'.format( self.db.dbms, self.db.username, self.db.password, self.db.address.connect.host, self.db.database) self += File( self.basedir + '/config/config.inc.php', source=self.config) self.fpm = FPM('roundcube', address=self.address) self += self.fpm self += RoundcubeInit(self)
class Supervisor(batou.lib.supervisor.Supervisor): pidfile = Attribute(str, 'var/supervisord.pid', map=True)