def get_elasticsearch_image(version): return SourceImage( name='elasticsearch', parent=Image(namespace='yandex', repository='trusty'), scripts=[ 'curl http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -', 'echo "deb http://packages.elasticsearch.org/elasticsearch/{version_minor}/debian stable main"' ' > /etc/apt/sources.list.d/elasticsearch.list && apt-get update -q'.format( version_minor='.'.join(version.split('.')[:2])), 'apt-get update -q && ' 'apt-get install -y --no-install-recommends maven elasticsearch={version_full} openjdk-7-jdk'.format( version_full=version), 'git clone https://github.com/grmblfrz/elasticsearch-zookeeper.git /tmp/elasticsearch-zookeeper', 'cd /tmp/elasticsearch-zookeeper && git checkout v{version_full} && ' 'mvn package -Dmaven.test.skip=true -Dzookeeper.version=3.4.6'.format(version_full=version), '/usr/share/elasticsearch/bin/plugin -v ' ' -u file:///tmp/elasticsearch-zookeeper/target/releases/elasticsearch-zookeeper-{version_full}.zip ' ' -i elasticsearch-zookeeper-{version_full}'.format(version_full=version), '/usr/share/elasticsearch/bin/plugin -v -i elasticsearch/marvel/latest', '/usr/share/elasticsearch/bin/plugin -v -i mobz/elasticsearch-head', ], ports={'http': 9201, 'peer': 9301, 'jmx': 9401}, volumes={ 'logs': '/var/log/elasticsearch', 'data': '/var/lib/elasticsearch', 'config': '/etc/elasticsearch' }, files={'/scripts/elasticsearch.sh': resource_stream('elasticsearch.sh')}, command=['/scripts/elasticsearch.sh'], )
def make_server_image(): return SourceImage( name='zabbix-server', parent=make_zabbix_image(), ports={'zabbix-trapper': 10051}, scripts=[ 'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ' + 'zabbix-server-pgsql={version} '.format(version=version) + 'strace snmp-mibs-downloader fping nmap', 'ln -fs /usr/bin/fping /usr/sbin/', ], files={ '/scripts/zabbix.sh': resource_stream('zabbix.sh'), '/usr/lib/zabbix/alertscripts/golem-alert-handler.sh': resource_stream('golem-alert-handler.sh'), }, volumes={ 'logs': '/var/log/zabbix', 'config': '/etc/zabbix', }, command=['/scripts/zabbix.sh'], )
def get_nginx_image(): return SourceImage( name='nginx', parent=Image(namespace='yandex', repository='trusty'), env={'DEBIAN_FRONTEND': 'noninteractive'}, scripts=[ 'apt-add-repository -y ppa:nginx/stable', 'apt-get update', 'apt-get install -yy nginx-extras', 'rm -f /etc/nginx/sites-enabled/default', 'wget https://gist.githubusercontent.com/rrx/6217900/raw/' '78c2a4817dad9611ab602834d56d0f5b00bb3cc9/gencert.sh', 'bash gencert.sh localhost || true', 'cat localhost.crt localhost.key > /etc/ssl/private/server.pem', ], files={'/etc/nginx/nginx.conf': resource_stream('nginx.conf')}, volumes={ 'logs': '/var/log/nginx', 'sites': '/etc/nginx/sites-enabled', 'ssl': '/etc/nginx/certs', }, command=['nginx'], )
def make(version='1:2.4.1-1+trusty'): frontend_image = SourceImage( name='zabbix-frontend', parent=make_zabbix_image(), scripts=[ 'apt-get update && ' 'apt-get install -y zabbix-frontend-php={version} apache2 php5-pgsql' .format(version=version), 'chmod go+rx /etc/zabbix', ], files={ '/scripts/frontend.sh': resource_stream('frontend.sh'), }, command=['/scripts/frontend.sh'], ) postgres_image = SourceImage( name='postgres', parent=Image(repository='postgres', namespace=None, registry=None), ) postgres = Container( name='postgres', image=postgres_image, doors={'postgres': Door(port=5432, schema='postgres')}, volumes={ 'logs': LogVolume(dest='/var/log/postgresql'), 'data': DataVolume(dest='/var/lib/postgresql/data'), # 'config': ConfigVolume(dest='/etc/postgresql'), }, ) @cached def make_server_image(): return SourceImage( name='zabbix-server', parent=make_zabbix_image(), ports={'zabbix-trapper': 10051}, scripts=[ 'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ' + 'zabbix-server-pgsql={version} '.format(version=version) + 'strace snmp-mibs-downloader fping nmap', 'ln -fs /usr/bin/fping /usr/sbin/', ], files={ '/scripts/zabbix.sh': resource_stream('zabbix.sh'), '/usr/lib/zabbix/alertscripts/golem-alert-handler.sh': resource_stream('golem-alert-handler.sh'), }, volumes={ 'logs': '/var/log/zabbix', 'config': '/etc/zabbix', }, command=['/scripts/zabbix.sh'], ) backend = Container( name='zabbix-backend', image=make_server_image(), doors={'zabbix-trapper': Door(schema='zabbix-trapper')}, privileged=True, # needed for strace to work volumes={ 'logs': LogVolume( dest='/var/log/zabbix', files={ 'zabbix_server.log': LogFile(), 'snmptt.log': LogFile(), 'golem-alert.log': LogFile(), }, ), 'config': ConfigVolume( dest='/etc/zabbix', files={'zabbix_server.conf': None}, ), }, ) # This variable is needed for golem-alert-handler.sh script backend.env['GOLEM_ALERT_LOG'] = backend.volumes['logs'].files[ 'golem-alert.log'].fulldest def make_zabbix_server_conf(backend=backend, postgres=postgres): logfiles = backend.volumes['logs'].files config = { 'LogFile': logfiles['zabbix_server.log'].fulldest, 'LogFileSize': 0, 'PidFile': '/var/run/zabbix_server.pid', 'DBHost': postgres.ship.fqdn, 'DBName': 'zabbix', 'DBUser': '******', 'DBPassword': '', 'DBPort': postgres.doors['postgres'].port, 'StartPollers': 5, 'StartIPMIPollers': 0, 'StartTrappers': 1, 'JavaGateway': '127.0.0.1', 'StartJavaPollers': 0, 'StartVMwareCollectors': 0, 'VMwareFrequency': 10, 'VMwareCacheSize': '256K', 'SNMPTrapperFile': logfiles['snmptt.log'].fulldest, 'SenderFrequency': 10, 'CacheUpdateFrequency': 10, 'StartDBSyncers': 4, 'HistoryCacheSize': '2G', 'TrendCacheSize': '2G', 'HistoryTextCacheSize': '2G', 'ValueCacheSize': '2G', 'Timeout': 30, 'UnreachablePeriod': 10, 'UnavailableDelay': 10, 'UnreachableDelay': 10, 'AlertScriptsPath': '/usr/lib/zabbix/alertscripts', 'ExternalScripts': '/usr/lib/zabbix/externalscripts', 'ProxyConfigFrequency': 10, 'AllowRoot': 1, } return TextFile('\n'.join([ '{}={}'.format(key, value) for key, value in sorted(config.items()) ])) backend.volumes['config'].files[ 'zabbix_server.conf'] = make_zabbix_server_conf frontend = Container( name='zabbix-frontend', image=frontend_image, privileged=True, # needed to chmod /etc/zabbix doors={ 'http': Door( schema='http', urls={'default': Url('zabbix')}, ), }, volumes={ 'logs': LogVolume( dest='/var/log/apache2', files={ 'access.log': LogFile(), 'error.log': LogFile(), }, ), 'config-apache': ConfigVolume( dest='/etc/zabbix', files={ 'apache.conf': TextFile(resource_string('apache.conf')), 'web/zabbix.conf.php': TemplateFile( resource_string('zabbix.conf.php'), postgres=postgres, backend=backend, ), }, ), }, ) def make_db_task(name, script): return Task( name=name, image=make_server_image(), volumes={ 'scripts': ConfigVolume(dest='/scripts', files={'run.sh': script}), }, command=['/scripts/run.sh'], ) def make_reinit_script(): return TextFile( dedent('''#!/bin/bash cmd='psql -U postgres -h {door.host} -p {door.port}' echo 'drop database zabbix; create database zabbix;' | $cmd cd /usr/share/zabbix-server-pgsql cat schema.sql images.sql data.sql | $cmd zabbix '''.format(door=postgres.doors['postgres']))) reinit = make_db_task('reinit', make_reinit_script) def make_dump_script(): return TextFile( 'pg_dump -U postgres -h {door.host} -p {door.port} -c -C ' 'zabbix'.format(door=postgres.doors['postgres'])) dump = make_db_task('dump', make_dump_script) def make_restore_script(): return TextFile( 'psql -U postgres -h {door.host} -p {door.port}'.format( door=postgres.doors['postgres'])) restore = make_db_task('restore', make_restore_script) return [postgres, frontend, backend], [reinit, dump, restore]
def make(version='1:2.4.1-1+trusty'): frontend_image = SourceImage( name='zabbix-frontend', parent=make_zabbix_image(), scripts=[ 'apt-get update && ' 'apt-get install -y zabbix-frontend-php={version} apache2 php5-pgsql'.format(version=version), 'chmod go+rx /etc/zabbix', ], files={ '/scripts/frontend.sh': resource_stream('frontend.sh'), }, command=['/scripts/frontend.sh'], ) postgres_image = SourceImage( name='postgres', parent=Image(repository='postgres', namespace=None, registry=None), ) postgres = Container( name='postgres', image=postgres_image, doors={'postgres': Door(port=5432, schema='postgres')}, volumes={ 'logs': LogVolume(dest='/var/log/postgresql'), 'data': DataVolume(dest='/var/lib/postgresql/data'), # 'config': ConfigVolume(dest='/etc/postgresql'), }, ) @cached def make_server_image(): return SourceImage( name='zabbix-server', parent=make_zabbix_image(), ports={'zabbix-trapper': 10051}, scripts=[ 'apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ' + 'zabbix-server-pgsql={version} '.format(version=version) + 'strace snmp-mibs-downloader fping nmap', 'ln -fs /usr/bin/fping /usr/sbin/', ], files={ '/scripts/zabbix.sh': resource_stream('zabbix.sh'), '/usr/lib/zabbix/alertscripts/golem-alert-handler.sh': resource_stream('golem-alert-handler.sh'), }, volumes={ 'logs': '/var/log/zabbix', 'config': '/etc/zabbix', }, command=['/scripts/zabbix.sh'], ) backend = Container( name='zabbix-backend', image=make_server_image(), doors={'zabbix-trapper': Door(schema='zabbix-trapper')}, privileged=True, # needed for strace to work volumes={ 'logs': LogVolume( dest='/var/log/zabbix', files={ 'zabbix_server.log': LogFile(), 'snmptt.log': LogFile(), 'golem-alert.log': LogFile(), }, ), 'config': ConfigVolume( dest='/etc/zabbix', files={'zabbix_server.conf': None}, ), }, ) # This variable is needed for golem-alert-handler.sh script backend.env['GOLEM_ALERT_LOG'] = backend.volumes['logs'].files['golem-alert.log'].fulldest def make_zabbix_server_conf(backend=backend, postgres=postgres): logfiles = backend.volumes['logs'].files config = { 'LogFile': logfiles['zabbix_server.log'].fulldest, 'LogFileSize': 0, 'PidFile': '/var/run/zabbix_server.pid', 'DBHost': postgres.ship.fqdn, 'DBName': 'zabbix', 'DBUser': '******', 'DBPassword': '', 'DBPort': postgres.doors['postgres'].port, 'StartPollers': 5, 'StartIPMIPollers': 0, 'StartTrappers': 1, 'JavaGateway': '127.0.0.1', 'StartJavaPollers': 0, 'StartVMwareCollectors': 0, 'VMwareFrequency': 10, 'VMwareCacheSize': '256K', 'SNMPTrapperFile': logfiles['snmptt.log'].fulldest, 'SenderFrequency': 10, 'CacheUpdateFrequency': 10, 'StartDBSyncers': 4, 'HistoryCacheSize': '2G', 'TrendCacheSize': '2G', 'HistoryTextCacheSize': '2G', 'ValueCacheSize': '2G', 'Timeout': 30, 'UnreachablePeriod': 10, 'UnavailableDelay': 10, 'UnreachableDelay': 10, 'AlertScriptsPath': '/usr/lib/zabbix/alertscripts', 'ExternalScripts': '/usr/lib/zabbix/externalscripts', 'ProxyConfigFrequency': 10, 'AllowRoot': 1, } return TextFile('\n'.join(['{}={}'.format(key, value) for key, value in sorted(config.items())])) backend.volumes['config'].files['zabbix_server.conf'] = make_zabbix_server_conf frontend = Container( name='zabbix-frontend', image=frontend_image, privileged=True, # needed to chmod /etc/zabbix doors={ 'http': Door( schema='http', urls={'default': Url('zabbix')}, ), }, volumes={ 'logs': LogVolume( dest='/var/log/apache2', files={ 'access.log': LogFile(), 'error.log': LogFile(), }, ), 'config-apache': ConfigVolume( dest='/etc/zabbix', files={ 'apache.conf': TextFile(resource_string('apache.conf')), 'web/zabbix.conf.php': TemplateFile( resource_string('zabbix.conf.php'), postgres=postgres, backend=backend, ), }, ), }, ) def make_db_task(name, script): return Task( name=name, image=make_server_image(), volumes={ 'scripts': ConfigVolume( dest='/scripts', files={'run.sh': script} ), }, command=['/scripts/run.sh'], ) def make_reinit_script(): return TextFile(dedent('''#!/bin/bash cmd='psql -U postgres -h {door.host} -p {door.port}' echo 'drop database zabbix; create database zabbix;' | $cmd cd /usr/share/zabbix-server-pgsql cat schema.sql images.sql data.sql | $cmd zabbix '''.format(door=postgres.doors['postgres']))) reinit = make_db_task('reinit', make_reinit_script) def make_dump_script(): return TextFile('pg_dump -U postgres -h {door.host} -p {door.port} -c -C ' 'zabbix'.format(door=postgres.doors['postgres'])) dump = make_db_task('dump', make_dump_script) def make_restore_script(): return TextFile('psql -U postgres -h {door.host} -p {door.port}'.format(door=postgres.doors['postgres'])) restore = make_db_task('restore', make_restore_script) return [postgres, frontend, backend], [reinit, dump, restore]
def make_powny_builder( api_workers=4, api_max_requests=5000, extra_scripts=(), helpers_config=None, pip_pre=False, powny_version='==1.5.0', pypy_version='jit-74309-4ca3a10894aa', ): powny_yaml_path = os.path.join('/etc/powny', 'powny.yaml') # === Images === img_base = Image(namespace='yandex', repository='trusty') img_powny = SourceImage( name='powny', parent=img_base, env={ 'PATH': '$PATH:/opt/pypy3/bin', 'LANG': 'C.UTF-8', }, ports={'backdoor': 10023}, scripts=[ 'curl http://buildbot.pypy.org/nightly/py3k/pypy-c-{}-linux64.tar.bz2 2>/dev/null'.format(pypy_version) + '| tar -jxf -', 'mv pypy* /opt/pypy3', 'curl https://bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py 2>/dev/null | pypy3', 'easy_install pip==1.4.1', # json-formatter is needed to store exceptions and log extra fields in json logs. 'pip install --pre' ' json-formatter==0.1.0-alpha-20141128-1014-94cc025' ' powny{powny_version}'.format(powny_version=powny_version), ] + list(extra_scripts), entrypoint=['bash', '-c'], command=[stoppable('powny-$POWNY_APP -c {}'.format(powny_yaml_path))], ) img_gitapi = SourceImage( name='gitapi', parent=img_base, files={ '/post-receive': resource_stream('post-receive'), '/etc/ssh/sshd_config': resource_stream('sshd_config'), '/root/run.sh': resource_stream('run.sh'), }, ports={'ssh': 22}, scripts=[ 'apt-get -q update && apt-get install -y openssh-server && apt-get clean', 'useradd --non-unique --uid 0 --system --shell /usr/bin/git-shell -d / git', 'mkdir /run/sshd', 'chmod 0755 /run/sshd', 'sed -i -e "s/session required pam_loginuid.so//g" /etc/pam.d/sshd' ], command=['/root/run.sh'], ) # === Volumes === def make_rules_volume(): return DataVolume(dest='/var/lib/powny/rules', path='/var/lib/powny/rules') # === Containers === def make_logs_volume(*files): return LogVolume(dest='/var/log/powny', files={name: LogFile() for name in files}) def make_powny_container(name, app, memory=1024**3): app = app or name container = Container( name=name, image=img_powny, memory=memory, volumes={ 'config': None, 'logs': make_logs_volume('powny.log', 'powny.debug.log', 'powny.json.log'), 'rules': make_rules_volume(), }, env={'POWNY_APP': app}, doors={'backdoor': Door(schema='telnet', port=img_powny.ports['backdoor'])}, ) def make_logging_config(container): logging_config = yaml.load(resource_string('logging.yaml')) return logging_config def make_powny_config(container=container, helpers_config=helpers_config): logging_config = make_logging_config(container) assert 'zookeeper' in container.links, "Powny should be linked with zookeeper cluster to work" config = { 'core': { 'rules_dir': make_rules_volume().dest, }, 'api': { 'gunicorn': { 'bind': '0.0.0.0:80', 'workers': api_workers, 'max_requests': api_max_requests, }, }, 'backdoor': { 'enabled': (api_workers == 1), # Backdoor failed for multiprocess app 'port': img_powny.ports['backdoor'], }, 'backend': { 'nodes': [door.hostport for door in container.links['zookeeper']], }, 'logging': logging_config, } config['helpers'] = helpers_config or {} return YamlFile(config) container.volumes['config'] = ConfigVolume( dest=os.path.dirname(powny_yaml_path), files={os.path.basename(powny_yaml_path): make_powny_config}, ) return container class Builder: @staticmethod def gitapi(ssh_keys): dv_rules_git = DataVolume(dest='/var/lib/powny/rules.git', path='/var/lib/powny/rules.git') cv_etc_gitapi = ConfigVolume(dest='/etc/gitsplit', files={ 'gitsplit.conf': TextFile(text=textwrap.dedent(''' RULES_GIT_PATH={rules_git_dest} RULES_PATH={rules_dest} REV_LIMIT=10 ''').strip().format( rules_git_dest=dv_rules_git.dest, rules_dest=make_rules_volume().dest, )), }) cv_ssh_keys = ConfigVolume(dest='/var/lib/keys', files={ 'authorized_keys': TextFile(text='\n'.join(ssh_keys)), }) return Container( name='gitapi', image=img_gitapi, memory=128*1024*1024, volumes={ 'rules.git': dv_rules_git, 'rules': make_rules_volume(), 'config': cv_etc_gitapi, 'keys': cv_ssh_keys, 'logs': make_logs_volume('gitapi.log'), }, doors={'ssh': Door(schema='ssh', port=img_gitapi.ports['ssh'])}, ) @staticmethod def api(name='api'): cont = make_powny_container(name, 'api', memory=4096*1024*1024) cont.doors['http'] = Door(schema='http', port=80) return cont @staticmethod def worker(name='worker'): return make_powny_container(name, 'worker') @staticmethod def collector(name='collector'): return make_powny_container(name, 'collector') return Builder