def mysql(): """ Setup MySQL server, user and database """ from fabtools import require import fabtools require.mysql.server(password='******') with settings(mysql_user='******', mysql_password='******'): fabtools.mysql.create_user('bob', 'password', host='host1') fabtools.mysql.create_user('bob', 'password', host='host2') assert fabtools.mysql.user_exists('bob', host='host1') assert fabtools.mysql.user_exists('bob', host='host2') assert not fabtools.mysql.user_exists('bob', host='localhost') require.mysql.user('myuser', 'foo') assert fabtools.mysql.user_exists('myuser') require.mysql.database('mydb', owner='myuser') assert fabtools.mysql.database_exists('mydb') # Test that we can run queries as user foo with settings(mysql_user='******', mysql_password='******'): fabtools.mysql.query('select 1;') # Test that we can run queries without supplying the password require.file('.my.cnf', contents="[mysql]\npassword=foo") with settings(mysql_user='******'): fabtools.mysql.query('select 2;')
def deploy_config(): """Deploy config file to Flask instance folder.""" require.files.directory('%s/instance' % env.app_dir, use_sudo=True) require.file('%(app_dir)s/instance/config.py' % env, source='%(config_file)s' % env, use_sudo=True, group='www-data')
def process(name, **kwargs): """ Require a supervisor process """ from fabtools import require require.deb.package('supervisor') require.service.started('supervisor') # Set default parameters params = {} params.update(kwargs) params.setdefault('autorestart', 'true') params.setdefault('redirect_stderr', 'true') # Build config file from parameters lines = [] lines.append('[program:%(name)s]' % locals()) for key, value in sorted(params.items()): lines.append("%s=%s" % (key, value)) # Upload config file filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals() with watch(filename, True, reload_config): require.file(filename, contents='\n'.join(lines), use_sudo=True) # Start the process if needed if process_status(name) == 'STOPPED': start_process(name)
def upload_sqldump(app): if app.dbdump: if re.match('http(s)?://', app.dbdump): fname = 'dump.sql.gz' url = app.dbdump auth = '' else: latest = cdstar.get_latest_bitstream(app.dbdump) fname, url = latest.name, latest.url auth = '-u"{0}:{1}" '.format(os.environ['CDSTAR_USER_BACKUP'], os.environ['CDSTAR_PWD_BACKUP']) target = pathlib.PurePosixPath('/tmp') / fname run('curl -s -o {0} {1} {2}'.format(target, auth, url)) else: db_name = prompt('Replace with dump of local database:', default=app.name) sqldump = pathlib.Path(tempfile.mktemp(suffix='.sql.gz', prefix='%s-' % db_name)) target = pathlib.PurePosixPath('/tmp') / sqldump.name db_user = '******' if PLATFORM == 'windows' else '' local('pg_dump %s--no-owner --no-acl -Z 9 -f %s %s' % (db_user, sqldump, db_name)) require.file(str(target), source=str(sqldump)) sqldump.unlink() if app.stack == 'soundcomparisons': sudo('echo "drop database {0};" | mysql'.format(app.name)) require.mysql.database(app.name, owner=app.name) sudo('gunzip -c {0} | mysql -u {1} --password={1} -D {1}'.format(target, app.name), user=app.name) else: # TODO: assert supervisor.process_status(app.name) != 'RUNNING' if postgres.database_exists(app.name): require_postgres(app, drop=True) sudo('gunzip -c %s | psql -d %s' % (target, app.name), user=app.name) sudo('vacuumdb -zf %s' % app.name, user='******') files.remove(str(target))
def headless(): # How do I enable the "multiverse" repository? # http://askubuntu.com/questions/89096/how-do-i-enable-the-multiverse-repository sudo('sed -i "/^# deb.*multiverse/ s/^# //" /etc/apt/sources.list') timezone() execute(ntpd) require_deb_packages( ''' sudo screen htop nmap iotop mc ssh nano cpuid curl wget nfs-kernel-server nfs-common ''') require.directory('bin') require.file('.profile', contents=cfg('profile.conf'))
def require_postgres(app, drop=False): if drop: with cd('/var/lib/postgresql'): sudo('dropdb %s' % app.name, user='******') with shell_env(SYSTEMD_PAGER=''): require.postgres.server() require.postgres.user(app.name, password=app.name) require.postgres.database(app.name, owner=app.name) if app.pg_unaccent: sql = 'CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;' sudo('psql -c "%s" -d %s' % (sql, app.name), user='******') if app.pg_collkey: pg_dir, = run('find /usr/lib/postgresql/ -mindepth 1 -maxdepth 1 -type d').splitlines() pg_version = pathlib.PurePosixPath(pg_dir).name if not exists('/usr/lib/postgresql/%s/lib/collkey_icu.so' % pg_version): require.deb.packages(['postgresql-server-dev-%s' % pg_version, 'libicu-dev']) with cd('/tmp'): sudo_upload_template('pg_collkey.Makefile', dest='Makefile', pg_version=pg_version) require.file('collkey_icu.c', source=str(PG_COLLKEY_DIR / 'collkey_icu.c')) run('make') sudo('make install') with cd('/tmp'): require.file('collkey_icu.sql', source=str(PG_COLLKEY_DIR / 'collkey_icu.sql')) sudo('psql -f collkey_icu.sql -d %s' % app.name, user='******')
def setup_s3cmd(aws_access_key_id, aws_secret_access_key, s3cfg=None): """ Install s3cmd for multipart uploads to Amazon S3 """ s3cfg = s3cfg if s3cfg is not None else DEFAULT_S3_CFG # Install s3cmd if not is_file('/usr/local/bin/s3cmd'): require.file(url='http://sourceforge.net/projects/s3tools/files/s3cmd/1.5.0-alpha1/s3cmd-1.5.0-alpha1.tar.gz') if not is_dir('s3cmd-1.5.0-alpha1'): run('tar xzf s3cmd-1.5.0-alpha1.tar.gz') with cd('s3cmd-1.5.0-alpha1'): require.deb.package('python-setuptools') sudo('python setup.py install') # Optional dependencies require.python.package('python-magic', use_sudo=True) # S3 config file (including credentials) require.file( '/var/lib/postgresql/.s3cfg', contents=s3cfg.format( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key), use_sudo=True )
def installed_from_source(version=VERSION): """ Require Redis to be installed from source. The compiled binaries will be installed in ``/opt/redis-{version}/``. """ from fabtools import require require.user('redis', home='/var/lib/redis') dest_dir = '/opt/redis-%(version)s' % locals() require.directory(dest_dir, use_sudo=True, owner='redis') if not is_file('%(dest_dir)s/redis-server' % locals()): with cd('/tmp'): # Download and unpack the tarball tarball = 'redis-%(version)s.tar.gz' % locals() require.file(tarball, url='http://redis.googlecode.com/files/' + tarball) run('tar xzf %(tarball)s' % locals()) # Compile and install binaries require.deb.package('build-essential') with cd('redis-%(version)s' % locals()): run('make') for filename in BINARIES: sudo('cp -pf src/%(filename)s %(dest_dir)s/' % locals()) sudo('chown redis: %(dest_dir)s/%(filename)s' % locals())
def livebuild(proxy): print green("Livebuild with " + proxy) pkg('openssh-server live-build python git-core zsh', 'debootstrap') require.directory('/var/build/') mount_line('tmpfs /var/tmp tmpfs defaults 0 0') run_as_root('cat /etc/fstab') require.file('/etc/http_proxy', contents="http://%s/" % proxy, owner='root', group='root', mode='644', verify_remote=True, use_sudo=True) require.file('/etc/profile.d/proxy.sh', contents='export http_proxy="http://%s/"' % proxy, owner='root', group='root', mode='644', verify_remote=True, use_sudo=True) require.file('/usr/local/bin/manualbuild.sh', source='files/bin/manualbuild.sh', owner='root', group='root', mode='755', verify_remote=True, use_sudo=True) require.file('/usr/local/bin/gitbuild.sh', source='files/bin/gitbuild.sh', owner='root', group='root', mode='755', verify_remote=True, use_sudo=True) require.file('/etc/cron.daily/cacherepo', source='files/bin/cron_cacherepo', owner='root', group='root', mode='755', verify_remote=True, use_sudo=True)
def uninstall_couchdb(): ''' Uninstall CouchDB 1.3.0 or 1.5.0 ''' version = get_couchdb_version() require.file( url='http://apache.crihan.fr/dist/couchdb/source/' + '%s/apache-couchdb-%s.tar.gz' % (version, version)) run('tar -xzvf apache-couchdb-%s.tar.gz' % version) with cd('apache-couchdb-%s' % version): sudo('./configure') sudo('make uninstall') sudo('make distclean') su_delete('/usr/local/share/couchdb') su_delete('/usr/local/lib/couchdb') su_delete('/usr/local/var/lib/couchdb') su_delete('/usr/local/var/log/couchdb') su_delete('/usr/local/var/run/couchdb') su_delete('/usr/local/share/doc/couchdb') su_delete('/usr/local/bin/couchjs') su_delete('/usr/local/bin/couchdb') su_delete('apache-couchdb-%s' % version) su_delete('apache-couchdb-%s.tar.gz' % version) su_delete('/etc/supervisor/conf.d/couchdb.conf') su_delete('/etc/cozy/couchdb.login') supervisor.update_config() print(green('CouchDB %s successfully uninstalled' % version))
def install_node10(): ''' install node 0.10.26 ''' if is_pi(): with settings(warn_only=True): result = run('node -v') is_installed = result.find('v0.10.26') if is_installed != -1: print(green("Node.js is already installed")) return True version = '0.10.26' node_url = 'http://nodejs.org/dist/v{0}/node-v{0}-linux-arm-pi.tar.gz' require.file(url=node_url.format(version)) run('tar -xzvf node-v%s-linux-arm-pi.tar.gz' % version) delete_if_exists('/opt/node') require.directory('/opt/node', owner='root') sudo('mv node-v%s-linux-arm-pi/* /opt/node' % version) sudo('ln -s /opt/node/bin/node /usr/local/bin/node') sudo('ln -s /opt/node/bin/node /usr/bin/node') sudo('ln -s /opt/node/bin/npm /usr/local/bin/npm') sudo('ln -s /opt/node/bin/npm /usr/bin/npm') else: require.nodejs.installed_from_source('0.10.26') print(green('node 0.10.26 successfully installed'))
def install_nodejs(): """ Test low level API """ from fabtools import nodejs from fabtools import require from fabtools.files import is_file # Upload local copy of source archive to speed up tests local_cache = '~/.vagrant.d/cache/fabtools/node-v%s.tar.gz' % nodejs.DEFAULT_VERSION if os.path.exists(local_cache): put(local_cache) # Install Node.js from source if nodejs.version() != nodejs.DEFAULT_VERSION: nodejs.install_from_source() assert is_file('/usr/local/bin/node') assert nodejs.version() == nodejs.DEFAULT_VERSION # Install / uninstall global package if not nodejs.package_version('underscore'): nodejs.install_package('underscore', version='1.4.2') assert nodejs.package_version('underscore') == '1.4.2' assert is_file('/usr/local/lib/node_modules/underscore/underscore.js') nodejs.uninstall_package('underscore') assert nodejs.package_version('underscore') == None assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js') # Install / uninstall local package if not nodejs.package_version('underscore', local=True): nodejs.install_package('underscore', version='1.4.2', local=True) assert is_file('node_modules/underscore/underscore.js') assert nodejs.package_version('underscore', local=True) == '1.4.2' nodejs.uninstall_package('underscore', local=True) assert nodejs.package_version('underscore', local=True) == None assert not is_file('node_modules/underscore/underscore.js') # Install dependencies from package.json file require.directory('nodetest') with cd('nodetest'): require.file('package.json', contents=json.dumps({ 'name': 'nodetest', 'version': '1.0.0', 'dependencies': { 'underscore': '1.4.2' } })) nodejs.install_dependencies() assert is_file('node_modules/underscore/underscore.js') assert nodejs.package_version('underscore', local=True) == '1.4.2'
def install_node10(): ''' install node 0.10.26 ''' if is_arm(): with settings(warn_only=True): result = run('node -v') is_installed = result.find('v0.10.26') if is_installed != -1: print(green("Node.js is already installed")) return True version = '0.10.26' node_url = 'http://nodejs.org/dist/v{0}/node-v{0}-linux-arm-pi.tar.gz' require.file(url=node_url.format(version)) run('tar -xzvf node-v%s-linux-arm-pi.tar.gz' % version) delete_if_exists('/opt/node') require.directory('/opt/node', use_sudo=True, owner='root') sudo('mv node-v%s-linux-arm-pi/* /opt/node' % version) sudo('ln -s /opt/node/bin/node /usr/local/bin/node') sudo('ln -s /opt/node/bin/node /usr/bin/node') sudo('ln -s /opt/node/bin/npm /usr/local/bin/npm') sudo('ln -s /opt/node/bin/npm /usr/bin/npm') else: require.nodejs.installed_from_source('0.10.26') print(green('node 0.10.26 successfully installed'))
def uninstall_couchdb(): ''' Uninstall CouchDB 1.3.0 or 1.5.0 ''' version = get_couchdb_version() require.file(url='http://apache.crihan.fr/dist/couchdb/source/' + '%s/apache-couchdb-%s.tar.gz' % (version, version)) run('tar -xzvf apache-couchdb-%s.tar.gz' % version) with cd('apache-couchdb-%s' % version): sudo('./configure') sudo('make uninstall') sudo('make distclean') su_delete('/usr/local/share/couchdb') su_delete('/usr/local/lib/couchdb') su_delete('/usr/local/var/lib/couchdb') su_delete('/usr/local/var/log/couchdb') su_delete('/usr/local/var/run/couchdb') su_delete('/usr/local/share/doc/couchdb') su_delete('/usr/local/bin/couchjs') su_delete('/usr/local/bin/couchdb') su_delete('apache-couchdb-%s' % version) su_delete('apache-couchdb-%s.tar.gz' % version) su_delete('/etc/supervisor/conf.d/couchdb.conf') su_delete('/etc/cozy/couchdb.login') # removes dependency of CouchDB require.deb.uninstall('erlang', False, ['--auto-remove']) supervisor.update_config() print(green('CouchDB %s successfully uninstalled' % version))
def load_sqldump(app, url=DUMP_URL, md5=DUMP_MD5): if console.confirm('Fill the database with %s?' % url, default=False): _, _, filename = url.rpartition('/') with cd('/tmp'): require.file(filename, url=url, md5=md5) sudo('gunzip -c %s | psql %s' % (filename, app.name), user=app.name)
def devel(): """ Developer tools installation """ #TODO: gitconfig deb.install(['build-essential', 'git']) require.file('/etc/gitconfig', source='files/gitconfig', use_sudo=True)
def instance(name, version=VERSION, **kwargs): """ Require a Redis instance to be running The instance will be managed using supervisord. """ from fabtools import require installed_from_source(version) require.directory('/etc/redis', use_sudo=True, owner='redis') require.directory('/var/db/redis', use_sudo=True, owner='redis') require.directory('/var/log/redis', use_sudo=True, owner='redis') require.directory('/var/run/redis', use_sudo=True, owner='redis') # Required for background saving with settings(warn_only=True): require.system.sysctl('vm.overcommit_memory', '1') # Set default parameters params = {} params.update(kwargs) params.setdefault('bind', '127.0.0.1') params.setdefault('port', '6379') params.setdefault('logfile', '/var/log/redis/redis-%(name)s.log' % locals()) params.setdefault('loglevel', 'verbose') params.setdefault('dbfilename', '/var/db/redis/redis-%(name)s-dump.rdb' % locals()) params.setdefault('save', ['900 1', '300 10', '60 10000']) # Build config file from parameters # (keys such as 'save' may result in multiple config lines) lines = [] for key, value in sorted(params.items()): if isinstance(value, list): for elem in value: lines.append("%s %s" % (key, elem)) else: lines.append("%s %s" % (key, value)) redis_server = '/opt/redis-%(version)s/redis-server' % locals() config_filename = '/etc/redis/%(name)s.conf' % locals() # Upload config file context = dict(need_restart=False) def on_change(): context['need_restart'] = True with watch(config_filename, True, on_change): require.file(config_filename, contents='\n'.join(lines), use_sudo=True, owner='redis') # Use supervisord to manage process process_name = 'redis_%s' % name require.supervisor.process(process_name, user='******', directory='/var/run/redis', command="%(redis_server)s %(config_filename)s" % locals()) if context['need_restart']: fabtools.supervisor.restart_process(process_name)
def build_box(requirements, settings): """ Args: requirements (Requirements): settings (Settings): Yields: BuildBoxEnv: """ local('vagrant destroy -f') local('vagrant up') with vagrant_settings(): require.deb.ppa('ppa:spotify-jyrki/dh-virtualenv', auto_accept=True) require.deb.package(requirements.dh_virtualenv) require.deb.packages(requirements.project_package) require.directory('~/.ssh', mode='700') require.file('~/.ssh/config', contents=settings.ssh_config) gitconfig = os.path.expanduser('~/.gitconfig') if os.path.exists(gitconfig): require.file('~/.gitconfig', source=gitconfig) require.git.working_copy(settings.repo, branch=settings.branch, path=settings.project) yield BuildBoxEnv(Settings.project) local('vagrant destroy -f')
def setup_web(): require.deb.packages(['git', 'npm', 'nodejs-legacy', 'python3-dev', 'libxml2-dev', 'libxslt1-dev', 'libpq-dev']) require.nginx.server() require.nginx.site(env.server_name, template_source='nginx-site', port=80, server_alias='', static_path=env.static_path) # require.nginx.disabled('default') with settings(warn_only=True): if run('type bower').return_code: sudo('npm install -g bower') update_source() require.directory(env.home + '/logs') require.python.packages(['uwsgi', 'virtualenv', 'ipython', 'celery'], use_sudo=True) require.file('/etc/init/uwsgi.conf', source='uwsgi.conf', use_sudo=True) # require.file('/etc/init/celery.conf', source='celery.conf', use_sudo=True) require.directory('/etc/uwsgi', use_sudo=True) require.files.template_file('/etc/uwsgi/%s.ini' % env.project_name, template_source='uwsgi.ini', context=env, use_sudo=True) require.service.started('uwsgi') require.directory(env.home + '/bin') require.files.template_file(env.home + '/bin/djangorc', template_source='bin/djangorc', context=env) require.files.template_file(env.home + '/bin/loadenv', template_source='bin/loadenv', context=env) run('chmod +x ' + env.home + '/bin/loadenv')
def installed_from_source(version=VERSION): """ Require Redis to be installed from source """ from fabtools import require require.user('redis') dest_dir = '/opt/redis-%(version)s' % locals() require.directory(dest_dir, use_sudo=True, owner='redis') if not is_file('%(dest_dir)s/redis-server' % locals()): with cd('/tmp'): # Download and unpack the tarball tarball = 'redis-%(version)s.tar.gz' % locals() require.file(tarball, url='http://redis.googlecode.com/files/' + tarball) run('tar xzf %(tarball)s' % locals()) # Compile and install binaries require.deb.package('build-essential') with cd('redis-%(version)s' % locals()): run('make') for filename in BINARIES: sudo('cp -pf src/%(filename)s %(dest_dir)s/' % locals()) sudo('chown redis: %(dest_dir)s/%(filename)s' % locals())
def test_default_temp_dir(self, is_file, md5sum, put, umask, owner, mode): owner.return_value = 'root' umask.return_value = '0002' mode.return_value = '0664' from fabtools import require require.file('/var/tmp/foo', source=__file__, use_sudo=True) put.assert_called_with(__file__, '/var/tmp/foo', use_sudo=True, temp_dir='/tmp')
def process(name, **kwargs): """ Require a supervisor process """ from fabtools import require require.deb.package("supervisor") require.service.started("supervisor") # Set default parameters params = {} params.update(kwargs) params.setdefault("autorestart", "true") params.setdefault("redirect_stderr", "true") # Build config file from parameters lines = [] lines.append("[program:%(name)s]" % locals()) for key, value in sorted(params.items()): lines.append("%s=%s" % (key, value)) # Upload config file filename = "/etc/supervisor/conf.d/%(name)s.conf" % locals() with watch(filename, True, reload_config): require.file(filename, contents="\n".join(lines), use_sudo=True) # Start the process if needed if process_status(name) == "STOPPED": start_process(name)
def process(name, **kwargs): """ Require a supervisor process """ from fabtools import require require.deb.package('supervisor') require.service.started('supervisor') # Set default parameters params = {} params.update(kwargs) params.setdefault('autorestart', 'true') params.setdefault('redirect_stderr', 'true') # Build config file from parameters lines = [] lines.append('[program:%(name)s]' % locals()) for key, value in sorted(params.items()): lines.append("%s=%s" % (key, value)) # Upload config file filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals() with watch(filename, callback=update_config, use_sudo=True): require.file(filename, contents='\n'.join(lines), use_sudo=True) # Start the process if needed if process_status(name) == 'STOPPED': start_process(name)
def process(name, **kwargs): """ Require a supervisor process to be running. Keyword arguments will be used to build the program configuration file. Some useful arguments are: - ``command``: complete command including arguments (**required**) - ``directory``: absolute path to the working directory - ``user``: run the process as this user - ``stdout_logfile``: absolute path to the log file You should refer to the `supervisord documentation`_ for the complete list of allowed arguments. .. note:: the default values for the following arguments differs from the supervisord defaults: - ``autorestart``: defaults to ``true`` - ``redirect_stderr``: defaults to ``true`` Example:: from fabtools import require require.supervisor.process('myapp', command='/path/to/venv/bin/myapp --config production.ini --someflag', directory='/path/to/working/dir', user='******', stdout_logfile='/path/to/logs/myapp.log', ) .. _supervisord documentation: http://supervisord.org/configuration.html#program-x-section-values """ from fabtools import require require.deb.package("supervisor") require.service.started("supervisor") # Set default parameters params = {} params.update(kwargs) params.setdefault("autorestart", "true") params.setdefault("redirect_stderr", "true") # Build config file from parameters lines = [] lines.append("[program:%(name)s]" % locals()) for key, value in sorted(params.items()): lines.append("%s=%s" % (key, value)) # Upload config file filename = "/etc/supervisor/conf.d/%(name)s.conf" % locals() with watch(filename, callback=update_config, use_sudo=True): require.file(filename, contents="\n".join(lines), use_sudo=True) # Start the process if needed if process_status(name) == "STOPPED": start_process(name)
def ntpd(): require_deb_packages( ''' ntp ntpdate ''' ) require.file('/etc/ntp.conf', contents=cfg('ntp.conf'), use_sudo=True)
def copy_archive(app, archive): arc = 'archive.tgz' with cd('/tmp'): require.file(arc, source=archive) sudo('tar -xzf {0}'.format(arc)) sudo('rm -rf {0}/files/*'.format(app.www_dir)) sudo('mv archive/* {0}/files'.format(app.www_dir)) sudo('rm {0}'.format(arc))
def base(proxy='127.0.0.1:3142'): require.file("/etc/apt/apt.conf", contents='Acquire::http { Proxy "http://%s"; };' % proxy) keyrings() devel() if iputils.is_my_ip(proxy.split(':')[0]): aptcacher() livebuild(proxy)
def process(name, **kwargs): """ Require a supervisor process to be running. Keyword arguments will be used to build the program configuration file. Some useful arguments are: - ``command``: complete command including arguments (**required**) - ``directory``: absolute path to the working directory - ``user``: run the process as this user - ``stdout_logfile``: absolute path to the log file You should refer to the `supervisor documentation`_ for the complete list of allowed arguments. .. note:: the default values for the following arguments differs from the ``supervisor`` defaults: - ``autorestart``: defaults to ``true`` - ``redirect_stderr``: defaults to ``true`` Example:: from fabtools import require require.supervisor.process('myapp', command='/path/to/venv/bin/myapp --config production.ini --someflag', directory='/path/to/working/dir', user='******', stdout_logfile='/path/to/logs/myapp.log', ) .. _supervisor documentation: http://supervisord.org/configuration.html#program-x-section-values """ from fabtools import require require.deb.package('supervisor') require.service.started('supervisor') # Set default parameters params = {} params.update(kwargs) params.setdefault('autorestart', 'true') params.setdefault('redirect_stderr', 'true') # Build config file from parameters lines = [] lines.append('[program:%(name)s]' % locals()) for key, value in sorted(params.items()): lines.append("%s=%s" % (key, value)) # Upload config file filename = '/etc/supervisor/conf.d/%(name)s.conf' % locals() with watch(filename, callback=update_config, use_sudo=True): require.file(filename, contents='\n'.join(lines), use_sudo=True) # Start the process if needed if process_status(name) == 'STOPPED': start_process(name)
def install_couchdb(): ''' Install CouchDB 1.3.0 or 1.5.0 depending on the target distribution. ''' # Check if controller is already installed, . with settings(warn_only=True): result = run('curl -X GET http://127.0.0.1:5984/') is_installed = result.find('Welcome') if is_installed != -1: print(green("CouchDB is already installed")) return True packages = [ 'erlang', 'libicu-dev', 'libcurl4-openssl-dev', 'curl' ] if system.distrib_id() == 'Debian' \ and system.distrib_release().startswith('6'): packages.append('libmozjs-dev') else: packages.append('libmozjs185-dev') require.deb.packages(packages) version = get_couchdb_version() require.file( url='http://apache.crihan.fr/dist/couchdb/source/' + '%s/apache-couchdb-%s.tar.gz' % (version, version)) run('tar -xzvf apache-couchdb-%s.tar.gz' % version) with cd('apache-couchdb-%s' % version): run('./configure; make') result = sudo('make install') installed = result.find('You have installed Apache CouchDB,' + ' time to relax.') if installed == -1: print_failed('couchdb') su_delete('apache-couchdb-%s' % version) su_delete('apache-couchdb-%s.tar.gz' % version) require.users.user('couchdb', home='/usr/local/var/lib/couchdb') sudo('chown -R couchdb:couchdb /usr/local/etc/couchdb') sudo('chown -R couchdb:couchdb /usr/local/var/lib/couchdb') sudo('chown -R couchdb:couchdb /usr/local/var/log/couchdb') sudo('chown -R couchdb:couchdb /usr/local/var/run/couchdb') sudo('chmod 0770 /usr/local/etc/couchdb') sudo('chmod 0770 /usr/local/var/lib/couchdb') sudo('chmod 0770 /usr/local/var/log/couchdb') sudo('chmod 0770 /usr/local/var/run/couchdb') require.supervisor.process( 'couchdb', user='******', command='couchdb', autostart='true', environment='HOME=/usr/local/var/lib/couchdb') print(green('CouchDB %s successfully installed' % version))
def service_add(): """ Add the systemd service """ with cd('/lib/systemd/system'): require.file('gage-logger.service', source='gage-logger.service', use_sudo=True) sudo('systemctl enable gage-logger.service')
def provision_glassfish(): require.file('glassfish.zip', url='http://download.java.net/glassfish/3.1.2.2/release/glassfish-3.1.2.2.zip') run('unzip -o glassfish.zip') run('echo "AS_ADMIN_PASSWORD="******"AS_ADMIN_NEWPASSWORD=adminadmin" >> /tmp/password.txt') run('glassfish3/bin/asadmin --user admin --passwordfile /tmp/password.txt change-admin-password --domain_name domain1') run('echo "AS_ADMIN_PASSWORD=adminadmin" > /tmp/password.txt') run('glassfish3/bin/asadmin start-domain && glassfish3/bin/asadmin --passwordfile /tmp/password.txt --host localhost --port 4848 enable-secure-admin && glassfish3/bin/asadmin stop-domain') run('rm /tmp/password.txt')
def install_services(): install_redis() run('rm -rf /tmp/redis') require.file('/etc/redis.conf', source='files/redis.conf', use_sudo=True) require.file('/etc/redis_cache.conf', source='files/redis_cache.conf', use_sudo=True) if not files.exists('/var/run/supervisor.sock'): sudo('supervisord -c /etc/supervisor/supervisord.conf') supervisor.update_config() install_fluentd()
def setup_postgresql_backups(instance_name, aws_s3_prefix, aws_access_key_id, aws_secret_access_key, backup_script=None, backup_config=None): """ Setup daily SQL dumps of the Postgres server to Amazon S3 """ backup_script = backup_script if backup_script is not None else DEFAULT_BACKUP_SCRIPT backup_config = backup_config if backup_config is not None else DEFAULT_BACKUP_CONFIG puts(green('Setting up Postgresql backups')) # Cleanup old config files sudo('rm -f /etc/default/pg_backup') sudo('rm -f /etc/cron.d/pg_backup') # Skip if backups are not enabled if not env.enable_backups: puts('Backups not enabled, skipping...') return # Skip if backups are not configured if not aws_s3_prefix: puts('Target bucket not configured, skipping...') return setup_s3cmd(aws_access_key_id, aws_secret_access_key) require.deb.package('tree') # Postgres backup script backup_script_path = '/usr/local/bin/pg_backup_rotated.sh' config_name = 'pg_backup_{}'.format(instance_name) require.file( backup_script_path, contents=backup_script.format(config_name=config_name), owner='root', group='root', mode='0755', use_sudo=True ) # Postgres backup configuration file require.file( '/etc/default/{}'.format(config_name), contents=backup_config.format(aws_s3_prefix=aws_s3_prefix), owner='root', group='root', mode='0644', use_sudo=True ) # Backups directory require.directory('/var/backups/postgresql', owner='postgres', use_sudo=True) # Add crontab entry fabtools.cron.add_task(config_name, '50 1 * * *', 'postgres', backup_script_path)
def install_nodejs(): """ Test low level API """ from fabtools import nodejs from fabtools import require from fabtools.files import is_file # Install Node.js from source if nodejs.version() != nodejs.DEFAULT_VERSION: nodejs.install_from_source() assert is_file('/usr/local/bin/node') assert nodejs.version() == nodejs.DEFAULT_VERSION # Install / uninstall global package if not nodejs.package_version('underscore'): nodejs.install_package('underscore', version='1.4.2') assert nodejs.package_version('underscore') == '1.4.2' assert is_file('/usr/local/lib/node_modules/underscore/underscore.js') nodejs.uninstall_package('underscore') assert nodejs.package_version('underscore') is None assert not is_file('/usr/local/lib/node_modules/underscore/underscore.js') # Install / uninstall local package if not nodejs.package_version('underscore', local=True): nodejs.install_package('underscore', version='1.4.2', local=True) assert is_file('node_modules/underscore/underscore.js') assert nodejs.package_version('underscore', local=True) == '1.4.2' nodejs.uninstall_package('underscore', local=True) assert nodejs.package_version('underscore', local=True) is None assert not is_file('node_modules/underscore/underscore.js') # Install dependencies from package.json file require.directory('nodetest') with cd('nodetest'): require.file('package.json', contents=json.dumps({ 'name': 'nodetest', 'version': '1.0.0', 'dependencies': { 'underscore': '1.4.2' } })) nodejs.install_dependencies() assert is_file('node_modules/underscore/underscore.js') assert nodejs.package_version('underscore', local=True) == '1.4.2'
def update_config(config_path): """Update the Hadroid config.""" require.file( path=env.hadroid_config, source=config_path, owner=env.app_user, group=env.app_user, mode='0600', use_sudo=True, )
def provision(): updateIndex() require.file('.vimrc', source='vimrc') # install java7 require.oracle_jdk.installed() # install tomcat7 require.tomcat.installed() # install other packages require.deb.packages(['build-essential', 'python'], update=True) deb.upgrade()
def setup_opsin(): """Initial OPSIN setup.""" require.deb.packages(['openjdk-7-jre']) with cd('/opt'): require.file(url='https://bitbucket.org/dan2097/opsin/downloads/opsin-2.1.0-jar-with-dependencies.jar') require.file( path='/opt/opsin', contents='#!/bin/bash\nexec java -jar /opt/opsin-2.1.0-jar-with-dependencies.jar "$@"\n', mode='755' )
def provision(): updateIndex() require.file(".vimrc", source="vimrc") # install java7 require.oracle_jdk.installed() # install tomcat7 require.tomcat.installed() # install other packages require.deb.packages(["build-essential", "python"], update=True) deb.upgrade()
def _update_varnish_sites(directory): sites = run('find %s -mindepth 1 -maxdepth 1 -type f ' % directory, combine_stderr=False).splitlines() includes = ''.join('include "%s";\n' % s for s in sites) # work around requrie.files(contents='') not replacing contents = '# autogenerated\n%s' % includes require.file('/etc/varnish/sites.vcl', contents=contents, use_sudo=True, mode='644') service.restart('varnish')
def pip_wraper(): with lcd('/home/ihor/.pip'): require.directory('~/.pip', owner='web') #require.file('~/.pip/pip.conf', source='pip.conf') text = """ export WORKON_HOME=$HOME/.virtualenvs export PROJECT_HOME=$HOME/Devel source /usr/local/bin/virtualenvwrapper.sh """ require.directory('~/.virtualenvs', owner='web') require.file('~/.bashrc', contents=text)
def refresh_uwsgi(): require.file("/etc/uwsgi/apps-available/%s.ini" % env.app_name, contents=UWSGI_CONFIG_TPL % { 'user': env.user, 'group': env.group, 'app_root': env.app_root, 'app_env': env.app_env, }, use_sudo=True) sudo("ln -sf /etc/uwsgi/apps-available/%s.ini " "/etc/uwsgi/apps-enabled/%s.ini" % (env.app_name, env.app_name)) require.service.restarted('uwsgi')
def default_locale(name): """ Require the locale to be the default. """ from fabtools import require # Ensure the locale is available locale(name) # Make it the default contents = 'LANG="%s"\n' % name require.file('/etc/default/locale', contents, use_sudo=True)
def provision(): require.file('.vimrc', source='vimrc') require.deb.packages([ 'build-essential', 'openjdk-7-jdk', 'maven', 'ant', 'rake', 'git', 'vim' ], update=True) deb.upgrade()
def default_locale(name): """ Require the locale to be the default """ from fabtools import require # Ensure the locale is available locale(name) # Make it the default contents = 'LANG="%s"\n' % name require.file('/etc/default/locale', contents, use_sudo=True)
def qemu(central='127.0.0.1'): require_man(7, 'online-testing', use_sudo=True) if iputils.is_my_ip(central): pkg('libvirt0 qemu-kvm') require.file('/usr/local/bin/emu-vnc', source='files/bin/emu-vnc', owner='root', group='root', mode='755', verify_remote=True, use_sudo=True) else: pkg('nfs-common') mount_line('%s:/var/lib/libvirt/images /var/www nfs defaults 0 0' % central)
def setup_certbot(): require.file('/var/do.ini', source='../__KEYS__/DO_staging.ini') require.deb.uptodate_index() require.deb.packages( ['software-properties-common', 'python3-certbot-dns-digitalocean']) sudo('add-apt-repository universe') sudo('add-apt-repository ppa:certbot/certbot') require.deb.uptodate_index() sudo('apt-get install certbot python-certbot-nginx') sudo('certbot certonly -a dns-digitalocean -i nginx' ' -d "*.reckonsys.xyz" -d reckonsys.xyz' ' --server https://acme-v02.api.letsencrypt.org/directory')
def require_config(filepath, app, ctx): # We only set add a setting clld.files, if the corresponding directory exists; # otherwise the app would throw an error on startup. files_dir = app.www_dir / 'files' files = files_dir if exists(str(files_dir)) else None sudo_upload_template('config.ini', dest=str(filepath), context=ctx, files=files) if app.stack == 'django' and confirm('Recreate secret key?', default=True): key_chars = "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)" secret_key = "".join([random.choice(key_chars) for i in range(50)]) require.file( str(filepath.parent / 'secret_key'), contents=secret_key, use_sudo=True, mode='644')
def deploy_celery(): """Deploy celery service scripts to appropriate location.""" require.file('/etc/init.d/%(app_name)s-celeryd' % env, source='deploy/celeryd' % env, use_sudo=True, mode='755') require.files.template_file( '/etc/default/%(app_name)s-celeryd' % env, template_source='deploy/celeryd-default' % env, context=dict(app_name=env.app_name, app_dir=env.app_dir), use_sudo=True ) sudo('update-rc.d %(app_name)s-celeryd defaults' % env) require.directory('/var/log/%(app_name)s' % env, use_sudo=True, owner=env.app_user, group='www-data') require.directory('/var/run/%(app_name)s' % env, use_sudo=True, owner=env.app_user, group='www-data')
def nominatim(): pgconfig(for_import=True) with cd('/opt/osm'): nominatime_archive = 'Nominatim-%s.tar.bz2' % config.NOMINATIM_VERSION nominatim_url = 'http://www.nominatim.org/release/'\ + nominatime_archive require.file(url=nominatim_url, use_sudo=True, owner=config.GIS_USER) nominatim_dir = sudo('''tar tf %s | sed -e 's@/.*@@' | uniq''' % nominatime_archive, user=config.GIS_USER) sudo('''tar xvf ''' + nominatime_archive, user=config.GIS_USER) with cd(nominatim_dir): sudo('./autogen.sh', user=config.GIS_USER) sudo('./configure', user=config.GIS_USER) sudo('make', user=config.GIS_USER) context = { 'db_name': config.GIS_DB, 'db_user': config.GIS_USER, 'db_passowrd': config.GIS_PASSWORD, } require.files.template_file(path='settings/local.php', template_source='templates/local.php', context=context, use_sudo=True, owner=config.GIS_USER) with cd('data'): wiki_urls = [ 'http://www.nominatim.org/data/wikipedia_article.sql.bin', 'http://www.nominatim.org/data/wikipedia_redirect.sql.bin' ] for url in wiki_urls: require.file(url=url, use_sudo=True, owner=config.GIS_USER) sudo('./utils/setup.php --osm-file %s --all --osm2pgsql-cache %d' % (pbf_path(), config.RAM_SIZE / 4 * 3), user=config.GIS_USER) sudo('./utils/specialphrases.php --countries > sp_countries.sql', user=config.GIS_USER) sudo('psql -d %s -f sp_countries.sql' % config.GIS_DB, user=config.GIS_USER) sudo('./utils/specialphrases.php --wiki-import > sp.sql', user=config.GIS_USER) sudo('psql -d %s -f sp.sql' % config.GIS_DB, user=config.GIS_USER) require.directory('/var/www/nominatim', mode='755', owner=config.GIS_USER, use_sudo=True) sudo('./utils/setup.php --create-website /var/www/nominatim', user=config.GIS_USER) require.apache.site('200-nominatim.conf', template_source='templates/200-nominatim.conf') require.service.restarted('apache2') pgconfig(for_import=False)
def setup(): local_clone_repos() require.files.directories([apps_dir, logs_dir]) require.files.directories(["/var/repo/"], use_sudo=True) require.deb.package("software-properties-common") sudo("add-apt-repository ppa:ondrej/php") sudo("add-apt-repository ppa:ondrej/apache2") sudo("add-apt-repository ppa:certbot/certbot") deb.update_index() deb.upgrade() require.git.command() require.deb.packages([ "mysql-server", "dpkg-dev", "php7.1", "php7.1-mysql", "php7.1-gd", "php7.1-dev", "php7.1-curl", "php7.1-cli", "php7.1-json", "php-apcu", "libpcre3-dev", "php-pear", "libapache2-mod-php7.1", "sendmail", "php7.1-mbstring", "python-pip", "python-certbot-apache" ]) sudo("yes '' | pecl install apc") sudo("pip install pygments") require.apache.server() for apache_mod in apache_mods: require.apache.module_enabled(apache_mod) push_repos() require.apache.site( hostname, template_contents=CONFIG_TPL, port=80, hostname=hostname, docroot=docroot, ) require.apache.site_enabled(hostname) require.apache.site_disabled('default') print("Please make sure %s is pointing to %s before certbot install" % (hostname, env.hosts)) sudo("certbot --apache") require.mysql.server(password=db_root_pass) with settings(mysql_user='******', mysql_password=db_root_pass): require.mysql.user(env.user, db_user_pass) grant_all(env.user) require.file("/etc/mysql/conf.d/mysql.cnf", "[mysqld]\nsql_mode=STRICT_ALL_TABLES", use_sudo=True) sudo("service mysql restart") require.nodejs.installed_from_source(version='8.9.1') with cd("%s/phabricator" % apps_dir): run("./bin/config set mysql.host localhost") run("./bin/config set mysql.user %s" % env.user) run("./bin/config set mysql.pass %s" % db_user_pass) run("./bin/config set phabricator.base-uri 'https://%s'" % hostname) run("./bin/storage upgrade --force") run("./bin/phd start") run("./bin/aphlict start")
def setup_rdkit(): """Initial RDKit setup.""" require.deb.packages([ 'build-essential', 'python-numpy', 'cmake', 'python-dev', 'sqlite3', 'libsqlite3-dev', 'libboost-dev', 'libboost-system-dev', 'libboost-thread-dev', 'libboost-serialization-dev', 'libboost-python-dev', 'libboost-regex-dev', 'wget' ]) with cd('/opt'): require.file( url= 'https://github.com/rdkit/rdkit/archive/Release_2016_03_1.tar.gz') sudo('tar -xvf Release_2016_03_1.tar.gz')
def setup_project(): """ Require working copy of project cloned from remote repository. """ require.directory(env.project_dir, use_sudo=True, owner=env.user, group="www-data") with cd(env.project_dir): require.git.working_copy(env.config["repository"], directly=True) require.file( os.path.join(env.project_dir, env.config["project"], "secrets.json"), json.dumps(env.secrets))
def copy_downloads(app, source_dir, pattern='*'): """copy downloads for the app""" require.directory(str(app.download_dir), use_sudo=True, mode='777') source_dir = pathlib.Path(source_dir) for f in source_dir.glob(pattern): require.file(str(app.download_dir / f.name), source=f, use_sudo=True, owner=app.name, group=app.name) require.directory(str(app.download_dir), use_sudo=True, mode='755')
def checkout(url, workdir, name): if not exists(env.deploy_key_path): run('mkdir -p %s/.ssh' % env.home) require.file(env.home + '/.ssh/config', source='_ops/ssh_config') require.file(env.deploy_key_path, source='/Users/alankang/.ssh/id_rsa') run('chmod 0600 ' + env.deploy_key_path) if exists(os.path.join(workdir, name)): with cd(os.path.join(workdir, name)): run("ssh-agent bash -c 'ssh-add %s; git pull'" % env.deploy_key_path) else: with cd(workdir): run("ssh-agent bash -c 'ssh-add %s; git clone %s %s'" % (env.deploy_key_path, url, name))