def update_filters(): changes = [] # Generate desired state as enabled_name => source_name config = blueprint.get('config', {}) filters = { '{}-{}.conf'.format(str(weight).zfill(2), conf): "{}.conf".format(conf) for weight, conf in config.iteritems() } # Get current state with silent(): enabled_filters = run('ls {}'.format(conf_enabled_path)).split() # Disable extra services if blueprint.get('auto_disable_conf', True): for link in set(enabled_filters) - set(filters.keys()): info('Disabling conf: {}', link) changes.append(link) with silent(), sudo(), cd(conf_enabled_path): debian.rm(link) # Enable services for target in set(filters.keys()) - set(enabled_filters): source = os.path.join(conf_available_path, filters[target]) info('Enabling conf: {}', target) changes.append(source) with silent(), sudo(), cd(conf_enabled_path): debian.ln(source, target) return changes
def update_modules(beat): changes = [] # Get desired state desired_modules = set(blueprint.get('{}.modules'.format(beat), [])) # Get current state with silent(): module_files = run('find /etc/{}/modules.d -iname "*.yml"'.format(beat)).split() enabled_modules = {os.path.basename(module).split('.')[0] for module in module_files} # Disable extra services for extra in enabled_modules - desired_modules: info('Disabling {} module: {}', beat, extra) changes.append(extra) with silent(), sudo(): run('{} modules disable {}'.format(beat, extra)) # Enable services for missing in desired_modules - enabled_modules: info('Enabling {} module: {}', beat, missing) changes.append(missing) with silent(), sudo(): run('{} modules enable {}'.format(beat, missing)) return changes
def dump(schema=None): """ Dump and download all configured, or given, schemas. :param schema: Specific shema to dump and download. """ if not schema: schemas = blueprint.get('schemas', {}).keys() for i, schema in enumerate(schemas, start=1): print("{i}. {schema}".format(i=i, schema=schema)) valid_indices = '[1-{}]+'.format(len(schemas)) schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices) schema = schemas[int(schema_choice) - 1] with sudo('postgres'): now = datetime.now().strftime('%Y-%m-%d') output_file = '/tmp/{}_{}.backup'.format(schema, now) filename = os.path.basename(output_file) options = dict(format='tar', output_file=output_file, schema=schema) info('Dumping schema {}...', schema) run('pg_dump -c -F {format} -f {output_file} {schema}'.format( **options)) info('Downloading dump...') local_file = '~/{}'.format(filename) files.get(output_file, local_file) with sudo(), silent(): debian.rm(output_file) info('New smoking hot dump at {}', local_file)
def create_app_root(): from .project import app_root with sudo(): # Create global apps root root_path = app_root() debian.mkdir(root_path, recursive=True)
def configure(): """ Enable/disable configured programs """ with sudo(): # Upload templates uploads = blueprint.upload("init/", "/etc/init/") uploads.extend(blueprint.upload("supervisord.conf", "/etc/")) # Disable previously enabled programs not configured programs-enabled changes = [] programs = blueprint.get("programs") or [] auto_disable = blueprint.get("auto_disable_programs", True) if auto_disable: with silent(): enabled_program_links = run("ls {}".format(programs_enabled_path)).split() for link in enabled_program_links: link_name = os.path.splitext(link)[0] # Without extension if link not in programs and link_name not in programs: changed = disable(link, do_reload=False) changes.append(changed) ### Enable programs from settings for program in programs: changed = enable(program, do_reload=False) changes.append(changed) ### Reload supervisor if new templates or any program has been enabled/disabled if uploads or any(changes): reload()
def configure(): """ Configure PureFTP """ with sudo(): # Echo configurations setup_config() for user in blueprint.get('users'): username, password = user['username'], user['password'] if 'homedir' in user: user_home = user['homedir'] else: user_home = os.path.join(ftp_root, username) passwd_path = '/etc/pure-ftpd/pureftpd.passwd' with settings(warn_only=True): if files.exists(passwd_path) and run('pure-pw show {}'.format( username)).return_code == 0: continue debian.mkdir(user_home, owner=ftp_user, group=ftp_group) prompts = { 'Password: '******'Enter it again: ': password } with settings(prompts=prompts): run('pure-pw useradd {} -u {} -g {} -d {}'.format(username, ftp_user, ftp_group, user_home)) run('pure-pw mkdb') restart()
def install(): with sudo(): info('Installing Ruby v1.9.3') debian.apt_get('install', 'ruby1.9.3') info('Installing Bundler') gem('install', 'bundler')
def service(name, action, check_status=True): c = fabric.context_managers with sudo('root'), c.settings(c.hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): info('Service: {} {}', name, action) if check_status: output = run('service {} status'.format(name), pty=False, combine_stderr=True) if output.return_code != 0: puts(indent(magenta(output))) return elif action in output: puts( indent('...has status {}'.format( magenta(output[len(name) + 1:])))) return output = run('service {} {}'.format(name, action), pty=False, combine_stderr=True) if output.return_code != 0: puts(indent(magenta(output)))
def install_solr(): with sudo(): version = blueprint.get("version") version_tuple = tuple(map(int, version.split("."))) archive = "solr-{}.tgz".format(version) if version_tuple < (4, 1, 0): archive = "apache-{}".format(archive) url = "https://archive.apache.org/dist/lucene/solr/{}/{}".format(version, archive) with cd("/tmp"): info("Download {} ({})", "Solr", version) run("wget {}".format(url)) info("Extracting archive...") with silent(): run("tar xzf {}".format(archive)) solr_version_dir = os.path.splitext(archive)[0] solr_version_path = os.path.join("/usr", "share", solr_version_dir) debian.chmod(solr_version_dir, 755, "solr", "solr", recursive=True) if files.exists(solr_version_path): info("Found same existing version, removing it...") debian.rm(solr_version_path, recursive=True) debian.mv(solr_version_dir, "/usr/share/") debian.ln(solr_version_path, solr_home) debian.rm(archive)
def configure(): """ Configure nginx and enable/disable sites """ with sudo(): # Upload templates context = {'num_cores': debian.nproc()} uploads = blueprint.upload('./', nginx_root, context) # Disable previously enabled sites not configured sites-enabled changes = [] sites = blueprint.get('sites') auto_disable_sites = blueprint.get('auto_disable_sites', True) if auto_disable_sites: with silent(): enabled_site_links = run( 'ls {}'.format(sites_enabled_path)).split() for link in enabled_site_links: link_name = os.path.splitext(link)[0] # Without extension if link not in sites and link_name not in sites: changed = disable(link, do_reload=False) changes.append(changed) ### Enable sites from settings for site in sites: changed = enable(site, do_reload=False) changes.append(changed) ### Reload nginx if new templates or any site has been enabled/disabled if uploads or any(changes): reload()
def enable(site, do_reload=True): """ Enable site :param site: Site to enable :param do_reload: Reload nginx service :return: Got enabled? """ enabled = False site = site if site.endswith( '.conf') or site == 'default' else '{}.conf'.format(site) with sudo(): available_site = os.path.join(sites_available_path, site) if not files.exists(available_site): warn('Invalid site: {}'.format(site)) else: with cd(sites_enabled_path): if not files.exists(site): info('Enabling site: {}', site) with silent(): debian.ln(available_site, site) enabled = True if do_reload: reload() return enabled
def install_solr(): with sudo(): version = blueprint.get('version') version_tuple = tuple(map(int, version.split('.'))) archive = 'solr-{}.tgz'.format(version) if version_tuple < (4, 1, 0): archive = 'apache-{}'.format(archive) url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive) with cd('/tmp'): info('Download {} ({})', 'Solr', version) run('wget {}'.format(url)) info('Extracting archive...') with silent(): run('tar xzf {}'.format(archive)) solr_version_dir = os.path.splitext(archive)[0] solr_version_path = os.path.join('/usr', 'share', solr_version_dir) debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True) if files.exists(solr_version_path): info('Found same existing version, removing it...') debian.rm(solr_version_path, recursive=True) debian.mv(solr_version_dir, '/usr/share/') debian.ln(solr_version_path, solr_home) debian.rm(archive)
def install_user(): with sudo(): user.create_service_user('solr', home=solr_home) # Home dir will be created by solr installation (symbolic link) debian.mkdir('/var/lib/solr', mode=755, owner='solr', group='solr') debian.mkdir('/var/log/solr', mode=755, owner='solr', group='solr')
def setup_schemas(drop=False): """ Create database schemas and grant user privileges :param drop: Drop existing schemas before creation """ schemas = blueprint.get('schemas', {}) with sudo('postgres'): for schema, config in schemas.iteritems(): user, password = config['user'], config.get('password') info('Creating user {}', user) if password: _client_exec("CREATE ROLE %(user)s WITH PASSWORD '%(password)s' LOGIN", user=user, password=password) else: _client_exec("CREATE ROLE %(user)s LOGIN", user=user) if drop: info('Droping schema {}', schema) _client_exec('DROP DATABASE %(name)s', name=schema) info('Creating schema {}', schema) _client_exec('CREATE DATABASE %(name)s', name=schema) info('Granting user {} to schema {}'.format(user, schema)) _client_exec("GRANT ALL PRIVILEGES ON DATABASE %(schema)s to %(user)s", schema=schema, user=user) for ext in blueprint.get('extensions', []): info('Creating extension {}'.format(ext)) _client_exec("CREATE EXTENSION IF NOT EXISTS %(ext)s", ext=ext, schema=schema)
def install(): with sudo(): from blues import java java.install() version = blueprint.get('version', '1.0') info('Adding apt repository for {} version {}', 'elasticsearch', version) repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format( version) debian.add_apt_repository(repository) info('Adding apt key for', repository) debian.add_apt_key( 'http://packages.elasticsearch.org/GPG-KEY-elasticsearch') debian.apt_get('update') # Install elasticsearch (and java) info('Installing {} version {}', 'elasticsearch', version) debian.apt_get('install', 'elasticsearch') # Install plugins plugins = blueprint.get('plugins', []) for plugin in plugins: info('Installing elasticsearch "{}" plugin...', plugin) install_plugin(plugin) # Enable on boot debian.add_rc_service('elasticsearch', priorities='defaults 95 10')
def setup(): """ Setup Logstash server """ from .elasticsearch import add_elastic_repo with sudo(): branch = blueprint.get('branch', '6.x') add_elastic_repo(branch) version = blueprint.get('version', 'latest') info('Installing {} version {}', 'logstash', version) package = 'logstash' + ('={}'.format(version) if version != 'latest' else '') debian.apt_get('install', package) # Enable on boot debian.add_rc_service('logstash') # prep custom folders debian.mkdir(conf_available_path) debian.mkdir(conf_enabled_path) # Install plugins plugins = blueprint.get('plugins', []) for plugin in plugins: info('Installing logstash "{}" plugin...', plugin) install_plugin(plugin) configure()
def install_user(): with sudo(): user.create_service_user("solr", home=solr_home) # Home dir will be created by solr installation (symbolic link) debian.mkdir("/var/lib/solr", mode=755, owner="solr", group="solr") debian.mkdir("/var/log/solr", mode=755, owner="solr", group="solr")
def install(): with sudo(): from blues import java java.install() version = blueprint.get('version', '1.0') info('Adding apt repository for {} version {}', 'elasticsearch', version) repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format(version) debian.add_apt_repository(repository) info('Adding apt key for', repository) debian.add_apt_key('http://packages.elasticsearch.org/GPG-KEY-elasticsearch') debian.apt_get('update') # Install elasticsearch (and java) info('Installing {} version {}', 'elasticsearch', version) debian.apt_get('install', 'elasticsearch') # Install plugins plugins = blueprint.get('plugins', []) for plugin in plugins: info('Installing elasticsearch "{}" plugin...', plugin) install_plugin(plugin) # Enable on boot debian.add_rc_service('elasticsearch', priorities='defaults 95 10')
def add_apt_ppa(name, accept=True, src=False): with sudo(), fabric.context_managers.cd('/etc/apt/sources.list.d'): source_list = '%s-%s.list' % (name.replace('/', '-'), lbs_codename()) if not fabric.contrib.files.exists(source_list): add_apt_repository('ppa:{}'.format(name), accept=accept, src=src) apt_get('update')
def generate_pgtune_conf(role='db'): """ Run pgtune and create pgtune.conf :param role: Which fabric role to place local pgtune.conf template under """ conf_path = postgres_root('postgresql.conf') with sudo(), silent(): output = run('pgtune -T Web -i {}'.format(conf_path)).strip() def parse(c): lines = [l for l in c.splitlines() if '# pgtune' in l] for line in lines: try: comment = line.index('#') line = line[:comment] except ValueError: pass clean = lambda s: s.strip('\n\r\t\'" ') key, _, value = line.partition('=') key, value = map(clean, (key, value)) if key: yield key, value or None tune_conf = dict(parse(output)) tune_conf.update(blueprint.get('pgtune', {})) tune_conf = '\n'.join((' = '.join(item)) for item in tune_conf.iteritems()) conf_dir = os.path.join(os.path.dirname(env['real_fabfile']), 'templates', role, 'postgres') conf_path = os.path.join(conf_dir, 'pgtune.conf') if not os.path.exists(conf_dir): os.makedirs(conf_dir) with open(conf_path, 'w+') as f: f.write(tune_conf)
def install(): with sudo(): info('Adding apt repository for {}', 'docker') repository = '[arch=amd64] https://download.docker.com/linux/ubuntu {} stable'.format( debian.lsb_codename()) debian.add_apt_repository(repository) gpg_key = 'https://download.docker.com/linux/ubuntu/gpg' info('Adding apt key from {}', gpg_key) debian.add_apt_key(gpg_key) debian.apt_get_update() # Install Docker version = blueprint.get('version', 'latest') info('Target version {}', version) package = 'docker-ce' + ('={}'.format(version) if version != 'latest' else '') info('Installing {}', package) debian.apt_get('install', package) if debian.lsb_release() == '14.04': blueprint.upload('./docker.default', '/etc/default/docker') else: blueprint.upload('./docker.service', '/etc/systemd/system/docker.service') debian.ln( '/etc/systemd/system/docker.service', '/etc/systemd/system/multi-user.target.wants/docker.service', ) debian.systemd_daemon_reload()
def install(): with sudo(): info("Installing Ruby v1.9.3") debian.apt_get("install", "ruby1.9.3") info("Installing Bundler") gem("install", "bundler")
def setup(): """ Install nfs-common """ with sudo(): debian.apt_get('install', 'nfs-common') configure()
def enable(program, do_reload=True): """ Enable program. :param program: Program to enable :param do_reload: Reload supervisor :return: Got enabled? """ enabled = False program = program if program.endswith(".conf") or program == "default" else "{}.conf".format(program) with sudo(): available_program = os.path.join(programs_available_path, program) if not files.exists(available_program): warn("Invalid program: {}".format(program)) else: with cd(programs_enabled_path): if not files.exists(program): info("Enabling program: {}", program) with silent(): debian.ln(available_program, program) enabled = True if do_reload: reload() return enabled
def install_system_dependencies(): """ Install system wide packages that application depends on. """ with sudo(), silent(): info('Install system dependencies') system_dependencies = blueprint.get('system_dependencies') if system_dependencies: dependencies = [] repositories = [] ppa_dependencies = [] for dependency in system_dependencies: dep, _, rep = dependency.partition('@') if rep: if rep not in repositories: repositories.append(rep) ppa_dependencies.append(dep) elif dep not in dependencies: dependencies.append(dep) debian.apt_get_update() debian.apt_get('install', *dependencies) if repositories: for repository in repositories: debian.add_apt_repository(repository, src=True) debian.apt_get_update() debian.apt_get('install', *ppa_dependencies)
def install_project_user(): """ Create project user and groups. Create user home dir. Disable ssh host checking. Create log dir. """ from .project import project_home with sudo(): info('Install application user') username = blueprint.get('project') home_path = project_home() # Setup groups for project user project_user_groups = ['app-data', 'www-data'] for group in project_user_groups: debian.groupadd(group, gid_min=10000) # Get UID for project user user.create_system_user(username, groups=project_user_groups, home=home_path) # Create application log path application_log_path = os.path.join('/var', 'log', username) debian.mkdir(application_log_path, group='app-data', mode=1775) # Configure ssh for github user.set_strict_host_checking(username, 'github.com')
def install(): with sudo(): debian.apt_get('install', 'pure-ftpd', 'openssl') # Create ftp user user.create_service_user(ftp_user, groups=[ftp_group]) # Create ftp root dir debian.mkdir(ftp_root, mode=1770, owner=ftp_user, group=ftp_group) # Set up symlinks debian.ln('/etc/pure-ftpd/conf/PureDB', '/etc/pure-ftpd/auth/PureDB') # Enable TLS run('echo 1 > /etc/pure-ftpd/conf/TLS') key_path = '/etc/ssl/private/pure-ftpd.pem' if not files.exists(key_path): prompts = { 'Country Name (2 letter code) [AU]:': '', 'State or Province Name (full name) [Some-State]:': '', 'Locality Name (eg, city) []:': '', 'Organization Name (eg, company) [Internet Widgits Pty Ltd]:': '', 'Organizational Unit Name (eg, section) []:': '', 'Common Name (e.g. server FQDN or YOUR name) []:': '', 'Email Address []:': '' } with settings(prompts=prompts): run('openssl req -x509 -nodes -newkey rsa:2048 -keyout {0} -out {0}'.format( key_path)) debian.chmod(key_path, 600)
def install_forwarder(): """ TODO: Build from github - wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz - gunzip <go> - mv go /usr/local/ - apt-get install unzip make ruby ruby-dev - wget https://github.com/elasticsearch/logstash-forwarder/archive/master.zip - unzip <forwarder> - cd <forwarder> - go build - gem install fpm - make deb - dpkg -i <forwarder> """ with sudo(): info('Adding apt repository for {}', 'logstash forwarder') debian.add_apt_repository( 'http://packages.elasticsearch.org/logstashforwarder/debian stable main' ) info('Installing {}', 'logstash forwarder') debian.apt_get('update') debian.apt_get('install', 'logstash-forwarder') # Upload init script blueprint.upload('forwarder/init.d/logstash-forwarder', '/etc/init.d/') debian.chmod('/etc/init.d/logstash-forwarder', mode=755) # Enable on boot debian.add_rc_service('logstash-forwarder')
def enable(conf, weight, do_restart=True): """ Enable logstash input/output provider :param conf: Input or output provider config file :param weight: Weight of provider :param do_restart: Restart service :return: Got enabled? """ enabled = False conf = conf if conf.endswith('.conf') else '{}.conf'.format(conf) with sudo(): available_conf = os.path.join(conf_available_path, conf) if not files.exists(available_conf): warn('Invalid conf: {}'.format(conf)) else: with cd(conf_enabled_path): weight = str(weight).zfill(2) conf = '{}-{}'.format(weight, conf) if not files.exists(conf): info('Enabling conf: {}', conf) with silent(): debian.ln(available_conf, conf) enabled = True if do_restart: restart('server') return enabled
def add_fstab(filesystem=None, mount_point=None, type='auto', options='rw', dump='0', pazz='0'): """ Add mount point configuration to /etc/fstab. If mount point already mounted on different file system then unmount. :param str filesystem: The partition or storage device to be mounted :param str mount_point: The mount point where <filesystem> is mounted to :param str type: The file system type (Default: auto) :param str options: Mount options of the filesystem (Default: rw) :param str dump: Used by the dump utility to decide when to make a backup, 0|1 (Default: 0) :param str pazz: Used by fsck to decide which order filesystems are to be checked (Default: 0) """ with sudo(): fstab_line = '{fs} {mount} {type} {options} {dump} {pazz}'.format( fs=filesystem, mount=mount_point, type=type, options=options, dump=dump, pazz=pazz ) validate_boot_options(options) # Add mount to /etc/fstab if not already there (?) with silent(): output = run('cat /etc/fstab') fstab = output.stdout if fstab_line not in fstab.split('\n'): # TODO: Handle comments info('Adding fstab: {} on {}', filesystem, mount_point) fabric.contrib.files.append('/etc/fstab', fstab_line, use_sudo=True) # Unmount any previous mismatching mount point mounted_file_system = get_mount(mount_point) if mounted_file_system and mounted_file_system != filesystem: unmount(mount_point)
def install_project_user(): """ Create project user and groups. Create user home dir. Disable ssh host checking. Create log dir. """ with sudo(): info('Install application user') username = blueprint.get('project') home_path = project_home() # Setup groups for project user project_user_groups = ['app-data', 'www-data'] for group in project_user_groups: debian.groupadd(group, gid_min=10000) # Get UID for project user user.create_system_user(username, groups=project_user_groups, home=home_path) # Configure ssh for github user.set_strict_host_checking(username, 'github.com') # Create application log path application_log_path = os.path.join('/var', 'log', username) debian.mkdir(application_log_path, group='app-data', mode=1775)
def install(): with sudo(): debian.apt_get('install', 'pure-ftpd', 'openssl') # Create ftp user user.create_service_user(ftp_user, groups=[ftp_group]) # Create ftp root dir debian.mkdir(ftp_root, mode=1770, owner=ftp_user, group=ftp_group) # Set up symlinks debian.ln('/etc/pure-ftpd/conf/PureDB', '/etc/pure-ftpd/auth/PureDB') # Enable TLS run('echo 1 > /etc/pure-ftpd/conf/TLS') key_path = '/etc/ssl/private/pure-ftpd.pem' if not files.exists(key_path): prompts = { 'Country Name (2 letter code) [AU]:': '', 'State or Province Name (full name) [Some-State]:': '', 'Locality Name (eg, city) []:': '', 'Organization Name (eg, company) [Internet Widgits Pty Ltd]:': '', 'Organizational Unit Name (eg, section) []:': '', 'Common Name (e.g. server FQDN or YOUR name) []:': '', 'Email Address []:': '' } with settings(prompts=prompts): run('openssl req -x509 -nodes -newkey rsa:2048 -keyout {0} -out {0}' .format(key_path)) debian.chmod(key_path, 600)
def configure_infra(): with sudo(): info('Adding license key to config') context = {"newrelic_key": blueprint.get('newrelic_key', None)} blueprint.upload('newrelic-infra.yml', '/etc/newrelic-infra.yml', context=context)
def configure(): """ Configure nginx and enable/disable sites """ with sudo(): # Upload templates context = { 'num_cores': debian.nproc() } uploads = blueprint.upload('./', nginx_root, context) # Disable previously enabled sites not configured sites-enabled changes = [] sites = blueprint.get('sites') auto_disable_sites = blueprint.get('auto_disable_sites', True) if auto_disable_sites: with silent(): enabled_site_links = run('ls {}'.format(sites_enabled_path)).split() for link in enabled_site_links: link_name = os.path.splitext(link)[0] # Without extension if link not in sites and link_name not in sites: changed = disable(link, do_reload=False) changes.append(changed) ### Enable sites from settings for site in sites: changed = enable(site, do_reload=False) changes.append(changed) ### Reload nginx if new templates or any site has been enabled/disabled if uploads or any(changes): reload()
def install(): with sudo(): # TODO: Change to java.install (openjdk)? debian.add_apt_ppa('webupd8team/java') debian.debconf_set_selections( 'shared/accepted-oracle-license-v1-1 select true', 'shared/accepted-oracle-license-v1-1 seen true') version = blueprint.get('version', '1.0') info('Adding apt repository for {} version {}', 'elasticsearch', version) repository = 'http://packages.elasticsearch.org/elasticsearch/{0}/debian stable main'.format( version) debian.add_apt_repository(repository) info('Adding apt key for', repository) debian.add_apt_key( 'http://packages.elasticsearch.org/GPG-KEY-elasticsearch') debian.apt_get('update') # Install elasticsearch (and java) info('Installing {} version {}', 'elasticsearch', version) debian.apt_get('install', 'oracle-java7-installer', 'elasticsearch') # Enable on boot debian.add_rc_service('elasticsearch', priorities='defaults 95 10')
def enable(program, do_reload=True): """ Enable program. :param program: Program to enable :param do_reload: Reload supervisor :return: Got enabled? """ enabled = False program = program if program.endswith( '.conf') or program == 'default' else '{}.conf'.format(program) with sudo(): available_program = os.path.join(programs_available_path, program) if not files.exists(available_program): warn('Invalid program: {}'.format(program)) else: with cd(programs_enabled_path): if not files.exists(program): info('Enabling program: {}', program) with silent(): debian.ln(available_program, program) enabled = True if do_reload: reload() return enabled
def setup_schemas(drop=False): """ Create database schemas and grant user privileges :param drop: Drop existing schemas before creation """ schemas = blueprint.get('schemas', {}) with sudo('postgres'): for schema, config in schemas.iteritems(): user, password = config['user'], config.get('password') info('Creating user {}', user) if password: _client_exec( "CREATE ROLE %(user)s WITH PASSWORD '%(password)s' LOGIN", user=user, password=password) else: _client_exec("CREATE ROLE %(user)s LOGIN", user=user) if drop: info('Droping schema {}', schema) _client_exec('DROP DATABASE %(name)s', name=schema) info('Creating schema {}', schema) _client_exec('CREATE DATABASE %(name)s', name=schema) info('Granting user {} to schema {}'.format(user, schema)) _client_exec( "GRANT ALL PRIVILEGES ON DATABASE %(schema)s to %(user)s", schema=schema, user=user)
def enable(conf, weight, do_restart=True): """ Enable logstash input/output provider :param conf: Input or output provider config file :param weight: Weight of provider :param do_restart: Restart service :return: Got enabled? """ enabled = False conf = conf if conf.endswith(".conf") else "{}.conf".format(conf) with sudo(): available_conf = os.path.join(conf_available_path, conf) if not files.exists(available_conf): warn("Invalid conf: {}".format(conf)) else: with cd(conf_enabled_path): weight = str(weight).zfill(2) conf = "{}-{}".format(weight, conf) if not files.exists(conf): info("Enabling conf: {}", conf) with silent(): debian.ln(available_conf, conf) enabled = True if do_restart: restart("server") return enabled
def configure(): """ Upload vassals """ with sudo(): # Upload templates blueprint.upload('init/', '/etc/init/')
def install_forwarder(): """ TODO: Build from github - wget https://storage.googleapis.com/golang/go1.4.1.linux-amd64.tar.gz - gunzip <go> - mv go /usr/local/ - apt-get install unzip make ruby ruby-dev - wget https://github.com/elasticsearch/logstash-forwarder/archive/master.zip - unzip <forwarder> - cd <forwarder> - go build - gem install fpm - make deb - dpkg -i <forwarder> """ with sudo(): info("Adding apt repository for {}", "logstash forwarder") debian.add_apt_repository("http://packages.elasticsearch.org/logstashforwarder/debian stable main") info("Installing {}", "logstash forwarder") debian.apt_get("update") debian.apt_get("install", "logstash-forwarder") # Upload init script blueprint.upload("forwarder/init.d/logstash-forwarder", "/etc/init.d/") debian.chmod("/etc/init.d/logstash-forwarder", mode=755) # Enable on boot debian.add_rc_service("logstash-forwarder")
def install(): from refabric.context_managers import sudo from .debian import add_apt_ppa, apt_get with sudo(): add_apt_ppa('vbernat/haproxy-1.5', src=True) apt_get('install', 'haproxy')
def install(): with sudo(): # Generate a root password and save it in root home root_conf_path = '/root/.my.cnf' if not fabric.contrib.files.exists(root_conf_path): root_pw = generate_password() blueprint.upload('root_my.cnf', '/root/.my.cnf', {'password': root_pw}) debian.chmod('/root/.my.cnf', mode=600) else: # TODO: use fabric.operations.get instead of cat when up to date with upstream with silent(): output = run('cat {}'.format(root_conf_path)) fd = StringIO(output) config_parser = ConfigParser.RawConfigParser() config_parser.readfp(fd) root_pw = config_parser.get('client', 'password') # Install external PPA info('Adding apt key for {}', __name__) run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A") info('Adding apt repository for {}', __name__) debian.add_apt_repository('http://repo.percona.com/apt trusty main') debian.apt_get('update') # Percona/MySQL base dependencies dependencies = ( 'percona-server-server', 'percona-server-client', 'libmysqlclient-dev', 'mysqltuner' ) # Configure debconf to autoset root password on installation prompts server_package = dependencies[0] debian.debconf_communicate('PURGE', server_package) with silent(): debian.debconf_set_selections( '{}/root_password password {}'.format(server_package, root_pw), '{}/root_password_again password {}'.format(server_package, root_pw) ) # Install package info('Installing {}', __name__) debian.apt_get('install', *dependencies) debian.debconf_communicate('PURGE', server_package) # Auto-answer mysql_secure_installation prompts prompts = { 'Enter current password for root (enter for none): ': root_pw, 'Change the root password? [Y/n] ': 'n', 'Remove anonymous users? [Y/n] ': 'Y', 'Disallow root login remotely? [Y/n] ': 'Y', 'Remove test database and access to it? [Y/n] ': 'Y', 'Reload privilege tables now? [Y/n] ': 'Y' } # Run mysql_secure_installation to remove test-db and remote root login with settings(prompts=prompts): run('mysql_secure_installation')
def configure(): """ Enable/disable configured programs """ with sudo(): # Upload templates uploads = blueprint.upload('init/', '/etc/init/') uploads.extend(blueprint.upload('supervisord.conf', '/etc/')) # Disable previously enabled programs not configured programs-enabled changes = [] programs = blueprint.get('programs') or [] auto_disable = blueprint.get('auto_disable_programs', True) if auto_disable: with silent(): enabled_program_links = run( 'ls {}'.format(programs_enabled_path)).split() for link in enabled_program_links: link_name = os.path.splitext(link)[0] # Without extension if link not in programs and link_name not in programs: changed = disable(link, do_reload=False) changes.append(changed) ### Enable programs from settings for program in programs: changed = enable(program, do_reload=False) changes.append(changed) ### Reload supervisor if new templates or any program has been enabled/disabled if uploads or any(changes): reload()
def enable(site, do_reload=True): """ Enable site :param site: Site to enable :param do_reload: Reload nginx service :return: Got enabled? """ enabled = False site = site if site.endswith('.conf') or site == 'default' else '{}.conf'.format(site) with sudo(): available_site = os.path.join(sites_available_path, site) if not files.exists(available_site): warn('Invalid site: {}'.format(site)) else: with cd(sites_enabled_path): if not files.exists(site): info('Enabling site: {}', site) with silent(): debian.ln(available_site, site) enabled = True if do_reload: reload() return enabled
def configure(): """ Configure PureFTP """ with sudo(): # Echo configurations setup_config() for user in blueprint.get('users'): username, password = user['username'], user['password'] if 'homedir' in user: user_home = user['homedir'] else: user_home = os.path.join(ftp_root, username) passwd_path = '/etc/pure-ftpd/pureftpd.passwd' with settings(warn_only=True): if files.exists(passwd_path) and run( 'pure-pw show {}'.format(username)).return_code == 0: continue debian.mkdir(user_home, owner=ftp_user, group=ftp_group) prompts = {'Password: '******'Enter it again: ': password} with settings(prompts=prompts): run('pure-pw useradd {} -u {} -g {} -d {}'.format( username, ftp_user, ftp_group, user_home)) run('pure-pw mkdb') restart()
def add_apt_ppa(name, accept=True, src=False): with sudo(), fabric.context_managers.cd('/etc/apt/sources.list.d'): source_list = '%s-%s.list' % (name.replace('/', '-'), lsb_codename()) if not fabric.contrib.files.exists(source_list): add_apt_repository('ppa:{}'.format(name), accept=accept, src=src) apt_get_update()
def install_user(): with sudo(): user.create_service_user('solr', home=solr_home) debian.mkdir(solr_home, mode=755, owner='solr', group='solr') debian.mkdir('/var/lib/solr', mode=755, owner='solr', group='solr') debian.mkdir('/var/log/solr', mode=755, owner='solr', group='solr')
def install(): with sudo(): packages = [ 'wkhtmltopdf', 'xvfb', 'xfonts-100dpi', 'xfonts-75dpi', 'xfonts-cyrillic' ] debian.apt_get('install', *packages)
def dump(schema=None, ignore_tables=''): """ Dump and download a schema. :param schema: Specific shema to dump and download. :param ignore_tables: Tables to skip, separated by | (pipe) """ if not schema: schemas = blueprint.get('schemas', {}).keys() for i, schema in enumerate(schemas, start=1): print("{i}. {schema}".format(i=i, schema=schema)) valid_indices = '[1-{}]+'.format(len(schemas)) schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices) schema = schemas[int(schema_choice)-1] now = datetime.now().strftime('%Y-%m-%d') output_file = '/tmp/{}_{}.backup.gz'.format(schema, now) filename = os.path.basename(output_file) info('Dumping schema {}...', schema) extra_args = [] for table in ignore_tables.split('|'): extra_args.append('--ignore-table={}.{}'.format(schema, table)) dump_cmd = 'mysqldump {} {} | gzip > {}'.format(schema, ' '.join(extra_args), output_file) run('sudo su root -c "{}"'.format(dump_cmd)) info('Downloading dump...') local_file = '~/%s' % filename fabric.contrib.files.get(output_file, local_file) with sudo(), silent(): debian.rm(output_file) info('New smoking hot dump at {}', local_file)
def locale_gen(locale, utf8=True): locale_gen_cmd = 'locale-gen {}'.format(locale) locale_gen_utf8_cmd = locale_gen_cmd + '.UTF-8' with sudo(): run(locale_gen_cmd) if utf8: run(locale_gen_utf8_cmd)
def configure(): """ Configure wkhtmltopdf """ destination = '/usr/local/bin/wkhtmltopdf.sh' blueprint.upload('wkhtmltopdf.sh', destination) with sudo(): debian.chmod(destination, '+x')
def flush(): """ Delete all cached keys """ info('Flushing Memcached...') with sudo(), silent(): run('echo "flush_all" | /bin/netcat -q 2 127.0.0.1 11211') info('Down the drain!')
def install(): with sudo(): info('Install python dependencies') debian.apt_get('install', 'python-dev', 'python-setuptools') run('easy_install pip') run('touch {}'.format(pip_log_file)) debian.chmod(pip_log_file, mode=777) pip('install', 'setuptools', '--upgrade')
def export(path, host, options='rw,async,no_root_squash,no_subtree_check'): config_line = '%s\t\t%s(%s)' % (path, host, options) with sudo(), cd('/etc'): if not files.contains('exports', config_line): info('Exporting: {}', path) files.append('exports', config_line) return True return False
def install_virtualenv(): """ Create a project virtualenv. """ with sudo(): virtualenv.install() with sudo_project(): virtualenv.create(virtualenv_path())
def install_system_dependencies(): """ Install system wide packages that application depends on. """ with sudo(): info('Install system dependencies') dependencies = blueprint.get('system_dependencies') if dependencies: debian.apt_get('install', *dependencies)