def install_solr(): with sudo(): version = blueprint.get("version") version_tuple = tuple(map(int, version.split("."))) archive = "solr-{}.tgz".format(version) if version_tuple < (4, 1, 0): archive = "apache-{}".format(archive) url = "https://archive.apache.org/dist/lucene/solr/{}/{}".format(version, archive) with cd("/tmp"): info("Download {} ({})", "Solr", version) run("wget {}".format(url)) info("Extracting archive...") with silent(): run("tar xzf {}".format(archive)) solr_version_dir = os.path.splitext(archive)[0] solr_version_path = os.path.join("/usr", "share", solr_version_dir) debian.chmod(solr_version_dir, 755, "solr", "solr", recursive=True) if files.exists(solr_version_path): info("Found same existing version, removing it...") debian.rm(solr_version_path, recursive=True) debian.mv(solr_version_dir, "/usr/share/") debian.ln(solr_version_path, solr_home) debian.rm(archive)
def dump(schema=None, ignore_tables=''): """ Dump and download a schema. :param schema: Specific shema to dump and download. :param ignore_tables: Tables to skip, separated by | (pipe) """ if not schema: schemas = blueprint.get('schemas', {}).keys() for i, schema in enumerate(schemas, start=1): print("{i}. {schema}".format(i=i, schema=schema)) valid_indices = '[1-{}]+'.format(len(schemas)) schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices) schema = schemas[int(schema_choice)-1] now = datetime.now().strftime('%Y-%m-%d') output_file = '/tmp/{}_{}.backup.gz'.format(schema, now) filename = os.path.basename(output_file) info('Dumping schema {}...', schema) extra_args = [] for table in ignore_tables.split('|'): extra_args.append('--ignore-table={}.{}'.format(schema, table)) dump_cmd = 'mysqldump {} {} | gzip > {}'.format(schema, ' '.join(extra_args), output_file) run('sudo su root -c "{}"'.format(dump_cmd)) info('Downloading dump...') local_file = '~/%s' % filename fabric.contrib.files.get(output_file, local_file) with sudo(), silent(): debian.rm(output_file) info('New smoking hot dump at {}', local_file)
def install_solr(): with sudo(): version = blueprint.get('version') version_tuple = tuple(map(int, version.split('.'))) archive = 'solr-{}.tgz'.format(version) if version_tuple < (4, 1, 0): archive = 'apache-{}'.format(archive) url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format(version, archive) with cd('/tmp'): info('Download {} ({})', 'Solr', version) run('wget {}'.format(url)) info('Extracting archive...') with silent(): run('tar xzf {}'.format(archive)) solr_version_dir = os.path.splitext(archive)[0] solr_version_path = os.path.join('/usr', 'share', solr_version_dir) debian.chmod(solr_version_dir, 755, 'solr', 'solr', recursive=True) if files.exists(solr_version_path): info('Found same existing version, removing it...') debian.rm(solr_version_path, recursive=True) debian.mv(solr_version_dir, '/usr/share/') debian.ln(solr_version_path, solr_home) debian.rm(archive)
def install(): with sudo(): # Generate a root password and save it in root home root_conf_path = '/root/.my.cnf' if not fabric.contrib.files.exists(root_conf_path): root_pw = generate_password() blueprint.upload('root_my.cnf', '/root/.my.cnf', {'password': root_pw}) debian.chmod('/root/.my.cnf', mode=600) else: # TODO: use fabric.operations.get instead of cat when up to date with upstream with silent(): output = run('cat {}'.format(root_conf_path)) fd = StringIO(output) config_parser = ConfigParser.RawConfigParser() config_parser.readfp(fd) root_pw = config_parser.get('client', 'password') # Install external PPA info('Adding apt key for {}', __name__) run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A") info('Adding apt repository for {}', __name__) debian.add_apt_repository('http://repo.percona.com/apt trusty main') debian.apt_get('update') # Percona/MySQL base dependencies dependencies = ( 'percona-server-server', 'percona-server-client', 'libmysqlclient-dev', 'mysqltuner' ) # Configure debconf to autoset root password on installation prompts server_package = dependencies[0] debian.debconf_communicate('PURGE', server_package) with silent(): debian.debconf_set_selections( '{}/root_password password {}'.format(server_package, root_pw), '{}/root_password_again password {}'.format(server_package, root_pw) ) # Install package info('Installing {}', __name__) debian.apt_get('install', *dependencies) debian.debconf_communicate('PURGE', server_package) # Auto-answer mysql_secure_installation prompts prompts = { 'Enter current password for root (enter for none): ': root_pw, 'Change the root password? [Y/n] ': 'n', 'Remove anonymous users? [Y/n] ': 'Y', 'Disallow root login remotely? [Y/n] ': 'Y', 'Remove test database and access to it? [Y/n] ': 'Y', 'Reload privilege tables now? [Y/n] ': 'Y' } # Run mysql_secure_installation to remove test-db and remote root login with settings(prompts=prompts): run('mysql_secure_installation')
def update_modules(beat): changes = [] # Get desired state desired_modules = set(blueprint.get('{}.modules'.format(beat), [])) # Get current state with silent(): module_files = run('find /etc/{}/modules.d -iname "*.yml"'.format(beat)).split() enabled_modules = {os.path.basename(module).split('.')[0] for module in module_files} # Disable extra services for extra in enabled_modules - desired_modules: info('Disabling {} module: {}', beat, extra) changes.append(extra) with silent(), sudo(): run('{} modules disable {}'.format(beat, extra)) # Enable services for missing in desired_modules - enabled_modules: info('Enabling {} module: {}', beat, missing) changes.append(missing) with silent(), sudo(): run('{} modules enable {}'.format(beat, missing)) return changes
def dump(schema=None): """ Dump and download all configured, or given, schemas. :param schema: Specific shema to dump and download. """ if not schema: schemas = blueprint.get('schemas', {}).keys() for i, schema in enumerate(schemas, start=1): print("{i}. {schema}".format(i=i, schema=schema)) valid_indices = '[1-{}]+'.format(len(schemas)) schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices) schema = schemas[int(schema_choice) - 1] with sudo('postgres'): now = datetime.now().strftime('%Y-%m-%d') output_file = '/tmp/{}_{}.backup'.format(schema, now) filename = os.path.basename(output_file) options = dict(format='tar', output_file=output_file, schema=schema) info('Dumping schema {}...', schema) run('pg_dump -c -F {format} -f {output_file} {schema}'.format( **options)) info('Downloading dump...') local_file = '~/{}'.format(filename) files.get(output_file, local_file) with sudo(), silent(): debian.rm(output_file) info('New smoking hot dump at {}', local_file)
def send_deploy_event(): newrelic_key = blueprint.get('newrelic_key', None) app_name = blueprint.get('app_name', None) if newrelic_key and app_name: url = 'https://api.newrelic.com/deployments.xml' headers = {'x-api-key': newrelic_key} with cd(python_path()): tag = run('git describe --tags') commit_hash = run('git rev-parse HEAD') deployer = git.get_local_commiter() payload = { 'deployment[app_name]': app_name, # 'application_id': '1234567', 'deployment[description]': tag, 'deployment[revision]': commit_hash, # 'deployment[changelog]': changes, 'deployment[user]': deployer, } response = requests.post(url, data=payload, headers=headers) info(response.text) else: info('No key found') #http://gc-taylor.com/blog/2013/02/11/fabric-task-for-notifying-new-relic-code-deploy/#sthash.5AnhN3An.sfju
def install(): with sudo(): info('Install python dependencies') debian.apt_get('install', 'python-dev', 'python-setuptools') run('easy_install pip') run('touch {}'.format(pip_log_file)) debian.chmod(pip_log_file, mode=777) pip('install', 'setuptools', '--upgrade')
def flush(): """ Delete all cached keys """ info('Flushing Memcached...') with sudo(), silent(): run('echo "flush_all" | /bin/netcat -q 2 127.0.0.1 11211') info('Down the drain!')
def pip_sync(manifest, quiet=False): pip('install', 'pip-tools', quiet=quiet) options = "" if quiet: options += ' -q' run('pip-sync{} {}'.format(options, manifest), pty=False)
def install(): with sudo(): # Generate a root password and save it in root home root_conf_path = '/root/.my.cnf' if not fabric.contrib.files.exists(root_conf_path): root_pw = generate_password() blueprint.upload('root_my.cnf', '/root/.my.cnf', {'password': root_pw}) debian.chmod('/root/.my.cnf', mode=600) else: # TODO: use fabric.operations.get instead of cat when up to date with upstream with silent(): output = run('cat {}'.format(root_conf_path)) fd = StringIO(output) config_parser = ConfigParser.RawConfigParser() config_parser.readfp(fd) root_pw = config_parser.get('client', 'password') # Install external PPA info('Adding apt key for {}', __name__) run("apt-key adv --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A" ) info('Adding apt repository for {}', __name__) debian.add_apt_repository('http://repo.percona.com/apt trusty main') debian.apt_get('update') # Percona/MySQL base dependencies dependencies = ('percona-server-server', 'percona-server-client', 'libmysqlclient-dev', 'mysqltuner') # Configure debconf to autoset root password on installation prompts server_package = dependencies[0] debian.debconf_communicate('PURGE', server_package) with silent(): debian.debconf_set_selections( '{}/root_password password {}'.format(server_package, root_pw), '{}/root_password_again password {}'.format( server_package, root_pw)) # Install package info('Installing {}', __name__) debian.apt_get('install', *dependencies) debian.debconf_communicate('PURGE', server_package) # Auto-answer mysql_secure_installation prompts prompts = { 'Enter current password for root (enter for none): ': root_pw, 'Change the root password? [Y/n] ': 'n', 'Remove anonymous users? [Y/n] ': 'Y', 'Disallow root login remotely? [Y/n] ': 'Y', 'Remove test database and access to it? [Y/n] ': 'Y', 'Reload privilege tables now? [Y/n] ': 'Y' } # Run mysql_secure_installation to remove test-db and remote root login with settings(prompts=prompts): run('mysql_secure_installation')
def create_server_ssl_cert(): with sudo(): info('Generating SSL certificate...') debian.mkdir('/etc/pki/tls/certs') debian.mkdir('/etc/pki/tls/private') with cd('/etc/pki/tls'): key = 'private/logstash-forwarder.key' crt = 'certs/logstash-forwarder.crt' run('openssl req -x509 -batch -nodes -days 3650 -newkey rsa:2048 -keyout {} -out {}'.format(key, crt))
def configure(): """ Configure newrelic server """ with sudo(): info('Adding license key to config') newrelic_key = blueprint.get('newrelic_key', None) run('nrsysmond-config --set license_key={}'.format(newrelic_key))
def ctl(command=None): """ Run rabbitmqctl with given command :param command: Control command to execute """ if not command: abort('No command given, $ fab rabbitmq.ctl:stop_app') with sudo(): run('rabbitmqctl {}'.format(command))
def reload(self, vassals=None): """ Touch reload specified vassals :param vassals: Vassals to reload """ for vassal_ini in vassals or self.list_vassals(): vassal_ini_path = os.path.join(self.get_config_path(), vassal_ini) with sudo(), silent(): run('touch {}'.format(vassal_ini_path))
def ctl(command=None): """ Run rabbitmqctl with given command :param command: :return: """ if not command: abort('No command given, $ fab rabbitmq.ctl:stop_app') with sudo(): run('rabbitmqctl {}'.format(command))
def top(vassal_name=None): """ Launch uwsgitop for vassal stats socket :param vassal_name: The vassal to show stats for (Default: project setting) """ # TODO: fix missing output with sudo(), hide_prefix(): vassal = vassal_name or blueprint.get('project') stats_path = os.path.join(tmpfs_path, '{}-stats.sock'.format(vassal)) run('uwsgitop {}'.format(stats_path))
def create(path): options = '' if python.requested_version() >= (3,): options += ' -p /usr/bin/python3' if not files.exists(path): info('Creating virtualenv: {}', path) run('virtualenv{options} {}'.format(path, options=options)) else: info('Virtualenv already exists: {}', path)
def fifo(vassal_name, command): """ Issue FIFO commands to a vassal. :param vassal_name: The vassal to command :param command: The FIFO command to issue See: http://uwsgi-docs.readthedocs.org/en/latest/MasterFIFO.html """ fifo_file = '/run/uwsgi/fifo-{}'.format(vassal_name) with sudo(), silent(): run('echo {} > {}'.format(command, fifo_file))
def reload(vassal_path=None): """ Reload uwsgi or reload specific vassal @ path, via touch. :param vassal_path: The absolute path to vassal ini to reload. If not given, the uwsgi service will reload """ if not vassal_path: debian.service('uwsgi', 'reload', check_status=False) else: vassal_name = os.path.splitext(os.path.basename(vassal_path))[0] with sudo(), silent(): info('Reloading {} uWSGI vassal', vassal_name) run('touch {}'.format(vassal_path))
def reload(vassal_path=None): """ Reload uwsgi or reload specific vassal @ path, via touch. :param vassal_path: The absolute path to vassal ini to reload. If not given, the uwsgi service will reload """ if not vassal_path: debian.service('uwsgi', 'reload', check_status=False) else: vassal_name = os.path.splitext(os.path.basename(vassal_path))[0] with sudo(): info('Reloading {} uWSGI vassal', vassal_name) run('touch {}'.format(vassal_path))
def install(): with sudo(): info('Downloading wowza') version = blueprint.get('wowza_version', '4.1.2') binary = 'WowzaStreamingEngine-{}.deb.bin'.format(version) version_path = version.replace('.', '-') url = 'http://www.wowza.com/downloads/WowzaStreamingEngine-{}/{}'.format( version_path, binary) run('wget -P /tmp/ {url}'.format(url=url)) debian.chmod('/tmp/{}'.format(binary), '+x') info('Installing wowza') run('/tmp/{}'.format(binary))
def install(): with sudo(): info('Downloading kibana') version = blueprint.get('version', '3.1.2') tar_file = 'kibana-{}.tar.gz'.format(version) run('wget -P /tmp/ https://download.elasticsearch.org/kibana/kibana/{f}'.format(f=tar_file)) # Extract and soft link kibana in web root web_root = '/srv/www/' debian.mkdir(web_root, mode=1775, owner='www-data', group='www-data') run('tar xzf /tmp/{f} -C {web_root}'.format(f=tar_file, web_root=web_root)) src_root = os.path.join(web_root, 'kibana-{version}'.format(version=version)) debian.chown(src_root, owner='www-data', group='www-data', recursive=True) debian.ln(src_root, '/srv/www/kibana')
def install_testing(): package_name = 'rabbitmq-server' debian.debconf_set_selections('%s rabbitmq-server/upgrade_previous note' % package_name) with sudo(): info('Adding apt key for {}', package_name) run("apt-key adv --keyserver pgp.mit.edu --recv-keys 0x056E8E56") info('Adding apt repository for {}', package_name) debian.add_apt_repository('http://www.rabbitmq.com/debian/ testing main') debian.apt_get_update() info('Installing {}', package_name) debian.apt_get('install', package_name)
def install(): with sudo(): info('Downloading wowza') version = blueprint.get('wowza_version', '4.1.2') binary = 'WowzaStreamingEngine-{}.deb.bin'.format(version) version_path = version.replace('.', '-') url = 'http://www.wowza.com/downloads/WowzaStreamingEngine-{}/{}'.format(version_path, binary) run('wget -P /tmp/ {url}'.format(url=url)) debian.chmod('/tmp/{}'.format(binary), '+x') info('Installing wowza') run('/tmp/{}'.format(binary))
def create_server_ssl_cert(): with sudo(): info("Generating SSL certificate...") debian.mkdir("/etc/pki/tls/certs") debian.mkdir("/etc/pki/tls/private") with cd("/etc/pki/tls"): hostname = debian.hostname() key = "private/logstash-forwarder.key" crt = "certs/logstash-forwarder.crt" run( "openssl req -x509 -batch -nodes -days 3650 -newkey rsa:2048 " "-keyout {} " "-out {} " '-subj "/CN={}"'.format(key, crt, hostname) )
def configure(): """ Configure nginx and enable/disable sites """ with sudo(): # Upload templates context = { 'num_cores': debian.nproc() } uploads = blueprint.upload('./', nginx_root, context) # Disable previously enabled sites not configured sites-enabled changes = [] sites = blueprint.get('sites') auto_disable_sites = blueprint.get('auto_disable_sites', True) if auto_disable_sites: with silent(): enabled_site_links = run('ls {}'.format(sites_enabled_path)).split() for link in enabled_site_links: link_name = os.path.splitext(link)[0] # Without extension if link not in sites and link_name not in sites: changed = disable(link, do_reload=False) changes.append(changed) ### Enable sites from settings for site in sites: changed = enable(site, do_reload=False) changes.append(changed) ### Reload nginx if new templates or any site has been enabled/disabled if uploads or any(changes): reload()
def generate_pgtune_conf(role='db'): """ Run pgtune and create pgtune.conf :param role: Which fabric role to place local pgtune.conf template under """ conf_path = postgres_root('postgresql.conf') with sudo(), silent(): output = run('pgtune -T Web -i {}'.format(conf_path)).strip() def parse(c): lines = [l for l in c.splitlines() if '# pgtune' in l] for line in lines: try: comment = line.index('#') line = line[:comment] except ValueError: pass clean = lambda s: s.strip('\n\r\t\'" ') key, _, value = line.partition('=') key, value = map(clean, (key, value)) if key: yield key, value or None tune_conf = dict(parse(output)) tune_conf.update(blueprint.get('pgtune', {})) tune_conf = '\n'.join((' = '.join(item)) for item in tune_conf.iteritems()) conf_dir = os.path.join(os.path.dirname(env['real_fabfile']), 'templates', role, 'postgres') conf_path = os.path.join(conf_dir, 'pgtune.conf') if not os.path.exists(conf_dir): os.makedirs(conf_dir) with open(conf_path, 'w+') as f: f.write(tune_conf)
def clone(url, branch=None, repository_path=None, **kwargs): """ Clone repository and branch. :param url: Git url to clone :param branch: Branch to checkout :param repository_path: Destination :param kwargs: Not used but here for easier kwarg passing :return: (destination, got_cloned bool) """ repository = parse_url(url, branch=branch) name = repository['name'] branch = repository['branch'] cloned = False if not repository_path: repository_path = os.path.join('.', name) if not files.exists(os.path.join(repository_path, '.git')): info('Cloning {}@{} into {}', url, branch, repository_path) with silent('warnings'): cmd = 'git clone -b {branch} {remote} {name}'.format(branch=branch, remote=url, name=name) output = run(cmd) if output.return_code != 0: warn('Failed to clone repository "{}", probably permission denied!'.format(name)) cloned = None else: cloned = True else: info('Git repository already cloned: {}', name) return repository_path, cloned
def configure_server(config, auto_disable_conf=True, **context): context.setdefault('use_ssl', True) context.setdefault('elasticsearch_host', '127.0.0.1') uploads = blueprint.upload('./server/', '/etc/logstash/', context) # Disable previously enabled conf not configured through config in settings changes = [] if auto_disable_conf: with silent(): enabled_conf_links = run('ls {}'.format(conf_enabled_path)).split() conf_prospects = [ '{}-{}.conf'.format(str(weight).zfill(2), conf) for weight, conf in config.iteritems() ] for link in enabled_conf_links: if link not in conf_prospects: changed = disable(link, do_restart=False) changes.append(changed) # Enable conf from settings for weight, conf in config.iteritems(): changed = enable(conf, weight, do_restart=False) changes.append(changed) return bool(uploads or any(changes))
def reset(branch, repository_path=None, **kwargs): """ Fetch, reset, clean and checkout repository branch. :return: commit """ commit = None if not repository_path: repository_path = debian.pwd() with cd(repository_path): name = os.path.basename(repository_path) info('Resetting git repository: {}@{}', name, branch) with silent('warnings'): commands = [ 'git fetch origin', # Fetch branches and tags 'git reset --hard HEAD', # Make hard reset to HEAD 'git clean -fdx', # Remove untracked files pyc, xxx~ etc 'git checkout HEAD', # Checkout HEAD 'git reset refs/remotes/origin/{} --hard'.format(branch) # Reset to branch ] output = run(' && '.join(commands)) if output.return_code != 0: warn('Failed to reset repository "{}", probably permission denied!'.format(name)) else: output = output.split(os.linesep)[-1][len('HEAD is now at '):] commit = output.split()[0] info('HEAD is now at: {}', output) return commit
def install_testing(): package_name = 'rabbitmq-server' debian.debconf_set_selections('%s rabbitmq-server/upgrade_previous note' % package_name) with sudo(): info('Adding apt key for {}', package_name) run("apt-key adv --keyserver pgp.mit.edu --recv-keys 0x056E8E56") info('Adding apt repository for {}', package_name) debian.add_apt_repository( 'http://www.rabbitmq.com/debian/ testing main') debian.apt_get('update') info('Installing {}', package_name) debian.apt_get('install', package_name)
def configure(): """ Enable/disable configured programs """ with sudo(): # Upload templates uploads = blueprint.upload("init/", "/etc/init/") uploads.extend(blueprint.upload("supervisord.conf", "/etc/")) # Disable previously enabled programs not configured programs-enabled changes = [] programs = blueprint.get("programs") or [] auto_disable = blueprint.get("auto_disable_programs", True) if auto_disable: with silent(): enabled_program_links = run("ls {}".format(programs_enabled_path)).split() for link in enabled_program_links: link_name = os.path.splitext(link)[0] # Without extension if link not in programs and link_name not in programs: changed = disable(link, do_reload=False) changes.append(changed) ### Enable programs from settings for program in programs: changed = enable(program, do_reload=False) changes.append(changed) ### Reload supervisor if new templates or any program has been enabled/disabled if uploads or any(changes): reload()
def reset(branch, repository_path=None, **kwargs): """ Fetch, reset, clean and checkout repository branch. :return: commit """ if not repository_path: repository_path = debian.pwd() with cd(repository_path): name = os.path.basename(repository_path) info('Resetting git repository: {}@{}', name, branch) commands = [ 'git fetch origin', # Fetch branches and tags 'git reset --hard HEAD', # Make hard reset to HEAD 'git clean -fdx', # Remove untracked files pyc, xxx~ etc 'git checkout HEAD', # Checkout HEAD 'git reset refs/remotes/origin/{} --hard'.format( branch) # Reset to branch ] with silent(): output = run(' && '.join(commands)) output = output.split(os.linesep)[-1].lstrip('HEAD is now at ') commit = output.split()[0] info('HEAD is now at: {}', output) return commit
def version(): if not hasattr(version, 'version'): version_string = run('python --version').stdout _, ver = version_string.split(' ') version.version = tuple(map(int, ver.split('.'))) return version.version
def info(scope=''): """ Get runtime information from redis itself """ with silent(), hide_prefix(): output = api.run('redis-cli info ' + scope) api.info(output)
def configure(): """ Configure nginx and enable/disable sites """ with sudo(): # Upload templates context = {'num_cores': debian.nproc()} uploads = blueprint.upload('./', nginx_root, context) # Disable previously enabled sites not configured sites-enabled changes = [] sites = blueprint.get('sites') auto_disable_sites = blueprint.get('auto_disable_sites', True) if auto_disable_sites: with silent(): enabled_site_links = run( 'ls {}'.format(sites_enabled_path)).split() for link in enabled_site_links: link_name = os.path.splitext(link)[0] # Without extension if link not in sites and link_name not in sites: changed = disable(link, do_reload=False) changes.append(changed) ### Enable sites from settings for site in sites: changed = enable(site, do_reload=False) changes.append(changed) ### Reload nginx if new templates or any site has been enabled/disabled if uploads or any(changes): reload()
def log(repository_path=None, commit='HEAD', count=1, path=None): """ Get log for repository and optional commit range. :param repository_path: Repository path :param commit: Commit to log, ex HEAD..origin :param path: Path or file to log :return: [(<commit>, <comment>), ...] """ if not repository_path: repository_path = debian.pwd() with cd(repository_path), silent(): cmd = 'git log --pretty=oneline {}'.format(commit) if count: cmd += ' -{}'.format(count) if path: cmd += ' -- {}'.format(path) output = run(cmd, pty=False) git_log = output.stdout.strip() git_log = [ col.strip() for row in git_log.split('\n') for col in row.split(' ', 1) if col ] git_log = zip(git_log[::2], git_log[1::2]) return git_log
def configure(): """ Enable/disable configured programs """ with sudo(): # Upload templates uploads = blueprint.upload('init/', '/etc/init/') uploads.extend(blueprint.upload('supervisord.conf', '/etc/')) # Disable previously enabled programs not configured programs-enabled changes = [] programs = blueprint.get('programs') or [] auto_disable = blueprint.get('auto_disable_programs', True) if auto_disable: with silent(): enabled_program_links = run( 'ls {}'.format(programs_enabled_path)).split() for link in enabled_program_links: link_name = os.path.splitext(link)[0] # Without extension if link not in programs and link_name not in programs: changed = disable(link, do_reload=False) changes.append(changed) ### Enable programs from settings for program in programs: changed = enable(program, do_reload=False) changes.append(changed) ### Reload supervisor if new templates or any program has been enabled/disabled if uploads or any(changes): reload()
def update_filters(): changes = [] # Generate desired state as enabled_name => source_name config = blueprint.get('config', {}) filters = { '{}-{}.conf'.format(str(weight).zfill(2), conf): "{}.conf".format(conf) for weight, conf in config.iteritems() } # Get current state with silent(): enabled_filters = run('ls {}'.format(conf_enabled_path)).split() # Disable extra services if blueprint.get('auto_disable_conf', True): for link in set(enabled_filters) - set(filters.keys()): info('Disabling conf: {}', link) changes.append(link) with silent(), sudo(), cd(conf_enabled_path): debian.rm(link) # Enable services for target in set(filters.keys()) - set(enabled_filters): source = os.path.join(conf_available_path, filters[target]) info('Enabling conf: {}', target) changes.append(source) with silent(), sudo(), cd(conf_enabled_path): debian.ln(source, target) return changes
def pip(command, *options, **kwargs): # TODO: change pip log location, per env? per user? # Perhaps we should just remove the log_file argument and let pip put it # where it belongs. info('Running pip {} {}', command, ' '.join(options)) bin = kwargs.pop('bin', 'pip3' if requested_version() >= (3,) else 'pip2') quiet = kwargs.pop('quiet', False) cmd = ('{pip} {command} {options} {verbosity} ' '--exists-action=s --log={log_file} --log-file={log_file}') run(cmd.format(pip=bin, command=command, options=' '.join(options), verbosity='-v' if not quiet else '-q', log_file=pip_log_file), pty=False)
def clone(url, branch=None, repository_path=None, **kwargs): repository = parse_url(url, branch=branch) name = repository['name'] branch = repository['branch'] if not repository_path: repository_path = os.path.join('.', name) if not files.exists(os.path.join(repository_path, '.git')): info('Cloning {}@{} into {}', url, branch, repository_path) cmd = 'git clone -b {branch} {remote} {name}'.format(branch=branch, remote=url, name=name) run(cmd) else: info('Git repository already cloned: {}', name) return repository_path
def manage(cmd=''): """ Run django management command """ if not cmd: cmd = prompt('Enter django management command:') with sudo_project(), cd(python_path()), virtualenv.activate( virtualenv_path()), hide_prefix(): return run('python {manage} {cmd}'.format(cmd=cmd, manage=blueprint.get( 'manage', 'manage.py')))
def install(): with sudo(): info('Downloading kibana') version = blueprint.get('version', '3.1.0') tar_file = 'kibana-{}.tar.gz'.format(version) run('wget -P /tmp/ https://download.elasticsearch.org/kibana/kibana/{f}' .format(f=tar_file)) # Extract and soft link kibana in web root web_root = '/srv/www/' debian.mkdir(web_root, mode=1775, owner='www-data', group='www-data') run('tar xzf /tmp/{f} -C {web_root}'.format(f=tar_file, web_root=web_root)) src_root = os.path.join(web_root, 'kibana-{version}'.format(version=version)) debian.chown(src_root, owner='www-data', group='www-data', recursive=True) debian.ln(src_root, '/srv/www/kibana')
def install_solr(): with sudo(): version = blueprint.get('version') version_tuple = tuple(map(int, version.split('.'))) archive = 'solr-{}.tgz'.format(version) if version_tuple < (4, 1, 0): archive = 'apache-{}'.format(archive) url = 'https://archive.apache.org/dist/lucene/solr/{}/{}'.format( version, archive) with cd('/tmp'): info('Download {} ({})', 'Solr', version) run('wget {}'.format(url)) info('Extracting archive...') with silent(): run('tar xzf {}'.format(archive)) debian.mv(os.path.splitext(archive)[0], solr_home) debian.chmod(solr_home, 755, 'solr', 'solr', recursive=True) debian.rm(archive)
def setup_shared_memory(): """ http://leopard.in.ua/2013/09/05/postgresql-sessting-shared-memory/ """ sysctl_path = '/etc/sysctl.conf' shmmax_configured = files.contains(sysctl_path, 'kernel.shmmax') shmall_configured = files.contains(sysctl_path, 'kernel.shmall') if not any([shmmax_configured, shmall_configured]): page_size = debian.page_size() phys_pages = debian.phys_pages() shmall = phys_pages / 2 shmmax = shmall * page_size shmmax_str = 'kernel.shmmax = {}'.format(shmmax) shmall_str = 'kernel.shmall = {}'.format(shmall) with sudo(): files.append(sysctl_path, shmmax_str, partial=True) files.append(sysctl_path, shmall_str, partial=True) run('sysctl -p') info('Added **{}** to {}', shmmax_str, sysctl_path) info('Added **{}** to {}', shmall_str, sysctl_path)
def current_tag(repository_path=None): """ Get most recent tag :param repository_path: Repository path :return: The most recent tag """ if not repository_path: repository_path = debian.pwd() with cd(repository_path), silent(): output = run('git describe --long --tags --dirty --always', pty=False) # 20141114.1-306-g72354ae-dirty return output.strip().rsplit('-', 2)[0]
def dump(schema=None, ignore_tables=''): """ Dump and download a schema. :param schema: Specific shema to dump and download. :param ignore_tables: Tables to skip, separated by | (pipe) """ if not schema: schemas = blueprint.get('schemas', {}).keys() for i, schema in enumerate(schemas, start=1): print("{i}. {schema}".format(i=i, schema=schema)) valid_indices = '[1-{}]+'.format(len(schemas)) schema_choice = prompt('Select schema to dump:', default='1', validate=valid_indices) schema = schemas[int(schema_choice) - 1] now = datetime.now().strftime('%Y-%m-%d') output_file = '/tmp/{}_{}.backup.gz'.format(schema, now) filename = os.path.basename(output_file) info('Dumping schema {}...', schema) extra_args = [] for table in ignore_tables.split('|'): extra_args.append('--ignore-table={}.{}'.format(schema, table)) dump_cmd = 'mysqldump {} {} | gzip > {}'.format(schema, ' '.join(extra_args), output_file) run('sudo su root -c "{}"'.format(dump_cmd)) info('Downloading dump...') local_file = '~/%s' % filename fabric.contrib.files.get(output_file, local_file) with sudo(), silent(): debian.rm(output_file) info('New smoking hot dump at {}', local_file)
def get_commit(repository_path=None, short=False): """ Get current checked out commit for cloned repository path. :param repository_path: Repository path :param short: Format git commit hash in short (7) format :return: Commit hash """ if not repository_path: repository_path = debian.pwd() with cd(repository_path), silent(): output = run('git rev-parse HEAD') commit = output.strip() if short: commit = commit[:7] return commit
def diff_stat(repository_path=None, commit='HEAD^', path=None): """ Get diff stats for path. :param repository_path: Repository path :param commit: Commit to diff against, ex 12345..67890 :param path: Path or file to diff :return: tuple(num files changed, num insertions, num deletions) """ if not repository_path: repository_path = debian.pwd() with cd(repository_path), silent(): # Example output (note leading space): # 719 files changed, 104452 insertions(+), 29309 deletions(-) # 1 file changed, 1 insertion(+) output = run('git diff --shortstat {} -- {}'.format(commit, path), pty=False) parts = output.strip().split(', ') if output else [] changed, insertions, deletions = 0, 0, 0 for part in parts: match = re.match(r'^\s*(\d+)\s+(.+)$', part) if not match: raise ValueError('no regex match for {!r} in {!r}'.format( part, output)) n, label = match.groups() if label.endswith('(+)'): insertions = int(n) elif label.endswith('(-)'): deletions = int(n) elif label.endswith('changed'): changed = int(n) else: raise ValueError('unexpected git output') return changed, insertions, deletions
def generate_pgtune_conf(role='db'): """ Run pgtune and create pgtune.conf :param role: Which fabric role to place local pgtune.conf template under """ conf_path = postgres_root('postgresql.conf') with sudo(), silent(): output = run('pgtune -T Web -i {}'.format(conf_path)).strip() def parse(c): lines = [l for l in c.splitlines() if '# pgtune' in l] for line in lines: try: comment = line.index('#') line = line[:comment] except ValueError: pass clean = lambda s: s.strip('\n\r\t\'" ') key, _, value = line.partition('=') key, value = map(clean, (key, value)) if key: yield key, value or None tune_conf = dict(parse(output)) tune_conf.update(blueprint.get('pgtune', {})) tune_conf = '\n'.join( (' = '.join(item)) for item in tune_conf.iteritems()) conf_dir = os.path.join(os.path.dirname(env['real_fabfile']), 'templates', role, 'postgres') conf_path = os.path.join(conf_dir, 'pgtune.conf') if not os.path.exists(conf_dir): os.makedirs(conf_dir) with open(conf_path, 'w+') as f: f.write(tune_conf)
def clone(url, branch=None, repository_path=None, **kwargs): """ Clone repository and branch. :param url: Git url to clone :param branch: Branch to checkout :param repository_path: Destination :param kwargs: Not used but here for easier kwarg passing :return: (destination, got_cloned bool) """ repository = parse_url(url, branch=branch) name = repository['name'] branch = repository['branch'] cloned = False if not repository_path: repository_path = os.path.join('.', name) if not files.exists(os.path.join(repository_path, '.git')): info('Cloning {}@{} into {}', url, branch, repository_path) with silent('warnings'): cmd = 'git clone -b {branch} {remote} {name}'.format(branch=branch, remote=url, name=name) output = run(cmd) if output.return_code != 0: warn( 'Failed to clone repository "{}", probably permission denied!'. format(name)) cloned = None else: cloned = True else: info('Git repository already cloned: {}', name) return repository_path, cloned
def reset(branch, repository_path=None, **kwargs): """ Fetch, reset, clean and checkout repository branch. :return: commit """ commit = None if not repository_path: repository_path = debian.pwd() with cd(repository_path): name = os.path.basename(repository_path) info('Resetting git repository: {}@{}', name, branch) with silent('warnings'): commands = [ 'git fetch origin', # Fetch branches and tags 'git reset --hard HEAD', # Make hard reset to HEAD 'git clean -fdx', # Remove untracked files pyc, xxx~ etc 'git checkout HEAD', # Checkout HEAD 'git reset refs/remotes/origin/{} --hard'.format( branch) # Reset to branch ] output = run(' && '.join(commands)) if output.return_code != 0: warn( 'Failed to reset repository "{}", probably permission denied!'. format(name)) else: output = output.split(os.linesep)[-1][len('HEAD is now at '):] commit = output.split()[0] info('HEAD is now at: {}', output) return commit