def setup_slaves(): print "Setting up slave %s" % env.host with settings(warn_only=True): sudo('service postgresql stop') host = env.roledefs['master'][0] if host not in env.addresses: raise KeyError("could not find master IP address") print host master_ip = env.addresses[host]['eth0'] append(pg_conf, 'wal_level = hot_standby', use_sudo = True) append(pg_conf, 'max_wal_senders = 5', use_sudo = True) append(pg_conf, 'wal_keep_segments = 32', use_sudo = True) append(pg_conf, 'hot_standby = on', use_sudo = True) with cd(os.path.dirname(data_path)): with settings(warn_only=True): sudo('mv main main.%d' % time.time() ) print "Starting backup from master %s" % master_ip sudo('pg_basebackup -P -x -h %s -U project -D main' % master_ip) sudo('rm -f main/backup_label') sudo('chown -R postgres:postgres main') print "Done." append(recovery_conf, "standby_mode = 'on'", use_sudo = True) append(recovery_conf, "primary_conninfo = 'host=%s user=project'" % master_ip, use_sudo = True) sudo('service postgresql start')
def setup_database(): """Create the application database and associated user with a temporary password""" with hide('output', 'running'): with settings(warn_only=True): psql("CREATE USER project UNENCRYPTED PASSWORD 'project';") psql("ALTER ROLE project REPLICATION LOGIN;") with settings(warn_only=True): psql("CREATE DATABASE project ENCODING 'UTF-8';") psql("GRANT ALL ON DATABASE project TO project;")
def _setup_database(): quer_criar_template = False quer_criar_user = False quer_criar_db = False puts_blue("Setup do Bano de Dados") with cd('%(django_user_home)s' % env): if 'postgis' in env.django_settings.DATABASES['default']['ENGINE']: # template postgis existe with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('psql -tAc "SELECT \'template_postgis_encontrado\';" template_postgis', user='******') if not 'template_postgis_encontrado' in res: # quer criar o template postgis? quer_criar_template = console.confirm(u'quer criar o template postgis?') if quer_criar_template: script_path = '%s/scripts/create_template_postgis-debian.sh' % base_path remote = '/opt/django/create_template_postgis-debian.sh' if not files.exists(remote, use_sudo=True, verbose=False): put( open(script_path), remote, use_sudo=True ) sudo('/bin/bash %s' % remote, user='******') # cria template postgis rolname = env.django_settings.DATABASES['default']['USER'] # user existe with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo( 'psql -tAc \ "SELECT \'user_encontrado\' FROM ( \ SELECT count(*) FROM pg_roles WHERE rolname=\'%s\'\ ) AS COUNT WHERE COUNT.count >= 1;"' % rolname, user='******' ) if not 'user_encontrado' in res: quer_criar_user = console.confirm(u'quer criar o user %s?' % rolname) if quer_criar_user: _create_postgre_user() database = env.django_settings.DATABASES['default']['NAME'] # user existe with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('psql -tAc "SELECT \'BD_encontrado\';" %s' % database, user='******') if not 'BD_encontrado' in res: quer_criar_db = console.confirm(u'quer criar o Database %s?' % database) if quer_criar_db: _create_postgre_database() _atualiza_projs(database)
def git_pull(): with cd(env.code_root): with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('git checkout -b %(branch)s' % env, user=env.django_user) if 'failed' in res: sudo('git checkout %(branch)s' % env, user=env.django_user) sudo('git pull origin %(branch)s' % env, user=env.django_user)
def _git_clone(): with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): with cd(env.code_root): res = sudo('git pull origin %(branch)s' % env, user=env.django_user) logging.error('code root not exists: %s' % res) if 'No such file or directory' in res: sudo('git clone %(repository)s %(code_root)s' % env, user=env.django_user)
def get_file_task(remote_path, local_path, user='******', password='******'): # Task to grab file from host and return result env.user = user env.password = password env.parallel = True with settings(warn_only=True): return get(remote_path, local_path)
def bower_install(): print yellow('\nInstalling Bower dependencies') with shell_env(HOME=env.deploy_user_home): with settings(password=env.sd_password): with cd('%s/%s' % (env.repos_dir, REPOSITORY[1])): sudo('npm install -d', user=env.deploy_user) sudo('bower update', user=env.deploy_user)
def pylint_file(filenames, **kwargs): """ apply pylint to the file specified, return the filename, score """ command = "pylint " if 'rcfile' in kwargs and kwargs['rcfile'] is not None: command += " --rcfile={0} ".format(kwargs['rcfile']) command = command + ' '.join(filenames) # we use fabric to run the pylint command, hiding the normal fab # output and warnings with hide('output', 'running', 'warnings'), settings(warn_only=True): result = local(command, capture=True) score = 0.0 # parse the output from pylint for the score for line in result.split('\n'): if re.match("E....:.", line): LOGGER.info(line) if "Your code has been rated at" in line: score = re.findall("\d+.\d\d", line)[0] score = float(score) return filenames, score
def _test(option, path, use_sudo): if isinstance(path, Path): path = str(path) with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): return run_or_sudo(f'test {option} {quote(path)}', use_sudo).succeeded
def _supervisor_restart(): with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('%(supervisorctl)s restart %(supervisor_program_name)s' % env) if 'ERROR' in res: print red_bg("%s NOT STARTED!" % env.supervisor_program_name) else: print green_bg("%s correctly started!" % env.supervisor_program_name)
def finish_configuring_slave(): parameter_default_values() with settings(host_string=env.pgslave_user_host): # upload repmgr.conf on slave server repmgr_context = dict(cluster_name=env.cluster_name, node_number=env.slave_node_number, sync_user=env.sync_user, sync_db=env.sync_db, sync_pass=env.sync_pass, ssh_port=env.master_ssh_port, ) repmgr_conf_file = 'conf/repmgr/repmgr.conf' if not isfile(repmgr_conf_file): repmgr_conf_file = '%s/%s' % (pg_fabrep_path, repmgr_conf_file) upload_template(repmgr_conf_file, env.master_pgdata_path, context=repmgr_context, backup=False) slave_postgresql_conf = "%spostgresql.conf" % env.slave_pgdata_path slave_postgresql_conf_bck = "%spostgresql.conf.bck" % env.slave_pgdata_path sudo('mv %s %s' % (slave_postgresql_conf, slave_postgresql_conf_bck)) sudo("sed '/hot_standby =/c hot_standby = on' %s > %s" % \ (slave_postgresql_conf_bck, slave_postgresql_conf)) sudo("mkdir -p %s" % env.slave_pgconf_path) sudo("cp %spg_hba.conf %s" % (env.slave_pgdata_path, env.slave_pgconf_path)) sudo("cp %spg_ident.conf %s" % (env.slave_pgdata_path, env.slave_pgconf_path)) sudo("cp %spostgresql.conf %s" % (env.slave_pgdata_path, env.slave_pgconf_path)) run("sudo -u postgres pg_ctl -D /var/lib/postgresql/%(postgres_version)s/%(cluster_name)s/ start" % env)
def restart_nginx(): print yellow('\nRestarting Gunicorn') with settings(password=env.sd_password): sudo('service nginx restart') #fab development:user=<conejo|rcanepa|pillin> update_repos bower_install grunt_build restart_nginx
def _standby_clone(): """ With "node1" server running, we want to use the clone standby command in repmgr to copy over the entire PostgreSQL database cluster onto the "node2" server. """ # manualy: # $ mkdir -p /var/lib/postgresql/9.1/testscluster/ # $ rsync -avz --rsh='ssh -p2222' [email protected]:/var/lib/postgresql/9.1/testscluster/ /var/lib/postgresql/9.1/testscluster/ with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): puts(green('Start cloning the master')) repmgr_clone_command = 'repmgr -D %(slave_pgdata_path)s -d %(sync_db)s -p %(cluster_port)s -U %(sync_user)s -R postgres --verbose standby clone %(pgmaster_ip)s' % env puts(green(repmgr_clone_command)) puts("-" * 40) res = sudo(repmgr_clone_command, user='******') if 'Can not connect to the remote host' in res or 'Connection to database failed' in res: puts("-" * 40) puts(green(repmgr_clone_command)) puts("-" * 40) puts("Master server is %s reachable." % red("NOT")) puts("%s you can try to CLONE the slave manually [%s]:" % (green("BUT"), red("at your own risk"))) puts("On the slave server:") puts("$ sudo -u postgres rsync -avz --rsh='ssh -p%(master_ssh_port)s' postgres@%(pgmaster_ip)s:%(master_pgdata_path)s %(slave_pgdata_path)s --exclude=pg_xlog* --exclude=pg_control --exclude=*.pid" % env) puts("Here:") puts("$ fab <cluster_task_name> finish_configuring_slave") abort("STOP...")
def _test_nginx_conf(): ''' Testa configurações do nginx ''' with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('nginx -t -c /etc/nginx/nginx.conf') if 'test failed' in res: abort(red('NGINX configuration test failed! Please review your parameters.'))
def test_nginx_conf(): with settings( hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('nginx -t -c /etc/nginx/nginx.conf') if 'test failed' in res: abort(red_bg( 'NGINX configuration test failed!' ' Please review your parameters.'))
def _create_django_user(): with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('useradd -d %(django_user_home)s -m -r %(django_user)s' % env) if 'already exists' in res: puts('User \'%(django_user)s\' already exists, will not be changed.' % env) return # set password sudo('passwd %(django_user)s' % env)
def _setup_permissions(): with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('usermod -G nginx %s' % env.django_user) if 'does not exist' in res: res = sudo('usermod -G www-data %s' % env.django_user) sudo('chown -R %(django_user)s:www-data %(django_user_home)s' % env) else: sudo('chown -R %(django_user)s:nginx %(django_user_home)s' % env) sudo('chmod -R g+x %(django_user_home)s' % env)
def _git_clone(): ''' Faz um clone de um repositório ''' puts_blue("== CLONE do repositório ...", 1, bg=107) with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): with cd('%(django_user_home)s' % env): sudo('git clone %(repository)s %(code_root)s' % env, user=env.django_user) with cd(env.code_root): sudo('git fetch' % env, user=env.django_user) sudo('git checkout %(branch)s' % env, user=env.django_user)
def update_repos(): print yellow('\nUpdating project GitHub repos') with shell_env(HOME=env.deploy_user_home): with settings(password=env.sd_password): if not exists(env.repos_dir): sudo('mkdir %s' % env.repos_dir) sudo('chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.repos_dir)) with cd(env.repos_dir): if not exists('%s/%s' % (env.repos_dir, REPOSITORY[1])): sudo('git clone %s %s' % (REPOSITORY[0], REPOSITORY[1]), user=env.deploy_user) with cd(REPOSITORY[1]): sudo('git checkout %s' % REPOSITORY[2], user=env.deploy_user) sudo('git pull origin %s' % REPOSITORY[2], user=env.deploy_user)
def iperf_server_pid(command, user="******", password="******"): """ Task to discover iperf (running in server mode) PID :param command: command to obtain iperf PID :param user: username of remote user :param password: password of remote user """ env.user = user env.password = password env.parallel = True with settings(warn_only=True): return run(command, pty=False, combine_stderr=True)
def iperf_server_pid(command, user='******', password='******'): """ Task to discover iperf (running in server mode) PID :param command: command to obtain iperf PID :param user: username of remote user :param password: password of remote user """ env.user = user env.password = password env.parallel = True with settings(warn_only=True): return run(command, pty=False, combine_stderr=True)
def iperf_server_command(command, user='******', password='******'): """ Run given iperf command on specific host :param command: iperf command with arguments :param user: username of remote user :param password: password of remote user """ env.user = user env.password = password env.parallel = True with settings(warn_only=True): return run(command, pty=False, combine_stderr=True)
def iperf_server_command(command, user="******", password="******"): """ Run given iperf command on specific host :param command: iperf command with arguments :param user: username of remote user :param password: password of remote user """ env.user = user env.password = password env.parallel = True with settings(warn_only=True): return run(command, pty=False, combine_stderr=True)
def _create_django_user(): ''' Função que cria e verifica se o usuário Django Existe ''' puts_blue(u" Cria usuário __django__ ...", 1, bg=107) puts_blue("== Verifica / Cria usuário 'django' ...", 1, bg=107) with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('useradd -d %(django_user_home)s -m -r %(django_user)s -s /bin/bash' % env) if 'already exists' in res: puts('User \'%(django_user)s\' already exists, will not be changed.' % env) return # set password sudo('passwd %(django_user)s' % env)
def dc_restart(service_name='', cmd=''): ''' Restarts a service, executing optional command before restart Execute as: fab using_project:<project-name> dc_restart:<service_name>[,<cmd>] ''' with settings(warn_only=True): dc_stop(service_name=service_name) dc_rm(service_name=service_name) rm_dangling_volumes() if cmd is not None and cmd != '': dc_run(options='', service_name=service_name, cmd=cmd) dc_up(options='-d', service_name=service_name) dc_logs(options='-f', service_name=service_name)
def _atualiza_projs(database_name): if os.path.exists('projs.json'): j = json.load(open('projs.json')) query_file_path = '/tmp/%(project)s_query.sql' % env fd = StringIO() fd.name = '%(project)s_query.sql' % env sudo('touch %s' % query_file_path) for key, value in j.items(): fd.write(value['postgis'] + '\n') srid_key = '<%s>' % key with settings(warn_only=True): res = sudo('grep -H "%s" /usr/share/proj/epsg ' % srid_key) if not srid_key in res: string_epsg = "%s %s" % (srid_key, value['proj4text']) sudo("echo '%s' >> %s" % (string_epsg, '/usr/share/proj/epsg')) put(fd, query_file_path, use_sudo=True) sudo("psql %s < %s" % (database_name, query_file_path), user='******')
def git_seed(commit=None, ignore_untracked_files=False): """ seed a git repository (and create if necessary) [remote] """ # check if the local repository is dirty dirty_working_copy = git_is_dirty(ignore_untracked_files) if dirty_working_copy: abort( 'Working copy is dirty. This check can be overridden by\n' ' try adding allow_dirty to your call.') # check if the remote repository exists and create it if necessary git_init() # use specified commit or HEAD commit = commit or git_head_rev() # finish execution if remote repository has commit already if git_exists(env.code_root, commit): puts(green('Commit ') + commit + green(' exists already')) return # push the commit to the remote repository # # (note that pushing to the master branch will not change the contents # of the working directory) puts(green('Pushing commit ') + commit) with settings(warn_only=True): force = (env.git_force_push) and '-f' or '' push = local( 'git push git+ssh://%s@%s:%s%s %s:refs/heads/master %s' % ( env.user, env.host, env.port, env.code_root, commit, force)) if push.failed: abort( '%s is a non-fast-forward\n' 'push. The seed will abort so you don\'t lose information.' ' If you are doing this\nintentionally try user' ' force_push and add it to your call.' % commit)
def _supervisor_restart(): ''' Restarta o supervisor ''' puts_blue(" __restart__ da aplicação via supervisor ...", 1, bg=107) with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('%(supervisorctl)s stop %(supervisor_program_name)s' % env) puts_blue("-> Parando App:") puts_blue(" Mensagem do supervisor: %s " % (res)) #start app res = sudo('%(supervisorctl)s start %(supervisor_program_name)s' % env) puts_blue("-> Reiniciando a App:") if 'ERROR' in res: puts_red("-> Erro ao reiniciar a app : %s" % env.supervisor_program_name) else: puts_green("==========================================") puts_green(" %s iniciado com sucesso " % env.supervisor_program_name) puts_green("==========================================") puts_blue(" Mensagem do supervisor: %s " % (res))
def pyflakes_file(filenames, verbose=False): """ _pyflakes_file_ Appyly pyflakes to file specified, return (filenames, score) """ command = 'pyflakes ' + ' '.join(filenames) # we use fabric to run the pyflakes command, hiding the normal fab # output and warnings with hide('output', 'running', 'warnings'), settings(warn_only=True): result = local(command, capture=True) flakes = 0 data = [x for x in result.split('\n') if x.strip()] if len(data) != 0: #We have at least one flake, find the rest flakes = count_flakes(data, verbose) + 1 else: flakes = 0 return filenames, flakes
def _setup_database(): with settings(warn_only=True): postgres.server() postgres.user(env.psql_user, password=env.psql_password) postgres.database(env.psql_db, owner=env.psql_user)
def setup(): parameter_default_values() # test configuration start if not test_configuration(): if not console.confirm("Configuration test %s! Do you want to continue?" % red_bg('failed'), default=False): abort("Aborting at user request.") # test configuration end if env.ask_confirmation: if not console.confirm("Are you sure you want to setup %s cluster?" % green(env.cluster_name.upper()), default=False): abort("Aborting at user request.") puts(green_bg('Start setup...')) start_time = datetime.now() # Start configuring master server with settings(host_string=env.pgmaster_user_host): print "%s configuring master server!" % green_bg("Start") _verify_sudo() _common_setup() with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo("pg_createcluster --start %(postgres_version)s %(cluster_name)s -p %(cluster_port)s" % env) if 'already exists' in res: puts(green("Cluster '%(cluster_name)s' already exists, will not be changed." % env)) if 'Error: port %s is already used' % env.cluster_port in res: puts(red(res)) with cd(env.master_pgdata_path): with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): run('''sudo -u postgres psql -p %(cluster_port)s -c "CREATE USER %(sync_user)s SUPERUSER ENCRYPTED PASSWORD '%(sync_pass)s';"''' % env, shell=False) run('''sudo -u postgres createdb -p %(cluster_port)s --owner %(sync_user)s --template template0 --encoding=UTF8 --lc-ctype=en_US.UTF-8 --lc-collate=en_US.UTF-8 %(sync_db)s''' % env, shell=False) run('''sudo -u postgres psql -p %(cluster_port)s -c "GRANT CREATE, CONNECT ON DATABASE %(sync_db)s TO %(sync_user)s WITH GRANT OPTION;"''' % env, shell=False) postgres_conf_file = 'conf/postgres/postgresql.conf' if not isfile(postgres_conf_file): postgres_conf_file = '%s/%s' % (pg_fabrep_path, postgres_conf_file) upload_template(postgres_conf_file, env.master_pgconf_path, context=env, backup=False) if env.verbose: puts("Uploaded postgresql.conf from %s to %s" % (green(abspath(postgres_conf_file)), green(env.master_pgconf_path))) # start preparing pg_hba.conf file replication_hba = '%sreplication_hba.conf' % env.master_pgconf_path pg_hba = '%spg_hba.conf' % env.master_pgconf_path pg_hba_bck = '%spg_hba.conf.bck' % env.master_pgconf_path pg_hba_file = 'conf/postgres/pg_hba.conf' if not isfile(pg_hba_file): pg_hba_file = '%s/%s' % (pg_fabrep_path, pg_hba_file) upload_template(pg_hba_file, replication_hba, context=env, backup=False) sudo('mv %s %s' % (pg_hba, pg_hba_bck)) sudo("sed -n -e '/START REPLICATION RULES/,/END REPLICATION RULES/!p' %s > %s" % (pg_hba_bck, pg_hba)) sudo("cat %s >> %s" % (replication_hba, pg_hba)) sudo("rm %s" % replication_hba) # upload repmgr.conf on master server repmgr_context = dict(cluster_name=env.cluster_name, node_number=env.master_node_number, sync_user=env.sync_user, sync_db=env.sync_db, sync_pass=env.sync_pass, ssh_port=22, # we don't need here ) repmgr_conf_file = 'conf/repmgr/repmgr.conf' if not isfile(repmgr_conf_file): repmgr_conf_file = '%s/%s' % (pg_fabrep_path, repmgr_conf_file) upload_template(repmgr_conf_file, env.master_pgdata_path, context=repmgr_context, backup=False) sudo("""echo -e "# Added by pg_fabrep\nexport PGDATA='/var/lib/postgresql/%(postgres_version)s/%(cluster_name)s'\nexport PGPORT=%(cluster_port)s">/var/lib/postgresql/.bash_profile""" % env) sudo("pg_ctlcluster %(postgres_version)s %(cluster_name)s restart" % env) # Start configuring the slave with settings(host_string=env.pgslave_user_host): print "%s configuring slave server!" % green_bg("Start") _verify_sudo() _common_setup() sudo('rm -rf %s' % env.slave_pgdata_path) sudo('mkdir -p %s' % env.slave_pgdata_path) sudo('chown postgres:postgres %s' % env.slave_pgdata_path) _standby_clone() finish_configuring_slave() # you can start it manually # TODO: ... # # repmgr register master # with settings(host_string=env.master_host_local_ip): # sudo('repmgr -f %srepmgr.conf --verbose master register' % \ # env.master_pgdata_path, user='******') # # repmgr register slave # with settings(host_string=env.slave_host): # sudo('repmgr -f %srepmgr.conf --verbose standby register' % \ # env.slave_pgdata_path, user='******') end_time = datetime.now() finish_message = '[%s] Correctly finished in %i seconds' % \ (green(end_time.strftime('%H:%M:%S')), (end_time - start_time).seconds) puts(finish_message)
def host_type(): with settings(warn_only=True): run('uname -a')
def run_command_with_remote_prompts(ip, username, password, command, prompts): set_fabric_environment(ip, username, password) operations.env.prompts = prompts with operations.settings(abort_on_prompts=False): return operations.run(command)
def supervisor_restart(): with settings( hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True): res = sudo('%(supervisorctl)s restart all' % env) if 'ERROR' in res: print red_bg("%s NOT STARTED!" % env.supervisor_program_name)
def clean(): """Clean the packaged zip""" with settings(warn_only=True): local('rm %s' % get_zip_file())
def grunt_build(): print yellow('\nBuilding project static files') with shell_env(HOME=env.deploy_user_home): with settings(password=env.sd_password): with cd('%s/%s' % (env.repos_dir, REPOSITORY[1])): sudo('grunt build')