def deploy_services(site=None, dryrun=0):
"""
Collects the configurations for all registered services and writes
the appropriate supervisord.conf file.
"""
dryrun = int(dryrun)
render_paths()
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print site
for cb in env._supervisor_create_service_callbacks:
ret = cb()
if isinstance(ret, basestring):
env.supervisor_services.append(ret)
# else:
# print 'invalid'
env.supervisor_services_rendered = "\n".join(env.supervisor_services)
# print env.supervisor_services_rendered
fn = common.render_to_file("supervisor_daemon.template.config")
if dryrun:
print open(fn).read()
else:
put(local_path=fn, remote_path=env.supervisor_config_path, use_sudo=True)
def install_apt(fn=None, update=0):
"""
Installs system packages listed in apt-requirements.txt.
"""
print 'Installing apt requirements...'
assert env[ROLE]
env.apt_fqfn = fn or find_template(env.apt_fn)
if not env.apt_fqfn:
return
assert os.path.isfile(env.apt_fqfn)
fd, tmp_fn = tempfile.mkstemp()
lines = [
_ for _ in open(env.apt_fqfn).readlines()
if _.strip() and not _.strip().startswith('#')
]
fout = open(tmp_fn, 'w')
fout.write('\n'.join(lines))
fout.close()
env.apt_fqfn = tmp_fn
if not env.is_local:
put(local_path=tmp_fn)
env.apt_fqfn = env.put_remote_path
if int(update):
sudo('apt-get update -y')
sudo('apt-get install -y `cat "%(apt_fqfn)s" | tr "\\n" " "`' % env)
def passwordless(username=None, pubkey=None):
"""
Configures the user to use an SSL key without a password.
Assumes you've run generate_keys() first.
"""
env.user_username = username or env.user
env.user_pubkey = pubkey or env.key_filename
assert os.path.isfile(env.user_pubkey), \
'Public key file "%s" does not exist.' % (env.user_pubkey,)
first = os.path.splitext(env.user_pubkey)[0]
env.user_pubkey = first+'.pub'
env.user_pemkey = first+'.pem'
env.user_home = env.user_home_template % env
# Upload the SSH key.
put(local_path=env.user_pubkey)
sudo('mkdir -p %(user_home)s/.ssh' % env)
sudo('cat %(put_remote_path)s >> %(user_home)s/.ssh/authorized_keys' % env)
sudo('rm -f %(put_remote_path)s' % env)
# Disable password.
sudo('cp /etc/sudoers %(user_tmp_sudoers_fn)s' % env)
sudo('echo "%(user_username)s ALL=(ALL) NOPASSWD: ALL" >> %(user_tmp_sudoers_fn)s' % env)
sudo('sudo EDITOR="cp %(user_tmp_sudoers_fn)s" visudo' % env)
sudo('service ssh reload')
print 'You should now be able to login with:'
print '\tssh -i %(user_pemkey)s %(user_username)s@%(host_string)s' % env
def passwordless(username=None, pubkey=None):
"""
Configures the user to use an SSL key without a password.
Assumes you've run generate_keys() first.
"""
env.user_username = username or env.user
env.user_pubkey = pubkey or env.key_filename
assert os.path.isfile(env.user_pubkey), \
'Public key file "%s" does not exist.' % (env.user_pubkey,)
first = os.path.splitext(env.user_pubkey)[0]
env.user_pubkey = first + '.pub'
env.user_pemkey = first + '.pem'
env.user_home = env.user_home_template % env
# Upload the SSH key.
put(local_path=env.user_pubkey)
sudo('mkdir -p %(user_home)s/.ssh' % env)
sudo('cat %(put_remote_path)s >> %(user_home)s/.ssh/authorized_keys' % env)
sudo('rm -f %(put_remote_path)s' % env)
# Disable password.
sudo('cp /etc/sudoers %(user_tmp_sudoers_fn)s' % env)
sudo(
'echo "%(user_username)s ALL=(ALL) NOPASSWD: ALL" >> %(user_tmp_sudoers_fn)s'
% env)
sudo('sudo EDITOR="cp %(user_tmp_sudoers_fn)s" visudo' % env)
sudo('service ssh reload')
print 'You should now be able to login with:'
print '\tssh -i %(user_pemkey)s %(user_username)s@%(host_string)s' % env
def install_apt(fn=None, update=0):
"""
Installs system packages listed in apt-requirements.txt.
"""
print 'Installing apt requirements...'
assert env[ROLE]
env.apt_fqfn = fn or find_template(env.apt_fn)
if not env.apt_fqfn:
return
assert os.path.isfile(env.apt_fqfn)
fd, tmp_fn = tempfile.mkstemp()
lines = [
_ for _ in open(env.apt_fqfn).readlines()
if _.strip() and not _.strip().startswith('#')
]
fout = open(tmp_fn, 'w')
fout.write('\n'.join(lines))
fout.close()
env.apt_fqfn = tmp_fn
if not env.is_local:
put(local_path=tmp_fn)
env.apt_fqfn = env.put_remote_path
if int(update):
sudo('apt-get update -y')
sudo('apt-get install -y `cat "%(apt_fqfn)s" | tr "\\n" " "`' % env)
def configure_modevasive():
env.apache_mods_enabled.append('mod-evasive')
# Write modsecurity.conf.
fn = common.render_to_file('apache_modevasive.template.conf')
put(local_path=fn, remote_path='/etc/apache2/mods-available/mod-evasive.conf', use_sudo=True)
def deploy_services(site=None, dryrun=0):
"""
Collects the configurations for all registered services and writes
the appropriate supervisord.conf file.
"""
dryrun = int(dryrun)
render_paths()
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print site
for cb in env._supervisor_create_service_callbacks:
ret = cb()
if isinstance(ret, basestring):
env.supervisor_services.append(ret)
# else:
# print 'invalid'
env.supervisor_services_rendered = '\n'.join(env.supervisor_services)
#print env.supervisor_services_rendered
fn = common.render_to_file('supervisor_daemon.template.config')
if dryrun:
print open(fn).read()
else:
put(local_path=fn,
remote_path=env.supervisor_config_path,
use_sudo=True)
def install_sql(name=None, site=None):
"""
Installs all custom SQL.
"""
#_settings = get_settings(site=site, role=env.ROLE)
set_db(name=name, site=site)
paths = glob.glob(env.db_install_sql_path_template % env)
#paths = glob.glob('%(src_dir)s/%(app_name)s/*/sql/*' % env)
def cmp_paths(d0, d1):
if d0[1] and d0[1] in d1[2]:
return -1
if d1[1] and d1[1] in d0[2]:
return +1
return cmp(d0[0], d1[0])
def get_paths(t):
"""
Returns SQL file paths in an execution order that respect dependencies.
"""
data = [] # [(path, view_name, content)]
for path in paths:
#print path
parts = path.split('.')
if len(parts) == 3 and parts[1] != t:
continue
content = open(path, 'r').read()
matches = re.findall('[\s\t]+VIEW[\s\t]+([a-zA-Z0-9_]+)',
content,
flags=re.IGNORECASE)
#assert matches, 'Unable to find view name: %s' % (p,)
view_name = ''
if matches:
view_name = matches[0]
data.append((path, view_name, content))
for d in sorted(data, cmp=cmp_paths):
yield d[0]
if 'postgres' in env.db_engine:
#print 'postgres'
for path in get_paths('postgresql'):
put(local_path=path)
#cmd = ("mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' %(db_name)s < %(put_remote_path)s") % env
cmd = (
"psql --host=%(db_host)s --user=%(db_user)s -d %(db_name)s -f %(put_remote_path)s"
) % env
#print cmd
sudo(cmd)
elif 'mysql' in env.db_engine:
for path in get_paths('mysql'):
put(local_path=path)
cmd = (
"mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' %(db_name)s < %(put_remote_path)s"
) % env
#print cmd
sudo(cmd)
else:
raise NotImplementedError
def configure_modevasive():
env.apache_mods_enabled.append('mod-evasive')
# Write modsecurity.conf.
fn = common.render_to_file('apache_modevasive.template.conf')
put(local_path=fn,
remote_path='/etc/apache2/mods-available/mod-evasive.conf',
use_sudo=True)
def static():
"""
Configures the server to use a static IP.
"""
fn = render_to_file('ip_interfaces_static.template')
put(local_path=fn, remote_path=env.ip_interfaces_fn, use_sudo=True)
#sudo('ifdown %(ip_interface)s' % env)
#sudo('ifup %(ip_interface)s' % env)
sudo(env.ip_network_restart_command % env)
def configure():
"""
Installs supervisor configuration and daemon.
"""
render_paths()
fn = common.render_to_file('supervisor_daemon.template.init')
put(local_path=fn, remote_path=env.supervisor_daemon_path, use_sudo=True)
sudo('chmod +x %(supervisor_daemon_path)s' % env)
sudo('update-rc.d supervisord defaults' % env)
def configure():
"""
Installs supervisor configuration and daemon.
"""
render_paths()
fn = common.render_to_file("supervisor_daemon.template.init")
put(local_path=fn, remote_path=env.supervisor_daemon_path, use_sudo=True)
sudo("chmod +x %(supervisor_daemon_path)s" % env)
sudo("update-rc.d supervisord defaults" % env)
def install_sql(name=None, site=None):
"""
Installs all custom SQL.
"""
# _settings = get_settings(site=site, role=env.ROLE)
set_db(name=name, site=site)
paths = glob.glob(env.db_install_sql_path_template % env)
# paths = glob.glob('%(src_dir)s/%(app_name)s/*/sql/*' % env)
def cmp_paths(d0, d1):
if d0[1] and d0[1] in d1[2]:
return -1
if d1[1] and d1[1] in d0[2]:
return +1
return cmp(d0[0], d1[0])
def get_paths(t):
"""
Returns SQL file paths in an execution order that respect dependencies.
"""
data = [] # [(path, view_name, content)]
for path in paths:
# print path
parts = path.split(".")
if len(parts) == 3 and parts[1] != t:
continue
content = open(path, "r").read()
matches = re.findall("[\s\t]+VIEW[\s\t]+([a-zA-Z0-9_]+)", content, flags=re.IGNORECASE)
# assert matches, 'Unable to find view name: %s' % (p,)
view_name = ""
if matches:
view_name = matches[0]
data.append((path, view_name, content))
for d in sorted(data, cmp=cmp_paths):
yield d[0]
if "postgres" in env.db_engine:
# print 'postgres'
for path in get_paths("postgresql"):
put(local_path=path)
# cmd = ("mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' %(db_name)s < %(put_remote_path)s") % env
cmd = ("psql --host=%(db_host)s --user=%(db_user)s -d %(db_name)s -f %(put_remote_path)s") % env
# print cmd
sudo(cmd)
elif "mysql" in env.db_engine:
for path in get_paths("mysql"):
put(local_path=path)
cmd = ("mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' %(db_name)s < %(put_remote_path)s") % env
# print cmd
sudo(cmd)
else:
raise NotImplementedError
def configure(full=1, site=ALL, delete_old=0):
"""
Configures Apache to host one or more websites.
"""
from burlap import service
print 'Configuring Apache...'
apache_specifics = set_apache_specifics()
if int(delete_old):
# Delete all existing enabled and available sites.
sudo('rm -f %(apache_sites_available)s/*' % env)
sudo('rm -f %(apache_sites_enabled)s/*' % env)
for site, site_data in common.iter_sites(site=site, setter=set_apache_site_specifics):
#print '-'*80
print site
#continue
print 'env.apache_ssl_domain:',env.apache_ssl_domain
print 'env.apache_ssl_domain_template:',env.apache_ssl_domain_template
fn = common.render_to_file('django.template.wsgi')
put(local_path=fn, remote_path=env.apache_django_wsgi, use_sudo=True)
if env.apache_ssl:
env.apache_ssl_certificates = list(iter_certificates())
fn = common.render_to_file('apache_site.template.conf')
env.apache_site_conf = site+'.conf'
env.apache_site_conf_fqfn = os.path.join(env.apache_sites_available, env.apache_site_conf)
put(local_path=fn, remote_path=env.apache_site_conf_fqfn, use_sudo=True)
sudo('a2ensite %(apache_site_conf)s' % env)
#return
if service.is_selected(APACHE2_MODEVASIVE):
configure_modevasive()
if service.is_selected(APACHE2_MODSECURITY):
configure_modsecurity()
for mod_enabled in env.apache_mods_enabled:
env.apache_mod_enabled = mod_enabled
sudo('a2enmod %(apache_mod_enabled)s' % env)
if int(full):
# Write master Apache configuration file.
fn = common.render_to_file('apache_httpd.template.conf')
put(local_path=fn, remote_path=env.apache_conf, use_sudo=True)
# Write Apache listening ports configuration.
fn = common.render_to_file('apache_ports.template.conf')
put(local_path=fn, remote_path=env.apache_ports, use_sudo=True)
#sudo('mkdir -p %(apache_app_log_dir)s' % env)
#sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_app_log_dir)s' % env)
# sudo('mkdir -p %(apache_log_dir)s' % env)
# sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_log_dir)s' % env)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_root)s' % env)
def install_yum(fn=None, update=0):
"""
Installs system packages listed in yum-requirements.txt.
"""
print 'Installing yum requirements...'
assert env[ROLE]
env.yum_fn = fn or find_template(env.yum_fn)
assert os.path.isfile(env.yum_fn)
update = int(update)
if env.is_local:
put(local_path=env.yum_fn)
env.yum_fn = env.put_remote_fn
if update:
sudo('yum update --assumeyes')
sudo('yum install --assumeyes $(cat %(yum_fn)s)' % env)
def install_yum(fn=None, update=0):
"""
Installs system packages listed in yum-requirements.txt.
"""
print 'Installing yum requirements...'
assert env[ROLE]
env.yum_fn = fn or find_template(env.yum_fn)
assert os.path.isfile(env.yum_fn)
update = int(update)
if env.is_local:
put(local_path=env.yum_fn)
env.yum_fn = env.put_remote_fn
if update:
sudo('yum update --assumeyes')
sudo('yum install --assumeyes $(cat %(yum_fn)s)' % env)
def configure():
"""
Configures rules for IPTables.
"""
if env.iptables_enabled:
fn = common.render_to_file('iptables.template.rules')
put(local_path=fn)
cmd = 'iptables-restore < %(put_remote_path)s; iptables-save > /etc/iptables.up.rules' % env
sudo(cmd)
enable()
restart()
else:
disable()
stop()
def install_ssl(site=ALL, dryrun=0):
apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(site=site, setter=set_apache_site_specifics):
print site
set_apache_site_specifics(site)
sudo('mkdir -p %(apache_ssl_dir)s' % env)
if env.apache_ssl:
for cert_type, local_cert_file, remote_cert_file in iter_certificates():
print '='*80
print 'Installing certificate %s...' % (remote_cert_file,)
if not int(dryrun):
put(
local_path=local_cert_file,
remote_path=remote_cert_file, use_sudo=True)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_ssl_dir)s' % env)
sudo('chmod -R %(apache_ssl_chmod)s %(apache_ssl_dir)s' % env)
def install_fixtures(name, site=None):
"""
Installs a set of Django fixtures.
"""
set_site(site)
render_remote_paths()
fixtures_paths = env.db_fixture_sets.get(name, [])
for fixture_path in fixtures_paths:
env.db_fq_fixture_path = os.path.join(env.remote_app_src_package_dir, fixture_path)
print "Loading %s..." % (env.db_fq_fixture_path,)
if not env.is_local and not files.exists(env.db_fq_fixture_path):
put(local_path=env.db_fq_fixture_path, remote_path="/tmp/data.json", use_sudo=True)
env.db_fq_fixture_path = env.put_remote_path
cmd = (
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s loaddata %(db_fq_fixture_path)s"
% env
)
print cmd
run(cmd)
def install_fixtures(name, site=None):
"""
Installs a set of Django fixtures.
"""
set_site(site)
render_remote_paths()
fixtures_paths = env.db_fixture_sets.get(name, [])
for fixture_path in fixtures_paths:
env.db_fq_fixture_path = os.path.join(env.remote_app_src_package_dir,
fixture_path)
print 'Loading %s...' % (env.db_fq_fixture_path, )
if not env.is_local and not files.exists(env.db_fq_fixture_path):
put(local_path=env.db_fq_fixture_path,
remote_path='/tmp/data.json',
use_sudo=True)
env.db_fq_fixture_path = env.put_remote_path
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s loaddata %(db_fq_fixture_path)s' % env
print cmd
run(cmd)
def deploy(clean=0):
"""
Copies the tarball to the target server.
Note, clean=1 will delete any dynamically generated files not included
in the tarball.
"""
tarball_path = get_tarball_path()
assert os.path.isfile(tarball_path), \
'No tarball found. Ensure you ran create() first.'
put(local_path=env.tarball_path)
env.remote_app_dir = env.remote_app_dir_template % env
env.remote_app_src_dir = env.remote_app_src_dir_template % env
env.remote_app_src_package_dir = env.remote_app_src_package_dir_template % env
if int(clean):
print 'Deleting old remote source...'
#sudo('[ -d %(remote_app_src_dir)s ] && rm -Rf %(remote_app_src_dir)s' % env)
sudo('rm -Rf %(remote_app_src_dir)s' % env)
sudo('mkdir -p %(remote_app_src_dir)s' % env)
print 'Extracting tarball...'
sudo('mkdir -p %(remote_app_src_dir)s' % env)
sudo('tar -xvzf %(put_remote_path)s -C %(remote_app_src_dir)s' % env)
for path in env.tarball_extra_dirs:
env.tarball_extra_dir_path = path % env
if path.startswith('/'):
sudo('mkdir -p %(tarball_extra_dir_path)s' % env)
else:
sudo('mkdir -p %(remote_app_dir)s/%(tarball_extra_dir_path)s' % env)
# Mark executables.
print 'Marking source files as executable...'
sudo('chmod +x %(remote_app_src_package_dir)s/*' % env)
sudo('chmod -R %(apache_chmod)s %(remote_app_src_package_dir)s' % env)
sudo('chown -R %(apache_user)s:%(apache_group)s %(remote_app_dir)s' % env)
def deploy(clean=0):
"""
Copies the tarball to the target server.
Note, clean=1 will delete any dynamically generated files not included
in the tarball.
"""
tarball_path = get_tarball_path()
assert os.path.isfile(tarball_path), \
'No tarball found. Ensure you ran create() first.'
put(local_path=env.tarball_path)
env.remote_app_dir = env.remote_app_dir_template % env
env.remote_app_src_dir = env.remote_app_src_dir_template % env
env.remote_app_src_package_dir = env.remote_app_src_package_dir_template % env
if int(clean):
print 'Deleting old remote source...'
#sudo('[ -d %(remote_app_src_dir)s ] && rm -Rf %(remote_app_src_dir)s' % env)
sudo('rm -Rf %(remote_app_src_dir)s' % env)
sudo('mkdir -p %(remote_app_src_dir)s' % env)
print 'Extracting tarball...'
sudo('mkdir -p %(remote_app_src_dir)s' % env)
sudo('tar -xvzf %(put_remote_path)s -C %(remote_app_src_dir)s' % env)
for path in env.tarball_extra_dirs:
env.tarball_extra_dir_path = path % env
if path.startswith('/'):
sudo('mkdir -p %(tarball_extra_dir_path)s' % env)
else:
sudo('mkdir -p %(remote_app_dir)s/%(tarball_extra_dir_path)s' %
env)
# Mark executables.
print 'Marking source files as executable...'
sudo('chmod +x %(remote_app_src_package_dir)s/*' % env)
sudo('chmod -R %(apache_chmod)s %(remote_app_src_package_dir)s' % env)
sudo('chown -R %(apache_user)s:%(apache_group)s %(remote_app_dir)s' % env)
def configure_modsecurity():
env.apache_mods_enabled.append('mod-security')
env.apache_mods_enabled.append('headers')
# Write modsecurity.conf.
fn = common.render_to_file('apache_modsecurity.template.conf')
put(local_path=fn, remote_path='/etc/modsecurity/modsecurity.conf', use_sudo=True)
# Write OWASP rules.
env.apache_modsecurity_download_filename = '/tmp/owasp-modsecurity-crs.tar.gz'
sudo('cd /tmp; wget --output-document=%(apache_modsecurity_download_filename)s %(apache_modsecurity_download_url)s' % env)
env.apache_modsecurity_download_top = sudo("cd /tmp; tar tzf %(apache_modsecurity_download_filename)s | sed -e 's@/.*@@' | uniq" % env)
sudo('cd /tmp; tar -zxvf %(apache_modsecurity_download_filename)s' % env)
sudo('cd /tmp; cp -R %(apache_modsecurity_download_top)s/* /etc/modsecurity/' % env)
sudo('mv /etc/modsecurity/modsecurity_crs_10_setup.conf.example /etc/modsecurity/modsecurity_crs_10_setup.conf' % env)
sudo('rm -f /etc/modsecurity/activated_rules/*')
sudo('cd /etc/modsecurity/base_rules; for f in * ; do ln -s /etc/modsecurity/base_rules/$f /etc/modsecurity/activated_rules/$f ; done')
sudo('cd /etc/modsecurity/optional_rules; for f in * ; do ln -s /etc/modsecurity/optional_rules/$f /etc/modsecurity/activated_rules/$f ; done')
env.apache_httpd_conf_append.append('Include "/etc/modsecurity/activated_rules/*.conf"')
def deploy(site=None, dryrun=0):
"""
Writes entire crontab to the host.
"""
cron_crontabs = []
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print 'site:', site
print 'cron_crontabs_selected:', env.cron_crontabs_selected
for selected_crontab in env.cron_crontabs_selected:
for line in env.cron_crontabs_available.get(selected_crontab, []):
cron_crontabs.append(line % env)
if not cron_crontabs:
return
cron_crontabs = env.cron_crontab_headers + cron_crontabs
cron_crontabs.append('\n')
env.cron_crontabs_rendered = '\n'.join(cron_crontabs)
fn = common.write_to_file(content=env.cron_crontabs_rendered)
if not int(dryrun):
put(local_path=fn)
sudo('crontab -u %(cron_user)s %(put_remote_path)s' % env)
def deploy(site=None, dryrun=0):
"""
Writes entire crontab to the host.
"""
cron_crontabs = []
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print 'site:',site
print 'cron_crontabs_selected:',env.cron_crontabs_selected
for selected_crontab in env.cron_crontabs_selected:
for line in env.cron_crontabs_available.get(selected_crontab, []):
cron_crontabs.append(line % env)
if not cron_crontabs:
return
cron_crontabs = env.cron_crontab_headers + cron_crontabs
cron_crontabs.append('\n')
env.cron_crontabs_rendered = '\n'.join(cron_crontabs)
fn = common.write_to_file(content=env.cron_crontabs_rendered)
if not int(dryrun):
put(local_path=fn)
sudo('crontab -u %(cron_user)s %(put_remote_path)s' % env)
def install_ssl(site=ALL, dryrun=0):
apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(site=site,
setter=set_apache_site_specifics):
print site
set_apache_site_specifics(site)
sudo('mkdir -p %(apache_ssl_dir)s' % env)
if env.apache_ssl:
for cert_type, local_cert_file, remote_cert_file in iter_certificates(
):
print '=' * 80
print 'Installing certificate %s...' % (remote_cert_file, )
if not int(dryrun):
put(local_path=local_cert_file,
remote_path=remote_cert_file,
use_sudo=True)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_ssl_dir)s' % env)
sudo('chmod -R %(apache_ssl_chmod)s %(apache_ssl_dir)s' % env)
def configure_modsecurity():
env.apache_mods_enabled.append('mod-security')
env.apache_mods_enabled.append('headers')
# Write modsecurity.conf.
fn = common.render_to_file('apache_modsecurity.template.conf')
put(local_path=fn,
remote_path='/etc/modsecurity/modsecurity.conf',
use_sudo=True)
# Write OWASP rules.
env.apache_modsecurity_download_filename = '/tmp/owasp-modsecurity-crs.tar.gz'
sudo(
'cd /tmp; wget --output-document=%(apache_modsecurity_download_filename)s %(apache_modsecurity_download_url)s'
% env)
env.apache_modsecurity_download_top = sudo(
"cd /tmp; tar tzf %(apache_modsecurity_download_filename)s | sed -e 's@/.*@@' | uniq"
% env)
sudo('cd /tmp; tar -zxvf %(apache_modsecurity_download_filename)s' % env)
sudo(
'cd /tmp; cp -R %(apache_modsecurity_download_top)s/* /etc/modsecurity/'
% env)
sudo(
'mv /etc/modsecurity/modsecurity_crs_10_setup.conf.example /etc/modsecurity/modsecurity_crs_10_setup.conf'
% env)
sudo('rm -f /etc/modsecurity/activated_rules/*')
sudo(
'cd /etc/modsecurity/base_rules; for f in * ; do ln -s /etc/modsecurity/base_rules/$f /etc/modsecurity/activated_rules/$f ; done'
)
sudo(
'cd /etc/modsecurity/optional_rules; for f in * ; do ln -s /etc/modsecurity/optional_rules/$f /etc/modsecurity/activated_rules/$f ; done'
)
env.apache_httpd_conf_append.append(
'Include "/etc/modsecurity/activated_rules/*.conf"')
def load(db_dump_fn, dryrun=0, force_upload=0):
"""
Restores a database snapshot onto the target database server.
"""
print "!" * 80
print "db.load.site:", env.SITE
print "db.load.role:", env.ROLE
env.db_dump_fn = db_dump_fn
set_db(site=env.SITE, role=env.ROLE)
dryrun = int(dryrun)
# Copy snapshot file to target.
missing_local_dump_error = ("Database dump file %(db_dump_fn)s does not exist.") % env
if env.is_local:
env.db_remote_dump_fn = db_dump_fn
else:
env.db_remote_dump_fn = "/tmp/" + os.path.split(env.db_dump_fn)[-1]
if int(force_upload) or (not dryrun and not env.is_local and not files.exists(env.db_dump_fn)):
assert os.path.isfile(env.db_dump_fn), missing_local_dump_error
print "Uploading database snapshot..."
put(local_path=env.db_dump_fn, remote_path=env.db_remote_dump_fn)
if env.is_local:
assert os.path.isfile(env.db_dump_fn), missing_local_dump_error
if env.db_load_command:
run(env.db_load_command % env)
elif "postgres" in env.db_engine:
with settings(warn_only=True):
cmd = "dropdb --user=%(db_postgresql_postgres_user)s %(db_name)s" % env
print cmd
if not dryrun:
run(cmd)
cmd = 'psql --user=%(db_postgresql_postgres_user)s -c "CREATE DATABASE %(db_name)s;"' % env
print cmd
if not dryrun:
run(cmd)
with settings(warn_only=True):
cmd = 'psql --user=%(db_postgresql_postgres_user)s -c "DROP OWNED BY %(db_user)s CASCADE;"' % env
print cmd
if not dryrun:
run(cmd)
cmd = (
'psql --user=%(db_postgresql_postgres_user)s -c "DROP USER IF EXISTS %(db_user)s; '
"CREATE USER %(db_user)s WITH PASSWORD '%(db_password)s'; "
'GRANT ALL PRIVILEGES ON DATABASE %(db_name)s to %(db_user)s;"'
) % env
print cmd
if not dryrun:
run(cmd)
for createlang in env.db_postgresql_createlangs:
env.db_createlang = createlang
cmd = "createlang -U %(db_postgresql_postgres_user)s %(db_createlang)s %(db_name)s || true" % env
print cmd
if not dryrun:
run(cmd)
cmd = (
"gunzip -c %(db_remote_dump_fn)s | pg_restore -U %(db_postgresql_postgres_user)s --create --dbname=%(db_name)s"
% env
)
print cmd
if not dryrun:
run(cmd)
elif "mysql" in env.db_engine:
# Drop the database if it's there.
# cmd = ("mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' "
cmd = (
"mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' "
"--execute='DROP DATABASE IF EXISTS %(db_name)s'"
) % env
run(cmd)
# Now, create the database.
# cmd = ("mysqladmin -h %(db_host)s -u %(db_user)s -p'%(db_password)s' "
cmd = ("mysqladmin -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' " "create %(db_name)s") % env
run(cmd)
# TODO:create user
# DROP USER '<username>'@'%';
# CREATE USER '<username>'@'%' IDENTIFIED BY '<password>';
# GRANT ALL PRIVILEGES ON *.* TO '<username>'@'%' WITH GRANT OPTION;
# FLUSH PRIVILEGES;
# set collation to unicode?
# ALTER DATABASE <database> CHARACTER SET utf8 COLLATE utf8_general_ci;
# Raise max packet limitation.
run(
(
"mysql -v -h %(db_host)s -D %(db_name)s -u %(db_root_user)s "
'-p"%(db_root_password)s" --execute="SET global '
"net_buffer_length=%(db_mysql_net_buffer_length)s; SET global "
'max_allowed_packet=%(db_mysql_max_allowed_packet)s;"'
)
% env
)
# Run any server-specific commands (e.g. to setup permissions) before
# we load the data.
for command in env.db_mysql_preload_commands:
run(command % env)
# Restore the database content from the dump file.
env.db_dump_fn = db_dump_fn
cmd = (
"gunzip < %(db_dump_fn)s | mysql -u %(db_root_user)s "
"--password=%(db_root_password)s --host=%(db_host)s "
"-D %(db_name)s"
) % env
run(cmd)
else:
raise NotImplemented
def configure(name=None, site=None, _role=None, dryrun=0):
"""
Configures a fresh install of the database
"""
assert env[ROLE]
require("app_name")
set_db(name=name, site=site, role=_role)
# print 'site:',env[SITE]
# print 'role:',env[ROLE]
env.dryrun = int(dryrun)
if "postgres" in env.db_engine:
env.pg_ver = run('psql --version | grep -o -E "[0-9]+.[0-9]+"')
print "PostgreSQL version %(pg_ver)s detected." % env
print "Backing up PostgreSQL configuration files..."
sudo(
"cp /etc/postgresql/%(pg_ver)s/main/postgresql.conf /etc/postgresql/%(pg_ver)s/main/postgresql.conf.$(date +%%Y%%m%%d%%H%%M).bak"
% env
)
sudo(
"cp /etc/postgresql/%(pg_ver)s/main/pg_hba.conf /etc/postgresql/%(pg_ver)s/main/pg_hba.conf.$(date +%%Y%%m%%d%%H%%M).bak"
% env
)
print "Allowing remote connections..."
fn = common.render_to_file("pg_hba.template.conf")
put(local_path=fn, remote_path="/etc/postgresql/%(pg_ver)s/main/pg_hba.conf" % env, use_sudo=True)
# Don't do this. Keep it locked down and use an SSH tunnel instead.
# See common.tunnel()
# sudo('sed -i "s/#listen_addresses = \'localhost\'/listen_addresses = \'*\'/g" /etc/postgresql/%(pg_ver)s/main/postgresql.conf' % env)
print "Enabling auto-vacuuming..."
sudo('sed -i "s/#autovacuum = on/autovacuum = on/g" /etc/postgresql/%(pg_ver)s/main/postgresql.conf' % env)
sudo('sed -i "s/#track_counts = on/track_counts = on/g" /etc/postgresql/%(pg_ver)s/main/postgresql.conf' % env)
# Set UTF-8 as the default database encoding.
sudo(
'psql --user=postgres --no-password --command="'
"UPDATE pg_database SET datistemplate = FALSE WHERE datname = 'template1';"
"DROP DATABASE template1;"
"CREATE DATABASE template1 WITH TEMPLATE = template0 ENCODING = 'UNICODE';"
"UPDATE pg_database SET datistemplate = TRUE WHERE datname = 'template1';"
"\c template1\n"
"VACUUM FREEZE;"
"UPDATE pg_database SET datallowconn = FALSE WHERE datname = 'template1';\""
)
elif "mysql" in env.db_engine:
if env.db_allow_remote_connections:
# Enable remote connections.
sudo("sed -i 's/127.0.0.1/0.0.0.0/g' %(db_mysql_conf)s" % env)
# Enable root logins from remote connections.
sudo(
"mysql -u %(db_root_user)s -p\"%(db_root_password)s\" --execute=\"USE mysql; GRANT ALL ON *.* to %(db_root_user)s@'%%' IDENTIFIED BY '%(db_root_password)s'; FLUSH PRIVILEGES;\""
% env
)
sudo("service mysql restart")
else:
print "No database parameters found."
def configure(name=None, site=None, _role=None, dryrun=0):
"""
Configures a fresh install of the database
"""
assert env[ROLE]
require('app_name')
set_db(name=name, site=site, role=_role)
# print 'site:',env[SITE]
# print 'role:',env[ROLE]
env.dryrun = int(dryrun)
if 'postgres' in env.db_engine:
env.pg_ver = run('psql --version | grep -o -E "[0-9]+.[0-9]+"')
print 'PostgreSQL version %(pg_ver)s detected.' % env
print 'Backing up PostgreSQL configuration files...'
sudo(
'cp /etc/postgresql/%(pg_ver)s/main/postgresql.conf /etc/postgresql/%(pg_ver)s/main/postgresql.conf.$(date +%%Y%%m%%d%%H%%M).bak'
% env)
sudo(
'cp /etc/postgresql/%(pg_ver)s/main/pg_hba.conf /etc/postgresql/%(pg_ver)s/main/pg_hba.conf.$(date +%%Y%%m%%d%%H%%M).bak'
% env)
print 'Allowing remote connections...'
fn = common.render_to_file('pg_hba.template.conf')
put(local_path=fn,
remote_path='/etc/postgresql/%(pg_ver)s/main/pg_hba.conf' % env,
use_sudo=True)
# Don't do this. Keep it locked down and use an SSH tunnel instead.
# See common.tunnel()
#sudo('sed -i "s/#listen_addresses = \'localhost\'/listen_addresses = \'*\'/g" /etc/postgresql/%(pg_ver)s/main/postgresql.conf' % env)
print 'Enabling auto-vacuuming...'
sudo(
'sed -i "s/#autovacuum = on/autovacuum = on/g" /etc/postgresql/%(pg_ver)s/main/postgresql.conf'
% env)
sudo(
'sed -i "s/#track_counts = on/track_counts = on/g" /etc/postgresql/%(pg_ver)s/main/postgresql.conf'
% env)
# Set UTF-8 as the default database encoding.
sudo(
'psql --user=postgres --no-password --command="'
'UPDATE pg_database SET datistemplate = FALSE WHERE datname = \'template1\';'
'DROP DATABASE template1;'
'CREATE DATABASE template1 WITH TEMPLATE = template0 ENCODING = \'UNICODE\';'
'UPDATE pg_database SET datistemplate = TRUE WHERE datname = \'template1\';'
'\c template1\n'
'VACUUM FREEZE;'
'UPDATE pg_database SET datallowconn = FALSE WHERE datname = \'template1\';"'
)
elif 'mysql' in env.db_engine:
if env.db_allow_remote_connections:
# Enable remote connections.
sudo("sed -i 's/127.0.0.1/0.0.0.0/g' %(db_mysql_conf)s" % env)
# Enable root logins from remote connections.
sudo(
'mysql -u %(db_root_user)s -p"%(db_root_password)s" --execute="USE mysql; GRANT ALL ON *.* to %(db_root_user)s@\'%%\' IDENTIFIED BY \'%(db_root_password)s\'; FLUSH PRIVILEGES;"'
% env)
sudo('service mysql restart')
else:
print 'No database parameters found.'
def load(db_dump_fn, dryrun=0, force_upload=0):
"""
Restores a database snapshot onto the target database server.
"""
print '!' * 80
print 'db.load.site:', env.SITE
print 'db.load.role:', env.ROLE
env.db_dump_fn = db_dump_fn
set_db(site=env.SITE, role=env.ROLE)
dryrun = int(dryrun)
# Copy snapshot file to target.
missing_local_dump_error = (
"Database dump file %(db_dump_fn)s does not exist.") % env
if env.is_local:
env.db_remote_dump_fn = db_dump_fn
else:
env.db_remote_dump_fn = '/tmp/' + os.path.split(env.db_dump_fn)[-1]
if int(force_upload) or (not dryrun and not env.is_local
and not files.exists(env.db_dump_fn)):
assert os.path.isfile(env.db_dump_fn), \
missing_local_dump_error
print 'Uploading database snapshot...'
put(local_path=env.db_dump_fn, remote_path=env.db_remote_dump_fn)
if env.is_local:
assert os.path.isfile(env.db_dump_fn), \
missing_local_dump_error
if env.db_load_command:
run(env.db_load_command % env)
elif 'postgres' in env.db_engine:
with settings(warn_only=True):
cmd = 'dropdb --user=%(db_postgresql_postgres_user)s %(db_name)s' % env
print cmd
if not dryrun:
run(cmd)
cmd = 'psql --user=%(db_postgresql_postgres_user)s -c "CREATE DATABASE %(db_name)s;"' % env
print cmd
if not dryrun:
run(cmd)
with settings(warn_only=True):
cmd = 'psql --user=%(db_postgresql_postgres_user)s -c "DROP OWNED BY %(db_user)s CASCADE;"' % env
print cmd
if not dryrun:
run(cmd)
cmd = (
'psql --user=%(db_postgresql_postgres_user)s -c "DROP USER IF EXISTS %(db_user)s; '
'CREATE USER %(db_user)s WITH PASSWORD \'%(db_password)s\'; '
'GRANT ALL PRIVILEGES ON DATABASE %(db_name)s to %(db_user)s;"'
) % env
print cmd
if not dryrun:
run(cmd)
for createlang in env.db_postgresql_createlangs:
env.db_createlang = createlang
cmd = 'createlang -U %(db_postgresql_postgres_user)s %(db_createlang)s %(db_name)s || true' % env
print cmd
if not dryrun:
run(cmd)
cmd = 'gunzip -c %(db_remote_dump_fn)s | pg_restore -U %(db_postgresql_postgres_user)s --create --dbname=%(db_name)s' % env
print cmd
if not dryrun:
run(cmd)
elif 'mysql' in env.db_engine:
# Drop the database if it's there.
#cmd = ("mysql -v -h %(db_host)s -u %(db_user)s -p'%(db_password)s' "
cmd = (
"mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' "
"--execute='DROP DATABASE IF EXISTS %(db_name)s'") % env
run(cmd)
# Now, create the database.
#cmd = ("mysqladmin -h %(db_host)s -u %(db_user)s -p'%(db_password)s' "
cmd = (
"mysqladmin -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' "
"create %(db_name)s") % env
run(cmd)
#TODO:create user
# DROP USER '<username>'@'%';
# CREATE USER '<username>'@'%' IDENTIFIED BY '<password>';
# GRANT ALL PRIVILEGES ON *.* TO '<username>'@'%' WITH GRANT OPTION;
# FLUSH PRIVILEGES;
#set collation to unicode?
#ALTER DATABASE <database> CHARACTER SET utf8 COLLATE utf8_general_ci;
# Raise max packet limitation.
run(('mysql -v -h %(db_host)s -D %(db_name)s -u %(db_root_user)s '
'-p"%(db_root_password)s" --execute="SET global '
'net_buffer_length=%(db_mysql_net_buffer_length)s; SET global '
'max_allowed_packet=%(db_mysql_max_allowed_packet)s;"') % env)
# Run any server-specific commands (e.g. to setup permissions) before
# we load the data.
for command in env.db_mysql_preload_commands:
run(command % env)
# Restore the database content from the dump file.
env.db_dump_fn = db_dump_fn
cmd = ('gunzip < %(db_dump_fn)s | mysql -u %(db_root_user)s '
'--password=%(db_root_password)s --host=%(db_host)s '
'-D %(db_name)s') % env
run(cmd)
else:
raise NotImplemented
def configure(full=1, site=ALL, delete_old=0):
"""
Configures Apache to host one or more websites.
"""
from burlap import service
print 'Configuring Apache...'
apache_specifics = set_apache_specifics()
if int(delete_old):
# Delete all existing enabled and available sites.
sudo('rm -f %(apache_sites_available)s/*' % env)
sudo('rm -f %(apache_sites_enabled)s/*' % env)
for site, site_data in common.iter_sites(site=site,
setter=set_apache_site_specifics):
#print '-'*80
print site
#continue
print 'env.apache_ssl_domain:', env.apache_ssl_domain
print 'env.apache_ssl_domain_template:', env.apache_ssl_domain_template
fn = common.render_to_file('django.template.wsgi')
put(local_path=fn, remote_path=env.apache_django_wsgi, use_sudo=True)
if env.apache_ssl:
env.apache_ssl_certificates = list(iter_certificates())
fn = common.render_to_file('apache_site.template.conf')
env.apache_site_conf = site + '.conf'
env.apache_site_conf_fqfn = os.path.join(env.apache_sites_available,
env.apache_site_conf)
put(local_path=fn,
remote_path=env.apache_site_conf_fqfn,
use_sudo=True)
sudo('a2ensite %(apache_site_conf)s' % env)
#return
if service.is_selected(APACHE2_MODEVASIVE):
configure_modevasive()
if service.is_selected(APACHE2_MODSECURITY):
configure_modsecurity()
for mod_enabled in env.apache_mods_enabled:
env.apache_mod_enabled = mod_enabled
sudo('a2enmod %(apache_mod_enabled)s' % env)
if int(full):
# Write master Apache configuration file.
fn = common.render_to_file('apache_httpd.template.conf')
put(local_path=fn, remote_path=env.apache_conf, use_sudo=True)
# Write Apache listening ports configuration.
fn = common.render_to_file('apache_ports.template.conf')
put(local_path=fn, remote_path=env.apache_ports, use_sudo=True)
#sudo('mkdir -p %(apache_app_log_dir)s' % env)
#sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_app_log_dir)s' % env)
# sudo('mkdir -p %(apache_log_dir)s' % env)
# sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_log_dir)s' % env)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_root)s' % env)
