def install_auth_basic_user_file(site=None):
"""
Installs users for basic httpd auth.
"""
apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(site=site,
setter=set_apache_site_specifics):
print '~' * 80
print 'Site:', site
#env.update(env_default)
#env.update(env.sites[site])
#set_apache_site_specifics(site)
print 'env.apache_auth_basic:', env.apache_auth_basic
if not env.apache_auth_basic:
continue
#assert env.apache_auth_basic, 'This site is not configured for Apache basic authenticated.'
assert env.apache_auth_basic_users, 'No apache auth users specified.'
for username, password in env.apache_auth_basic_users:
env.apache_auth_basic_username = username
env.apache_auth_basic_password = password
if files.exists(env.apache_auth_basic_authuserfile):
sudo(
'htpasswd -b %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s'
% env)
else:
sudo(
'htpasswd -b -c %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s'
% env)
def install_ssl(self, site=ALL):
from burlap.common import iter_sites
verbose = self.verbose
self.get_apache_settings()
apache_specifics = self.set_apache_specifics()
for site, site_data in iter_sites(
site=site, setter=self.set_apache_site_specifics):
site_secure = site + '_secure'
if site_secure not in self.genv.sites:
continue
self.set_apache_site_specifics(site_secure)
self.sudo_or_dryrun('mkdir -p %(apache_ssl_dir)s' % self.genv)
if self.genv.apache_ssl:
for cert_type, local_cert_file, remote_cert_file in self.iter_certificates(
):
if verbose:
print('=' * 80)
print('Installing certificate %s...' %
(remote_cert_file, ))
self.put_or_dryrun(local_path=local_cert_file,
remote_path=remote_cert_file,
use_sudo=True)
self.sudo_or_dryrun('mkdir -p %(apache_ssl_dir)s' % self.genv)
self.sudo_or_dryrun(
'chown -R %(apache_user)s:%(apache_group)s %(apache_ssl_dir)s' %
self.genv)
self.sudo_or_dryrun(
'chmod -R %(apache_ssl_chmod)s %(apache_ssl_dir)s' % self.genv)
def deploy_services(site=None, dryrun=0):
"""
Collects the configurations for all registered services and writes
the appropriate supervisord.conf file.
"""
dryrun = int(dryrun)
render_paths()
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print site
for cb in env._supervisor_create_service_callbacks:
ret = cb()
if isinstance(ret, basestring):
env.supervisor_services.append(ret)
# else:
# print 'invalid'
env.supervisor_services_rendered = "\n".join(env.supervisor_services)
# print env.supervisor_services_rendered
fn = common.render_to_file("supervisor_daemon.template.config")
if dryrun:
print open(fn).read()
else:
put(local_path=fn, remote_path=env.supervisor_config_path, use_sudo=True)
def restart(site=common.ALL):
for site, site_data in common.iter_sites(
site=site, renderer=lambda: set_db(name='default')):
print site
#set_db(name=name, site=site)
if 'postgres' in env.db_engine:
sudo('service postgresql restart; sleep 3')
def deploy_services(site=None, dryrun=0):
"""
Collects the configurations for all registered services and writes
the appropriate supervisord.conf file.
"""
dryrun = int(dryrun)
render_paths()
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print site
for cb in env._supervisor_create_service_callbacks:
ret = cb()
if isinstance(ret, basestring):
env.supervisor_services.append(ret)
# else:
# print 'invalid'
env.supervisor_services_rendered = '\n'.join(env.supervisor_services)
#print env.supervisor_services_rendered
fn = common.render_to_file('supervisor_daemon.template.config')
if dryrun:
print open(fn).read()
else:
put(local_path=fn,
remote_path=env.supervisor_config_path,
use_sudo=True)
def install_ssl(self, site=ALL):
from burlap.common import iter_sites
verbose = self.verbose
self.get_apache_settings()
apache_specifics = self.set_apache_specifics()
for site, site_data in iter_sites(site=site, setter=self.set_apache_site_specifics):
site_secure = site+'_secure'
if site_secure not in self.genv.sites:
continue
self.set_apache_site_specifics(site_secure)
self.sudo_or_dryrun('mkdir -p %(apache_ssl_dir)s' % self.genv)
if self.genv.apache_ssl:
for cert_type, local_cert_file, remote_cert_file in self.iter_certificates():
if verbose:
print('='*80)
print('Installing certificate %s...' % (remote_cert_file,))
self.put_or_dryrun(
local_path=local_cert_file,
remote_path=remote_cert_file,
use_sudo=True)
self.sudo_or_dryrun('mkdir -p %(apache_ssl_dir)s' % self.genv)
self.sudo_or_dryrun('chown -R %(apache_user)s:%(apache_group)s %(apache_ssl_dir)s' % self.genv)
self.sudo_or_dryrun('chmod -R %(apache_ssl_chmod)s %(apache_ssl_dir)s' % self.genv)
def install_auth_basic_user_file(self, site=None):
"""
Installs users for basic httpd auth.
"""
from burlap.common import iter_sites
self.get_apache_settings()
apache_specifics = self.set_apache_specifics()
for site, site_data in iter_sites(site=site, setter=self.set_apache_site_specifics):
if self.verbose:
print('~'*80, file=sys.stderr)
print('Site:',site, file=sys.stderr)
print('env.apache_auth_basic:', self.genv.apache_auth_basic, file=sys.stderr)
if not self.genv.apache_auth_basic:
continue
#assert self.genv.apache_auth_basic, 'This site is not configured for Apache basic authenticated.'
assert self.genv.apache_auth_basic_users, 'No apache auth users specified.'
for username,password in self.genv.apache_auth_basic_users:
self.genv.apache_auth_basic_username = username
self.genv.apache_auth_basic_password = password
if self.files.exists(self.genv.apache_auth_basic_authuserfile):
self.sudo_or_dryrun('htpasswd -b %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s' % self.genv)
else:
self.sudo_or_dryrun('htpasswd -b -c %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s' % self.genv)
def install_auth_basic_user_file(site=None):
"""
Installs users for basic httpd auth.
"""
apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(site=site, setter=set_apache_site_specifics):
print '~'*80
print 'Site:',site
#env.update(env_default)
#env.update(env.sites[site])
#set_apache_site_specifics(site)
print 'env.apache_auth_basic:',env.apache_auth_basic
if not env.apache_auth_basic:
continue
#assert env.apache_auth_basic, 'This site is not configured for Apache basic authenticated.'
assert env.apache_auth_basic_users, 'No apache auth users specified.'
for username,password in env.apache_auth_basic_users:
env.apache_auth_basic_username = username
env.apache_auth_basic_password = password
if files.exists(env.apache_auth_basic_authuserfile):
sudo('htpasswd -b %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s' % env)
else:
sudo('htpasswd -b -c %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s' % env)
def generate_csr(domain='', r=None):
"""
Creates a certificate signing request to be submitted to a formal
certificate authority to generate a certificate.
Note, the provider may say the CSR must be created on the target server,
but this is not necessary.
"""
from apache import set_apache_specifics, set_apache_site_specifics
env.ssl_domain = domain or env.ssl_domain
role = r or env.ROLE or ALL
ssl_dst = 'roles/%s/ssl' % (role, )
print('ssl_dst:', ssl_dst)
if not os.path.isdir(ssl_dst):
os.makedirs(ssl_dst)
#apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(setter=set_apache_site_specifics):
print('site:', site, file=sys.stderr)
#
assert env.ssl_domain, 'No SSL domain defined.'
#2048?
env.ssl_base_dst = '%s/%s' % (ssl_dst, env.ssl_domain.replace(
'*.', ''))
env.ssl_csr_year = date.today().year
cmd = 'openssl req -nodes -newkey rsa:%(ssl_length)s -subj "/C=%(ssl_country)s/ST=%(ssl_state)s/L=%(ssl_city)s/O=%(ssl_organization)s/CN=%(ssl_domain)s" -keyout %(ssl_base_dst)s.%(ssl_csr_year)i.key -out %(ssl_base_dst)s.%(ssl_csr_year)i.csr' % env
local_or_dryrun(cmd)
def install_auth_basic_user_file(self, site=None):
"""
Installs users for basic httpd auth.
"""
from burlap.common import iter_sites
self.get_apache_settings()
apache_specifics = self.set_apache_specifics()
for site, site_data in iter_sites(
site=site, setter=self.set_apache_site_specifics):
if self.verbose:
print('~' * 80, file=sys.stderr)
print('Site:', site, file=sys.stderr)
print('env.apache_auth_basic:',
self.genv.apache_auth_basic,
file=sys.stderr)
if not self.genv.apache_auth_basic:
continue
#assert self.genv.apache_auth_basic, 'This site is not configured for Apache basic authenticated.'
assert self.genv.apache_auth_basic_users, 'No apache auth users specified.'
for username, password in self.genv.apache_auth_basic_users:
self.genv.apache_auth_basic_username = username
self.genv.apache_auth_basic_password = password
if self.files.exists(self.genv.apache_auth_basic_authuserfile):
self.sudo_or_dryrun(
'htpasswd -b %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s'
% self.genv)
else:
self.sudo_or_dryrun(
'htpasswd -b -c %(apache_auth_basic_authuserfile)s %(apache_auth_basic_username)s %(apache_auth_basic_password)s'
% self.genv)
def configure(full=1, site=ALL, delete_old=0):
"""
Configures Apache to host one or more websites.
"""
from burlap import service
print 'Configuring Apache...'
apache_specifics = set_apache_specifics()
if int(delete_old):
# Delete all existing enabled and available sites.
sudo('rm -f %(apache_sites_available)s/*' % env)
sudo('rm -f %(apache_sites_enabled)s/*' % env)
for site, site_data in common.iter_sites(site=site, setter=set_apache_site_specifics):
#print '-'*80
print site
#continue
print 'env.apache_ssl_domain:',env.apache_ssl_domain
print 'env.apache_ssl_domain_template:',env.apache_ssl_domain_template
fn = common.render_to_file('django.template.wsgi')
put(local_path=fn, remote_path=env.apache_django_wsgi, use_sudo=True)
if env.apache_ssl:
env.apache_ssl_certificates = list(iter_certificates())
fn = common.render_to_file('apache_site.template.conf')
env.apache_site_conf = site+'.conf'
env.apache_site_conf_fqfn = os.path.join(env.apache_sites_available, env.apache_site_conf)
put(local_path=fn, remote_path=env.apache_site_conf_fqfn, use_sudo=True)
sudo('a2ensite %(apache_site_conf)s' % env)
#return
if service.is_selected(APACHE2_MODEVASIVE):
configure_modevasive()
if service.is_selected(APACHE2_MODSECURITY):
configure_modsecurity()
for mod_enabled in env.apache_mods_enabled:
env.apache_mod_enabled = mod_enabled
sudo('a2enmod %(apache_mod_enabled)s' % env)
if int(full):
# Write master Apache configuration file.
fn = common.render_to_file('apache_httpd.template.conf')
put(local_path=fn, remote_path=env.apache_conf, use_sudo=True)
# Write Apache listening ports configuration.
fn = common.render_to_file('apache_ports.template.conf')
put(local_path=fn, remote_path=env.apache_ports, use_sudo=True)
#sudo('mkdir -p %(apache_app_log_dir)s' % env)
#sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_app_log_dir)s' % env)
# sudo('mkdir -p %(apache_log_dir)s' % env)
# sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_log_dir)s' % env)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_root)s' % env)
def iter_unique_databases(site=None):
prior_database_names = set()
for site, site_data in common.iter_sites(site=site, no_secure=True):
set_db(site=site)
key = (env.db_name, env.db_user, env.db_host, env.db_engine)
if key in prior_database_names:
continue
prior_database_names.add(key)
env.SITE = site
yield site, site_data
def iter_unique_databases(site=None):
prior_database_names = set()
for site, site_data in common.iter_sites(site=site, no_secure=True):
set_db(site=site)
key = (env.db_name, env.db_user, env.db_host, env.db_engine)
if key in prior_database_names:
continue
prior_database_names.add(key)
env.SITE = site
yield site, site_data
def _configure(self, site=None, full=0, only_data=0):
"""
Installs and configures RabbitMQ.
"""
from burlap.dj import get_settings
from burlap import packager
from burlap.common import iter_sites
full = int(full)
# assert self.env.erlang_cookie
if full and not only_data:
packager.install_required(type=SYSTEM, service=RABBITMQ)
#render_paths()
params = set() # [(user,vhost)]
for site, site_data in iter_sites(site=site,
renderer=self.render_paths,
no_secure=True):
if self.verbose:
print('!' * 80, file=sys.stderr)
print('site:', site, file=sys.stderr)
_settings = get_settings(site=site)
#print '_settings:',_settings
if not _settings:
continue
if hasattr(_settings, 'BROKER_USER') and hasattr(
_settings, 'BROKER_VHOST'):
if self.verbose:
print('RabbitMQ:', _settings.BROKER_USER,
_settings.BROKER_VHOST)
params.add((_settings.BROKER_USER, _settings.BROKER_PASSWORD,
_settings.BROKER_VHOST))
params = sorted(list(params))
if not only_data:
for user, password, vhost in params:
self.env.broker_user = user
self.env.broker_password = password
self.env.broker_vhost = vhost
with settings(warn_only=True):
self.sudo_or_dryrun(
'rabbitmqctl add_user %(rabbitmq_broker_user)s %(rabbitmq_broker_password)s'
% self.genv)
cmd = 'rabbitmqctl add_vhost %(rabbitmq_broker_vhost)s' % self.genv
self.sudo_or_dryrun(cmd)
cmd = 'rabbitmqctl set_permissions -p %(rabbitmq_broker_vhost)s %(rabbitmq_broker_user)s ".*" ".*" ".*"' % self.genv
self.sudo_or_dryrun(cmd)
return params
def deploy(self, site=None):
"""
Writes entire crontab to the host.
"""
from burlap.common import get_current_hostname, iter_sites
cron_crontabs = []
hostname = get_current_hostname()
target_sites = self.genv.available_sites_by_host.get(hostname, None)
if self.verbose:
print('hostname: "%s"' % (hostname, ), file=sys.stderr)
for site, site_data in iter_sites(site=site):
if self.verbose:
print('site:', site, file=sys.stderr)
env = self.render_paths(type(self.genv)(self.genv))
# Only load site configurations that are allowed for this host.
if target_sites is None:
pass
else:
assert isinstance(target_sites, (tuple, list))
if site not in target_sites:
print('Skipping:', site, file=sys.stderr)
continue
if self.verbose:
print('env.crontabs_selected:',
self.env.crontabs_selected,
file=sys.stderr)
for selected_crontab in self.env.crontabs_selected:
lines = self.env.crontabs_available.get(selected_crontab, [])
if self.verbose:
print('lines:', lines, file=sys.stderr)
for line in lines:
cron_crontabs.append(line % env)
if not cron_crontabs:
return
cron_crontabs = self.env.crontab_headers + cron_crontabs
cron_crontabs.append('\n')
env.crontabs_rendered = '\n'.join(cron_crontabs)
fn = self.write_to_file(content=env.crontabs_rendered)
if self.dryrun:
print('echo %s > %s' % (env.crontabs_rendered, fn))
self.put_or_dryrun(local_path=fn)
env.put_remote_path = self.genv.put_remote_path
self.sudo_or_dryrun('crontab -u %(cron_user)s %(put_remote_path)s' %
env)
def optimize_wsgi_processes(self):
"""
Based on the number of sites per server and the number of resources on the server,
calculates the optimal number of processes that should be allocated for each WSGI site.
"""
from burlap.common import iter_sites
#self.env.wsgi_processes = 5
self.env.wsgi_server_memory_gb = 8
verbose = self.verbose
self.get_apache_settings()
apache_specifics = self.set_apache_specifics()
all_sites = list(iter_sites(site=ALL, setter=self.set_apache_site_specifics))
def deploy_services(self, site=None):
"""
Collects the configurations for all registered services and writes
the appropriate supervisord.conf file.
"""
from burlap.common import iter_sites
verbose = self.verbose
self.render_paths()
supervisor_services = []
process_groups = []
for site, site_data in iter_sites(site=site,
renderer=self.render_paths):
if verbose:
print(site)
for cb in self.genv._supervisor_create_service_callbacks:
ret = cb()
if isinstance(ret, basestring):
supervisor_services.append(ret)
elif isinstance(ret, tuple):
assert len(ret) == 2
conf_name, conf_content = ret
if verbose:
print('conf_name:', conf_name)
print('conf_content:', conf_content)
remote_fn = os.path.join(self.env.conf_dir, conf_name)
local_fn = self.write_to_file(conf_content)
self.put_or_dryrun(local_path=local_fn,
remote_path=remote_fn,
use_sudo=True)
process_groups.append(os.path.splitext(conf_name)[0])
self.env.services_rendered = '\n'.join(supervisor_services)
fn = self.render_to_file(self.env.config_template)
self.put_or_dryrun(local_path=fn,
remote_path=self.env.config_path,
use_sudo=True)
for pg in process_groups:
self.sudo_or_dryrun('supervisorctl add %s' % pg)
#TODO:are all these really necessary?
self.sudo_or_dryrun('supervisorctl restart all')
self.sudo_or_dryrun('supervisorctl reread')
self.sudo_or_dryrun('supervisorctl update')
def optimize_wsgi_processes(self):
"""
Based on the number of sites per server and the number of resources on the server,
calculates the optimal number of processes that should be allocated for each WSGI site.
"""
from burlap.common import iter_sites
#self.env.wsgi_processes = 5
self.env.wsgi_server_memory_gb = 8
verbose = self.verbose
self.get_apache_settings()
apache_specifics = self.set_apache_specifics()
all_sites = list(
iter_sites(site=ALL, setter=self.set_apache_site_specifics))
def deploy(self, site=None):
"""
Writes entire crontab to the host.
"""
from burlap.common import get_current_hostname, iter_sites
cron_crontabs = []
hostname = get_current_hostname()
target_sites = self.genv.available_sites_by_host.get(hostname, None)
if self.verbose:
print('hostname: "%s"' % (hostname,), file=sys.stderr)
for site, site_data in iter_sites(site=site):
if self.verbose:
print('site:', site, file=sys.stderr)
env = self.render_paths(type(self.genv)(self.genv))
# Only load site configurations that are allowed for this host.
if target_sites is None:
pass
else:
assert isinstance(target_sites, (tuple, list))
if site not in target_sites:
print('Skipping:', site, file=sys.stderr)
continue
if self.verbose:
print('env.crontabs_selected:', self.env.crontabs_selected, file=sys.stderr)
for selected_crontab in self.env.crontabs_selected:
lines = self.env.crontabs_available.get(selected_crontab, [])
if self.verbose:
print('lines:', lines, file=sys.stderr)
for line in lines:
cron_crontabs.append(line % env)
if not cron_crontabs:
return
cron_crontabs = self.env.crontab_headers + cron_crontabs
cron_crontabs.append('\n')
env.crontabs_rendered = '\n'.join(cron_crontabs)
fn = self.write_to_file(content=env.crontabs_rendered)
if self.dryrun:
print('echo %s > %s' % (env.crontabs_rendered, fn))
self.put_or_dryrun(local_path=fn)
env.put_remote_path = self.genv.put_remote_path
self.sudo_or_dryrun('crontab -u %(cron_user)s %(put_remote_path)s' % env)
def execute_sql(fn, name='default', site=None):
"""
Executes an arbitrary SQL file.
"""
from burlap.dj import set_db
from burlap.db import load_db_set
assert os.path.isfile(fn), 'Missing file: %s' % fn
site_summary = {} # {site: ret}
for site, site_data in common.iter_sites(site=site, no_secure=True):
try:
set_db(name=name, site=site)
load_db_set(name=name)
env.SITE = site
put_or_dryrun(local_path=fn)
with settings(warn_only=True):
ret = None
if 'postgres' in env.db_engine or 'postgis' in env.db_engine:
ret = run_or_dryrun(
"psql --host=%(db_host)s --user=%(db_user)s -d %(db_name)s -f %(put_remote_path)s"
% env)
elif 'mysql' in env.db_engine:
ret = run_or_dryrun(
"mysql -h %(db_host)s -u %(db_user)s -p'%(db_password)s' %(db_name)s < %(put_remote_path)s"
% env)
else:
raise NotImplementedError('Unknown database type: %s' %
env.db_engine)
print('ret:', ret)
site_summary[site] = ret
except KeyError as e:
site_summary[site] = 'Error: %s' % str(e)
pass
print('-' * 80)
print('Site Summary:')
for site, ret in sorted(site_summary.items(), key=lambda o: o[0]):
print(site, ret)
def deploy_services(self, site=None):
"""
Collects the configurations for all registered services and writes
the appropriate supervisord.conf file.
"""
from burlap.common import iter_sites
verbose = self.verbose
self.render_paths()
supervisor_services = []
process_groups = []
for site, site_data in iter_sites(site=site, renderer=self.render_paths):
if verbose:
print(site)
for cb in self.genv._supervisor_create_service_callbacks:
ret = cb()
if isinstance(ret, basestring):
supervisor_services.append(ret)
elif isinstance(ret, tuple):
assert len(ret) == 2
conf_name, conf_content = ret
if verbose:
print('conf_name:', conf_name)
print('conf_content:', conf_content)
remote_fn = os.path.join(self.env.conf_dir, conf_name)
local_fn = self.write_to_file(conf_content)
self.put_or_dryrun(local_path=local_fn, remote_path=remote_fn, use_sudo=True)
process_groups.append(os.path.splitext(conf_name)[0])
self.env.services_rendered = '\n'.join(supervisor_services)
fn = self.render_to_file(self.env.config_template)
self.put_or_dryrun(local_path=fn, remote_path=self.env.config_path, use_sudo=True)
for pg in process_groups:
self.sudo_or_dryrun('supervisorctl add %s' % pg)
#TODO:are all these really necessary?
self.sudo_or_dryrun('supervisorctl restart all')
self.sudo_or_dryrun('supervisorctl reread')
self.sudo_or_dryrun('supervisorctl update')
def _configure(self, site=None, full=0, only_data=0):
"""
Installs and configures RabbitMQ.
"""
from burlap.dj import get_settings
from burlap import packager
from burlap.common import iter_sites
full = int(full)
# assert self.env.erlang_cookie
if full and not only_data:
packager.install_required(type=SYSTEM, service=RABBITMQ)
#render_paths()
params = set() # [(user,vhost)]
for site, site_data in iter_sites(site=site, renderer=self.render_paths, no_secure=True):
if self.verbose:
print('!'*80, file=sys.stderr)
print('site:', site, file=sys.stderr)
_settings = get_settings(site=site)
#print '_settings:',_settings
if not _settings:
continue
if hasattr(_settings, 'BROKER_USER') and hasattr(_settings, 'BROKER_VHOST'):
if self.verbose:
print('RabbitMQ:',_settings.BROKER_USER, _settings.BROKER_VHOST)
params.add((_settings.BROKER_USER, _settings.BROKER_PASSWORD, _settings.BROKER_VHOST))
params = sorted(list(params))
if not only_data:
for user, password, vhost in params:
self.env.broker_user = user
self.env.broker_password = password
self.env.broker_vhost = vhost
with settings(warn_only=True):
self.sudo_or_dryrun('rabbitmqctl add_user %(rabbitmq_broker_user)s %(rabbitmq_broker_password)s' % self.genv)
cmd = 'rabbitmqctl add_vhost %(rabbitmq_broker_vhost)s' % self.genv
self.sudo_or_dryrun(cmd)
cmd = 'rabbitmqctl set_permissions -p %(rabbitmq_broker_vhost)s %(rabbitmq_broker_user)s ".*" ".*" ".*"' % self.genv
self.sudo_or_dryrun(cmd)
return params
def execute_sql(fn, name='default', site=None):
"""
Executes an arbitrary SQL file.
"""
from burlap.dj import set_db
from burlap.db import load_db_set
assert os.path.isfile(fn), 'Missing file: %s' % fn
site_summary = {} # {site: ret}
for site, site_data in common.iter_sites(site=site, no_secure=True):
try:
set_db(name=name, site=site)
load_db_set(name=name)
env.SITE = site
put_or_dryrun(local_path=fn)
with settings(warn_only=True):
ret = None
if 'postgres' in env.db_engine or 'postgis' in env.db_engine:
ret = run_or_dryrun("psql --host=%(db_host)s --user=%(db_user)s -d %(db_name)s -f %(put_remote_path)s" % env)
elif 'mysql' in env.db_engine:
ret = run_or_dryrun("mysql -h %(db_host)s -u %(db_user)s -p'%(db_password)s' %(db_name)s < %(put_remote_path)s" % env)
else:
raise NotImplementedError('Unknown database type: %s' % env.db_engine)
print('ret:', ret)
site_summary[site] = ret
except KeyError as e:
site_summary[site] = 'Error: %s' % str(e)
pass
print('-'*80)
print('Site Summary:')
for site, ret in sorted(site_summary.items(), key=lambda o: o[0]):
print(site, ret)
def loaddata(path, site=None):
"""
Runs the Dango loaddata management command.
By default, runs on only the current site.
Pass site=all to run on all sites.
"""
render_remote_paths()
site = site or env.SITE
env._loaddata_path = path
for site, site_data in common.iter_sites(site=site, no_secure=True):
try:
set_db(site=site)
env.SITE = site
cmd = ('export SITE=%(SITE)s; export ROLE=%(ROLE)s; '
'cd %(shell_default_dir)s; '
'./manage loaddata %(_loaddata_path)s') % env
sudo_or_dryrun(cmd)
except KeyError:
pass
def install_ssl(site=ALL, dryrun=0):
apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(site=site, setter=set_apache_site_specifics):
print site
set_apache_site_specifics(site)
sudo('mkdir -p %(apache_ssl_dir)s' % env)
if env.apache_ssl:
for cert_type, local_cert_file, remote_cert_file in iter_certificates():
print '='*80
print 'Installing certificate %s...' % (remote_cert_file,)
if not int(dryrun):
put(
local_path=local_cert_file,
remote_path=remote_cert_file, use_sudo=True)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_ssl_dir)s' % env)
sudo('chmod -R %(apache_ssl_chmod)s %(apache_ssl_dir)s' % env)
def loaddata(path, site=None):
"""
Runs the Dango loaddata management command.
By default, runs on only the current site.
Pass site=all to run on all sites.
"""
render_remote_paths()
site = site or env.SITE
env._loaddata_path = path
for site, site_data in common.iter_sites(site=site, no_secure=True):
try:
set_db(site=site)
env.SITE = site
cmd = ('export SITE=%(SITE)s; export ROLE=%(ROLE)s; '
'cd %(shell_default_dir)s; '
'./manage loaddata %(_loaddata_path)s') % env
sudo_or_dryrun(cmd)
except KeyError:
pass
def install_ssl(site=ALL, dryrun=0):
apache_specifics = set_apache_specifics()
for site, site_data in common.iter_sites(site=site,
setter=set_apache_site_specifics):
print site
set_apache_site_specifics(site)
sudo('mkdir -p %(apache_ssl_dir)s' % env)
if env.apache_ssl:
for cert_type, local_cert_file, remote_cert_file in iter_certificates(
):
print '=' * 80
print 'Installing certificate %s...' % (remote_cert_file, )
if not int(dryrun):
put(local_path=local_cert_file,
remote_path=remote_cert_file,
use_sudo=True)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_ssl_dir)s' % env)
sudo('chmod -R %(apache_ssl_chmod)s %(apache_ssl_dir)s' % env)
def deploy(site=None, dryrun=0):
"""
Writes entire crontab to the host.
"""
cron_crontabs = []
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print 'site:',site
print 'cron_crontabs_selected:',env.cron_crontabs_selected
for selected_crontab in env.cron_crontabs_selected:
for line in env.cron_crontabs_available.get(selected_crontab, []):
cron_crontabs.append(line % env)
if not cron_crontabs:
return
cron_crontabs = env.cron_crontab_headers + cron_crontabs
cron_crontabs.append('\n')
env.cron_crontabs_rendered = '\n'.join(cron_crontabs)
fn = common.write_to_file(content=env.cron_crontabs_rendered)
if not int(dryrun):
put(local_path=fn)
sudo('crontab -u %(cron_user)s %(put_remote_path)s' % env)
def deploy(site=None, dryrun=0):
"""
Writes entire crontab to the host.
"""
cron_crontabs = []
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print 'site:', site
print 'cron_crontabs_selected:', env.cron_crontabs_selected
for selected_crontab in env.cron_crontabs_selected:
for line in env.cron_crontabs_available.get(selected_crontab, []):
cron_crontabs.append(line % env)
if not cron_crontabs:
return
cron_crontabs = env.cron_crontab_headers + cron_crontabs
cron_crontabs.append('\n')
env.cron_crontabs_rendered = '\n'.join(cron_crontabs)
fn = common.write_to_file(content=env.cron_crontabs_rendered)
if not int(dryrun):
put(local_path=fn)
sudo('crontab -u %(cron_user)s %(put_remote_path)s' % env)
def configure(site=None, full=0, dryrun=0):
"""
Installs and configures RabbitMQ.
"""
full = int(full)
dryrun = int(dryrun)
from burlap import package
# assert env.rabbitmq_erlang_cookie
if full:
package.install_required(type=package.common.SYSTEM, service=RABBITMQ)
#render_paths()
params = set() # [(user,vhost)]
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print '!'*80
print site
_settings = common.get_settings(site=site)
#print '_settings:',_settings
if not _settings:
continue
print 'RabbitMQ:',_settings.BROKER_USER, _settings.BROKER_VHOST
params.add((_settings.BROKER_USER, _settings.BROKER_VHOST))
for user, vhost in params:
env.rabbitmq_broker_user = user
env.rabbitmq_broker_vhost = vhost
with settings(warn_only=True):
cmd = 'rabbitmqctl add_vhost %(rabbitmq_broker_vhost)s' % env
print cmd
if not dryrun:
sudo(cmd)
cmd = 'rabbitmqctl set_permissions -p %(rabbitmq_broker_vhost)s %(rabbitmq_broker_user)s ".*" ".*" ".*"' % env
print cmd
if not dryrun:
sudo(cmd)
def configure(site=None, full=0, dryrun=0):
"""
Installs and configures RabbitMQ.
"""
full = int(full)
dryrun = int(dryrun)
from burlap import package
# assert env.rabbitmq_erlang_cookie
if full:
package.install_required(type=package.common.SYSTEM, service=RABBITMQ)
#render_paths()
params = set() # [(user,vhost)]
for site, site_data in common.iter_sites(site=site, renderer=render_paths):
print '!' * 80
print site
_settings = common.get_settings(site=site)
#print '_settings:',_settings
if not _settings:
continue
print 'RabbitMQ:', _settings.BROKER_USER, _settings.BROKER_VHOST
params.add((_settings.BROKER_USER, _settings.BROKER_VHOST))
for user, vhost in params:
env.rabbitmq_broker_user = user
env.rabbitmq_broker_vhost = vhost
with settings(warn_only=True):
cmd = 'rabbitmqctl add_vhost %(rabbitmq_broker_vhost)s' % env
print cmd
if not dryrun:
sudo(cmd)
cmd = 'rabbitmqctl set_permissions -p %(rabbitmq_broker_vhost)s %(rabbitmq_broker_user)s ".*" ".*" ".*"' % env
print cmd
if not dryrun:
sudo(cmd)
def restart(site=common.ALL):
for site, site_data in common.iter_sites(site=site, renderer=lambda: set_db(name="default")):
print site
# set_db(name=name, site=site)
if "postgres" in env.db_engine:
sudo("service postgresql restart; sleep 3")
def configure(full=1, site=ALL, delete_old=0):
"""
Configures Apache to host one or more websites.
"""
from burlap import service
print 'Configuring Apache...'
apache_specifics = set_apache_specifics()
if int(delete_old):
# Delete all existing enabled and available sites.
sudo('rm -f %(apache_sites_available)s/*' % env)
sudo('rm -f %(apache_sites_enabled)s/*' % env)
for site, site_data in common.iter_sites(site=site,
setter=set_apache_site_specifics):
#print '-'*80
print site
#continue
print 'env.apache_ssl_domain:', env.apache_ssl_domain
print 'env.apache_ssl_domain_template:', env.apache_ssl_domain_template
fn = common.render_to_file('django.template.wsgi')
put(local_path=fn, remote_path=env.apache_django_wsgi, use_sudo=True)
if env.apache_ssl:
env.apache_ssl_certificates = list(iter_certificates())
fn = common.render_to_file('apache_site.template.conf')
env.apache_site_conf = site + '.conf'
env.apache_site_conf_fqfn = os.path.join(env.apache_sites_available,
env.apache_site_conf)
put(local_path=fn,
remote_path=env.apache_site_conf_fqfn,
use_sudo=True)
sudo('a2ensite %(apache_site_conf)s' % env)
#return
if service.is_selected(APACHE2_MODEVASIVE):
configure_modevasive()
if service.is_selected(APACHE2_MODSECURITY):
configure_modsecurity()
for mod_enabled in env.apache_mods_enabled:
env.apache_mod_enabled = mod_enabled
sudo('a2enmod %(apache_mod_enabled)s' % env)
if int(full):
# Write master Apache configuration file.
fn = common.render_to_file('apache_httpd.template.conf')
put(local_path=fn, remote_path=env.apache_conf, use_sudo=True)
# Write Apache listening ports configuration.
fn = common.render_to_file('apache_ports.template.conf')
put(local_path=fn, remote_path=env.apache_ports, use_sudo=True)
#sudo('mkdir -p %(apache_app_log_dir)s' % env)
#sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_app_log_dir)s' % env)
# sudo('mkdir -p %(apache_log_dir)s' % env)
# sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_log_dir)s' % env)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_root)s' % env)
def configure_site(self, full=1, site=None, delete_old=0):
"""
Configures Apache to host one or more websites.
"""
from burlap.common import get_current_hostname, iter_sites
from burlap import service
print('Configuring Apache...', file=sys.stderr)
verbose = self.verbose
site = site or self.genv.SITE
apache_specifics = self.set_apache_specifics()
hostname = get_current_hostname()
target_sites = self.genv.available_sites_by_host.get(hostname, None)
if int(delete_old):
# Delete all existing enabled and available sites.
cmd = 'rm -f %(apache_sites_available)s/*' % self.genv
self.sudo_or_dryrun(cmd)
cmd = 'rm -f %(apache_sites_enabled)s/*' % self.genv
self.sudo_or_dryrun(cmd)
for site, site_data in iter_sites(site=site, setter=self.set_apache_site_specifics):
if self.verbose:
print('-'*80, file=sys.stderr)
print('Site:',site, file=sys.stderr)
print('-'*80, file=sys.stderr)
# Only load site configurations that are allowed for this host.
if target_sites is None:
pass
else:
assert isinstance(target_sites, (tuple, list))
if site not in target_sites:
continue
if self.verbose:
print('env.apache_ssl_domain:', self.genv.apache_ssl_domain, file=sys.stderr)
print('env.apache_ssl_domain_template:', self.genv.apache_ssl_domain_template, file=sys.stderr)
print('env.django_settings_module:', self.genv.django_settings_module, file=sys.stderr)
# raw_input('enter')
fn = self.render_to_file('django/django.template.wsgi', verbose=verbose)
remote_dir = os.path.split(self.genv.apache_django_wsgi)[0]
cmd = 'mkdir -p %s' % remote_dir
self.sudo_or_dryrun(cmd)
if self.verbose:
print(fn, file=sys.stderr)
self.put_or_dryrun(local_path=fn, remote_path=self.genv.apache_django_wsgi, use_sudo=True)
if self.genv.apache_ssl:
self.genv.apache_ssl_certificates = list(self.iter_certificates())
fn = self.render_to_file(self.env.site_template, verbose=verbose)
self.genv.apache_site_conf = site+'.conf'
self.genv.apache_site_conf_fqfn = os.path.join(self.genv.apache_sites_available, self.genv.apache_site_conf)
self.put_or_dryrun(local_path=fn, remote_path=self.genv.apache_site_conf_fqfn, use_sudo=True)
cmd = 'a2ensite %(apache_site_conf)s' % self.genv
self.sudo_or_dryrun(cmd)
# if service.is_selected(APACHE2_MODEVASIVE):
# configure_modevasive()
#
# if service.is_selected(APACHE2_MODSECURITY):
# configure_modsecurity()
for mod_enabled in self.genv.apache_mods_enabled:
self.genv.apache_mod_enabled = mod_enabled
cmd = 'a2enmod %(apache_mod_enabled)s' % self.genv
with settings(warn_only=True):
self.sudo_or_dryrun(cmd)
if int(full):
# Write master Apache configuration file.
fn = self.render_to_file('apache/apache_httpd.template.conf', verbose=verbose)
self.put_or_dryrun(local_path=fn, remote_path=self.genv.apache_conf, use_sudo=True)
# Write Apache listening ports configuration.
fn = self.render_to_file('apache/apache_ports.template.conf', verbose=verbose)
self.put_or_dryrun(local_path=fn, remote_path=self.genv.apache_ports, use_sudo=True)
#sudo_or_dryrun('mkdir -p %(apache_app_log_dir)s' % self.genv)
#sudo_or_dryrun('chown -R %(apache_user)s:%(apache_group)s %(apache_app_log_dir)s' % self.genv)
# self.sudo_or_dryrun('mkdir -p %(apache_log_dir)s' % self.genv)
# self.sudo_or_dryrun('chown -R %(apache_user)s:%(apache_group)s %(apache_log_dir)s' % self.genv)
cmd = 'chown -R %(apache_user)s:%(apache_group)s %(apache_root)s' % self.genv
self.sudo_or_dryrun(cmd)
def configure_site(self, full=1, site=None, delete_old=0):
"""
Configures Apache to host one or more websites.
"""
from burlap.common import get_current_hostname, iter_sites
from burlap import service
print('Configuring Apache...', file=sys.stderr)
verbose = self.verbose
site = site or self.genv.SITE
apache_specifics = self.set_apache_specifics()
hostname = get_current_hostname()
target_sites = self.genv.available_sites_by_host.get(hostname, None)
if int(delete_old):
# Delete all existing enabled and available sites.
cmd = 'rm -f %(apache_sites_available)s/*' % self.genv
self.sudo_or_dryrun(cmd)
cmd = 'rm -f %(apache_sites_enabled)s/*' % self.genv
self.sudo_or_dryrun(cmd)
for site, site_data in iter_sites(
site=site, setter=self.set_apache_site_specifics):
if self.verbose:
print('-' * 80, file=sys.stderr)
print('Site:', site, file=sys.stderr)
print('-' * 80, file=sys.stderr)
# Only load site configurations that are allowed for this host.
if target_sites is None:
pass
else:
assert isinstance(target_sites, (tuple, list))
if site not in target_sites:
continue
if self.verbose:
print('env.apache_ssl_domain:',
self.genv.apache_ssl_domain,
file=sys.stderr)
print('env.apache_ssl_domain_template:',
self.genv.apache_ssl_domain_template,
file=sys.stderr)
print('env.django_settings_module:',
self.genv.django_settings_module,
file=sys.stderr)
# raw_input('enter')
fn = self.render_to_file('django/django.template.wsgi',
verbose=verbose)
remote_dir = os.path.split(self.genv.apache_django_wsgi)[0]
cmd = 'mkdir -p %s' % remote_dir
self.sudo_or_dryrun(cmd)
if self.verbose:
print(fn, file=sys.stderr)
self.put_or_dryrun(local_path=fn,
remote_path=self.genv.apache_django_wsgi,
use_sudo=True)
if self.genv.apache_ssl:
self.genv.apache_ssl_certificates = list(
self.iter_certificates())
fn = self.render_to_file(self.env.site_template, verbose=verbose)
self.genv.apache_site_conf = site + '.conf'
self.genv.apache_site_conf_fqfn = os.path.join(
self.genv.apache_sites_available, self.genv.apache_site_conf)
self.put_or_dryrun(local_path=fn,
remote_path=self.genv.apache_site_conf_fqfn,
use_sudo=True)
cmd = 'a2ensite %(apache_site_conf)s' % self.genv
self.sudo_or_dryrun(cmd)
# if service.is_selected(APACHE2_MODEVASIVE):
# configure_modevasive()
#
# if service.is_selected(APACHE2_MODSECURITY):
# configure_modsecurity()
for mod_enabled in self.genv.apache_mods_enabled:
self.genv.apache_mod_enabled = mod_enabled
cmd = 'a2enmod %(apache_mod_enabled)s' % self.genv
with settings(warn_only=True):
self.sudo_or_dryrun(cmd)
if int(full):
# Write master Apache configuration file.
fn = self.render_to_file('apache/apache_httpd.template.conf',
verbose=verbose)
self.put_or_dryrun(local_path=fn,
remote_path=self.genv.apache_conf,
use_sudo=True)
# Write Apache listening ports configuration.
fn = self.render_to_file('apache/apache_ports.template.conf',
verbose=verbose)
self.put_or_dryrun(local_path=fn,
remote_path=self.genv.apache_ports,
use_sudo=True)
#sudo_or_dryrun('mkdir -p %(apache_app_log_dir)s' % self.genv)
#sudo_or_dryrun('chown -R %(apache_user)s:%(apache_group)s %(apache_app_log_dir)s' % self.genv)
# self.sudo_or_dryrun('mkdir -p %(apache_log_dir)s' % self.genv)
# self.sudo_or_dryrun('chown -R %(apache_user)s:%(apache_group)s %(apache_log_dir)s' % self.genv)
cmd = 'chown -R %(apache_user)s:%(apache_group)s %(apache_root)s' % self.genv
self.sudo_or_dryrun(cmd)
def list_sites(site='all', *args, **kwargs):
from burlap.common import iter_sites
kwargs['site'] = site
for site, data in iter_sites(*args, **kwargs):
print(site)
