def migrate(app_name='', site=None, fake=0):
"""
Wrapper around Django's migrate command.
"""
print 'Migrating...'
set_site(site or env.SITE)
render_remote_paths()
env.db_migrate_fake = '--fake' if int(fake) else ''
if app_name:
env.db_app_name = app_name
run('export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s migrate %(db_app_name)s --noinput --delete-ghost-migrations %(db_migrate_fake)s -v 3 --traceback'
% env)
else:
# First migrate apps in a specific order if given.
for app_name in env.db_app_migration_order:
env.db_app_name = app_name
run('export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s migrate --noinput --delete-ghost-migrations %(db_migrate_fake)s %(db_app_name)s -v 3 --traceback'
% env)
# Then migrate everything else remaining.
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s migrate --noinput --delete-ghost-migrations %(db_migrate_fake)s -v 3 --traceback' % env
#print cmd
run(cmd)
def render_paths():
env.pip_path_versioned = env.pip_path % env
render_remote_paths()
if env.pip_virtual_env_dir_template:
env.pip_virtual_env_dir = env.pip_virtual_env_dir_template % env
if env.is_local:
env.pip_virtual_env_dir = os.path.abspath(env.pip_virtual_env_dir)
def migrate(app_name="", site=None, fake=0):
"""
Wrapper around Django's migrate command.
"""
print "Migrating..."
set_site(site or env.SITE)
render_remote_paths()
env.db_migrate_fake = "--fake" if int(fake) else ""
if app_name:
env.db_app_name = app_name
run(
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s migrate %(db_app_name)s --noinput --delete-ghost-migrations %(db_migrate_fake)s -v 3 --traceback"
% env
)
else:
# First migrate apps in a specific order if given.
for app_name in env.db_app_migration_order:
env.db_app_name = app_name
run(
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s migrate --noinput --delete-ghost-migrations %(db_migrate_fake)s %(db_app_name)s -v 3 --traceback"
% env
)
# Then migrate everything else remaining.
cmd = (
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s migrate --noinput --delete-ghost-migrations %(db_migrate_fake)s -v 3 --traceback"
% env
)
# print cmd
run(cmd)
def createsuperuser(username='******', email=None, password=None, site=None):
"""
Runs the Django createsuperuser management command.
"""
set_site(site)
render_remote_paths()
env.db_createsuperuser_username = username
env.db_createsuperuser_email = email or username
run('export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s createsuperuser --username=%(db_createsuperuser_username)s --email=%(db_createsuperuser_email)s'
% env)
def syncdb(site=None, all=0, dryrun=0):
"""
Wrapper around Django's syncdb command.
"""
set_site(site)
render_remote_paths()
env.db_syncdb_all_flag = '--all' if int(all) else ''
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s syncdb --noinput %(db_syncdb_all_flag)s -v 3 --traceback' % env
print 'cmd:', cmd
if not int(dryrun):
run(cmd)
def createsuperuser(username="******", email=None, password=None, site=None):
"""
Runs the Django createsuperuser management command.
"""
set_site(site)
render_remote_paths()
env.db_createsuperuser_username = username
env.db_createsuperuser_email = email or username
run(
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s createsuperuser --username=%(db_createsuperuser_username)s --email=%(db_createsuperuser_email)s"
% env
)
def install(package='', clean=0, all=0, upgrade=1):
"""
Installs the local cache of pip packages.
"""
print 'Installing pip requirements...'
assert env[ROLE]
require('is_local')
# Delete any pre-existing environment.
if int(clean):
clean_virtualenv()
render_remote_paths()
if env.pip_virtual_env_dir_template:
env.pip_virtual_env_dir = env.pip_virtual_env_dir_template % env
env.pip_local_cache_dir = env.pip_local_cache_dir_template % env
env.pip_path_versioned = env.pip_path % env
if env.is_local:
env.pip_cache_dir = os.path.abspath(env.pip_local_cache_dir % env)
else:
env.pip_cache_dir = env.pip_remote_cache_dir % env
env.pip_key_filename = os.path.abspath(env.key_filename)
local('rsync -avz --progress --rsh "ssh -i %(pip_key_filename)s" %(pip_local_cache_dir)s/* %(user)s@%(host_string)s:%(pip_remote_cache_dir)s' % env)
env.pip_upgrade_flag = ''
if int(upgrade):
env.pip_upgrade_flag = ' -U '
if int(all):
packages = list(iter_pip_requirements())
elif package:
packages = [package]
else:
packages = [k for k,v in check()]
env.pip_build_dir = tempfile.mkdtemp()
for package in packages:
env.pip_package = package
if env.is_local:
run(env.pip_install_command % env)
else:
sudo(env.pip_install_command % env)
if not env.is_local:
sudo('chown -R %(pip_user)s:%(pip_group)s %(remote_app_dir)s' % env)
sudo('chmod -R %(pip_chmod)s %(remote_app_dir)s' % env)
def syncdb(site=None, all=0, dryrun=0):
"""
Wrapper around Django's syncdb command.
"""
set_site(site)
render_remote_paths()
env.db_syncdb_all_flag = "--all" if int(all) else ""
cmd = (
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s syncdb --noinput %(db_syncdb_all_flag)s -v 3 --traceback"
% env
)
print "cmd:", cmd
if not int(dryrun):
run(cmd)
def sync_media(sync_set=None, dryrun=0):
"""
Uploads select media to an Apache accessible directory.
"""
apache_specifics = set_apache_specifics()
common.render_remote_paths()
site_data = env.sites[env.SITE]
env.update(site_data)
sync_sets = env.apache_sync_sets
if sync_set:
sync_sets = [sync_set]
for sync_set in sync_sets:
for paths in env.apache_sync_sets[sync_set]:
print 'paths:', paths
env.apache_sync_local_path = os.path.abspath(paths['local_path'] %
env)
if paths['local_path'].endswith(
'/') and not env.apache_sync_local_path.endswith('/'):
env.apache_sync_local_path += '/'
env.apache_sync_remote_path = paths['remote_path'] % env
print 'Syncing %s to %s...' % (env.apache_sync_local_path,
env.apache_sync_remote_path)
env.apache_tmp_chmod = paths.get('chmod', env.apache_chmod)
#with settings(warn_only=True):
sudo('mkdir -p %(apache_sync_remote_path)s' % env,
user=env.apache_user)
sudo('chmod -R %(apache_tmp_chmod)s %(apache_sync_remote_path)s' %
env,
user=env.apache_user)
cmd = (
'rsync -rvz --progress --recursive --no-p --no-g --rsh "ssh -i %(key_filename)s" %(apache_sync_local_path)s %(user)s@%(host_string)s:%(apache_sync_remote_path)s'
) % env
# print '!'*80
# print cmd
if not int(dryrun):
local(cmd)
sudo(
'chown -R %(apache_user)s:%(apache_group)s %(apache_sync_remote_path)s'
% env)
def install_fixtures(name, site=None):
"""
Installs a set of Django fixtures.
"""
set_site(site)
render_remote_paths()
fixtures_paths = env.db_fixture_sets.get(name, [])
for fixture_path in fixtures_paths:
env.db_fq_fixture_path = os.path.join(env.remote_app_src_package_dir, fixture_path)
print "Loading %s..." % (env.db_fq_fixture_path,)
if not env.is_local and not files.exists(env.db_fq_fixture_path):
put(local_path=env.db_fq_fixture_path, remote_path="/tmp/data.json", use_sudo=True)
env.db_fq_fixture_path = env.put_remote_path
cmd = (
"export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s loaddata %(db_fq_fixture_path)s"
% env
)
print cmd
run(cmd)
def install_fixtures(name, site=None):
"""
Installs a set of Django fixtures.
"""
set_site(site)
render_remote_paths()
fixtures_paths = env.db_fixture_sets.get(name, [])
for fixture_path in fixtures_paths:
env.db_fq_fixture_path = os.path.join(env.remote_app_src_package_dir,
fixture_path)
print 'Loading %s...' % (env.db_fq_fixture_path, )
if not env.is_local and not files.exists(env.db_fq_fixture_path):
put(local_path=env.db_fq_fixture_path,
remote_path='/tmp/data.json',
use_sudo=True)
env.db_fq_fixture_path = env.put_remote_path
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s loaddata %(db_fq_fixture_path)s' % env
print cmd
run(cmd)
def sync_media(sync_set=None, dryrun=0):
"""
Uploads select media to an Apache accessible directory.
"""
apache_specifics = set_apache_specifics()
common.render_remote_paths()
site_data = env.sites[env.SITE]
env.update(site_data)
sync_sets = env.apache_sync_sets
if sync_set:
sync_sets = [sync_set]
for sync_set in sync_sets:
for paths in env.apache_sync_sets[sync_set]:
print 'paths:',paths
env.apache_sync_local_path = os.path.abspath(paths['local_path'] % env)
if paths['local_path'].endswith('/') and not env.apache_sync_local_path.endswith('/'):
env.apache_sync_local_path += '/'
env.apache_sync_remote_path = paths['remote_path'] % env
print 'Syncing %s to %s...' % (env.apache_sync_local_path, env.apache_sync_remote_path)
env.apache_tmp_chmod = paths.get('chmod', env.apache_chmod)
#with settings(warn_only=True):
sudo('mkdir -p %(apache_sync_remote_path)s' % env, user=env.apache_user)
sudo('chmod -R %(apache_tmp_chmod)s %(apache_sync_remote_path)s' % env, user=env.apache_user)
cmd = ('rsync -rvz --progress --recursive --no-p --no-g --rsh "ssh -i %(key_filename)s" %(apache_sync_local_path)s %(user)s@%(host_string)s:%(apache_sync_remote_path)s') % env
# print '!'*80
# print cmd
if not int(dryrun):
local(cmd)
sudo('chown -R %(apache_user)s:%(apache_group)s %(apache_sync_remote_path)s' % env)
def sync(sync_set, dryrun=0, auto_invalidate=True):
"""
Uploads media to an Amazon S3 bucket using s3sync.
Requires the s3sync gem: sudo gem install s3sync
"""
env.dryrun = int(dryrun)
_settings = get_settings()
for k in _settings.__dict__.iterkeys():
if k.startswith('AWS_'):
env[k] = _settings.__dict__[k]
#local('which s3sync')
#print 'AWS_STATIC_BUCKET_NAME:',_settings.AWS_STATIC_BUCKET_NAME
common.render_remote_paths()
site_data = env.sites[env.SITE]
env.update(site_data)
rets = []
for paths in env.s3_sync_sets[sync_set]:
is_local = paths.get('is_local', True)
local_path = paths['local_path'] % env
remote_path = paths['remote_path']
local_path = local_path % env
if is_local:
local('which s3sync') #, capture=True)
env.s3_local_path = os.path.abspath(local_path)
else:
run('which s3sync')
env.s3_local_path = local_path
if local_path.endswith('/') and not env.s3_local_path.endswith('/'):
env.s3_local_path = env.s3_local_path + '/'
env.s3_remote_path = remote_path % env
print 'Syncing %s to %s...' % (env.s3_local_path, env.s3_remote_path)
cmd = ('export AWS_ACCESS_KEY_ID=%(AWS_ACCESS_KEY_ID)s; '\
'export AWS_SECRET_ACCESS_KEY=%(AWS_SECRET_ACCESS_KEY)s; '\
's3sync --recursive --verbose --progress --public-read '\
'%(s3_local_path)s %(s3_remote_path)s') % env
print cmd
if not int(dryrun):
if is_local:
rets.append(local(cmd, capture=True)) # can't see progress
#rets.append(run(cmd))
else:
rets.append(run(cmd))
if auto_invalidate:
for ret in rets:
print 's3sync:', ret
paths = re.findall('(?:Create|Update)\s+node\s+([^\n]+)',
ret,
flags=re.DOTALL | re.MULTILINE | re.IGNORECASE)
print 'paths:', paths
#TODO:handle more than 1000 paths?
invalidate(*paths)
def create(drop=0, name=None, dryrun=0, site=None, post_process=0):
"""
Creates the target database
"""
assert env[ROLE]
dryrun = int(dryrun)
render_remote_paths()
require('app_name')
env.db_drop_flag = '--drop' if int(drop) else ''
set_db(name=name, site=site)
print 'site:', env[SITE]
print 'role:', env[ROLE]
env.dryrun = int(dryrun)
if 'postgres' in env.db_engine:
env.src_dir = os.path.abspath(env.src_dir)
# This assumes the django-extensions app is installed, which
# provides the convenient sqlcreate command.
if env.is_local:
env.db_src_dir = env.src_dir
else:
env.db_src_dir = env.remote_app_src_dir
cmd = 'cd %(db_src_dir)s; export SITE=%(SITE)s; export ROLE=%(ROLE)s; %(django_manage)s sqlcreate --router=default %(db_drop_flag)s | psql --user=%(db_postgresql_postgres_user)s --no-password' % env
print cmd
if not dryrun:
sudo(cmd)
#run('psql --user=postgres -d %(db_name)s -c "REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM %(db_user)s_ro CASCADE; DROP ROLE IF EXISTS %(db_user)s_ro; DROP USER IF EXISTS %(db_user)s_ro; CREATE USER %(db_user)s_ro WITH PASSWORD \'readonly\'; GRANT SELECT ON ALL TABLES IN SCHEMA public TO %(db_user)s_ro;"')
with settings(warn_only=True):
cmd = 'createlang -U postgres plpgsql %(db_name)s' % env
print cmd
if not dryrun:
sudo(cmd)
elif 'mysql' in env.db_engine:
if int(drop):
cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute='DROP DATABASE IF EXISTS %(db_name)s'" % env
print cmd
if not int(dryrun):
sudo(cmd)
cmd = "mysqladmin -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' create %(db_name)s" % env
print cmd
if not int(dryrun):
sudo(cmd)
# Create user.
cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"GRANT USAGE ON *.* TO %(db_user)s@'%%'; DROP USER %(db_user)s@'%%';\"" % env
print cmd
if not int(dryrun):
run(cmd)
#cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"CREATE USER %(db_user)s@%(db_host)s IDENTIFIED BY '%(db_password)s';\"" % env
#cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"GRANT ALL PRIVILEGES ON %(db_name)s.* TO %(db_user)s@%(db_host)s IDENTIFIED BY '%(db_password)s';\"" % env
cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"GRANT ALL PRIVILEGES ON %(db_name)s.* TO %(db_user)s@'%%' IDENTIFIED BY '%(db_password)s';\"" % env
print cmd
if not int(dryrun):
run(cmd)
# Let the primary login do so from everywhere.
# cmd = 'mysql -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute="USE mysql; GRANT ALL ON %(db_name)s.* to %(db_user)s@\'%\' IDENTIFIED BY \'%(db_password)s\'; FLUSH PRIVILEGES;"'
# sudo(cmd)
else:
raise NotImplemented
if not env.dryrun and int(post_process):
post_create(name=name, dryrun=dryrun, site=site)
def render_paths():
common.render_remote_paths()
if env.rabbitmq_erlang_cookie_template:
env.rabbitmq_erlang_cookie = env.rabbitmq_erlang_cookie_template % env
def create(drop=0, name=None, dryrun=0, site=None, post_process=0):
"""
Creates the target database
"""
assert env[ROLE]
dryrun = int(dryrun)
render_remote_paths()
require("app_name")
env.db_drop_flag = "--drop" if int(drop) else ""
set_db(name=name, site=site)
print "site:", env[SITE]
print "role:", env[ROLE]
env.dryrun = int(dryrun)
if "postgres" in env.db_engine:
env.src_dir = os.path.abspath(env.src_dir)
# This assumes the django-extensions app is installed, which
# provides the convenient sqlcreate command.
if env.is_local:
env.db_src_dir = env.src_dir
else:
env.db_src_dir = env.remote_app_src_dir
cmd = (
"cd %(db_src_dir)s; export SITE=%(SITE)s; export ROLE=%(ROLE)s; %(django_manage)s sqlcreate --router=default %(db_drop_flag)s | psql --user=%(db_postgresql_postgres_user)s --no-password"
% env
)
print cmd
if not dryrun:
sudo(cmd)
# run('psql --user=postgres -d %(db_name)s -c "REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM %(db_user)s_ro CASCADE; DROP ROLE IF EXISTS %(db_user)s_ro; DROP USER IF EXISTS %(db_user)s_ro; CREATE USER %(db_user)s_ro WITH PASSWORD \'readonly\'; GRANT SELECT ON ALL TABLES IN SCHEMA public TO %(db_user)s_ro;"')
with settings(warn_only=True):
cmd = "createlang -U postgres plpgsql %(db_name)s" % env
print cmd
if not dryrun:
sudo(cmd)
elif "mysql" in env.db_engine:
if int(drop):
cmd = (
"mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute='DROP DATABASE IF EXISTS %(db_name)s'"
% env
)
print cmd
if not int(dryrun):
sudo(cmd)
cmd = "mysqladmin -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' create %(db_name)s" % env
print cmd
if not int(dryrun):
sudo(cmd)
# Create user.
cmd = (
"mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"GRANT USAGE ON *.* TO %(db_user)s@'%%'; DROP USER %(db_user)s@'%%';\""
% env
)
print cmd
if not int(dryrun):
run(cmd)
# cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"CREATE USER %(db_user)s@%(db_host)s IDENTIFIED BY '%(db_password)s';\"" % env
# cmd = "mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"GRANT ALL PRIVILEGES ON %(db_name)s.* TO %(db_user)s@%(db_host)s IDENTIFIED BY '%(db_password)s';\"" % env
cmd = (
"mysql -v -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute=\"GRANT ALL PRIVILEGES ON %(db_name)s.* TO %(db_user)s@'%%' IDENTIFIED BY '%(db_password)s';\""
% env
)
print cmd
if not int(dryrun):
run(cmd)
# Let the primary login do so from everywhere.
# cmd = 'mysql -h %(db_host)s -u %(db_root_user)s -p'%(db_root_password)s' --execute="USE mysql; GRANT ALL ON %(db_name)s.* to %(db_user)s@\'%\' IDENTIFIED BY \'%(db_password)s\'; FLUSH PRIVILEGES;"'
# sudo(cmd)
else:
raise NotImplemented
if not env.dryrun and int(post_process):
post_create(name=name, dryrun=dryrun, site=site)
def upgrade_pip():
render_remote_paths()
if env.pip_virtual_env_dir_template:
env.pip_virtual_env_dir = env.pip_virtual_env_dir_template % env
run(". %(pip_virtual_env_dir)s/bin/activate; pip install --upgrade setuptools" % env)
run(". %(pip_virtual_env_dir)s/bin/activate; pip install --upgrade distribute" % env)
def render_paths():
common.render_remote_paths()
def render_paths():
common.render_remote_paths()
if env.rabbitmq_erlang_cookie_template:
env.rabbitmq_erlang_cookie = env.rabbitmq_erlang_cookie_template % env
def sync(sync_set, dryrun=0, auto_invalidate=True):
"""
Uploads media to an Amazon S3 bucket using s3sync.
Requires the s3sync gem: sudo gem install s3sync
"""
env.dryrun = int(dryrun)
_settings = get_settings()
for k in _settings.__dict__.iterkeys():
if k.startswith("AWS_"):
env[k] = _settings.__dict__[k]
# local('which s3sync')
# print 'AWS_STATIC_BUCKET_NAME:',_settings.AWS_STATIC_BUCKET_NAME
common.render_remote_paths()
site_data = env.sites[env.SITE]
env.update(site_data)
rets = []
for paths in env.s3_sync_sets[sync_set]:
is_local = paths.get("is_local", True)
local_path = paths["local_path"] % env
remote_path = paths["remote_path"]
local_path = local_path % env
if is_local:
local("which s3sync") # , capture=True)
env.s3_local_path = os.path.abspath(local_path)
else:
run("which s3sync")
env.s3_local_path = local_path
if local_path.endswith("/") and not env.s3_local_path.endswith("/"):
env.s3_local_path = env.s3_local_path + "/"
env.s3_remote_path = remote_path % env
print "Syncing %s to %s..." % (env.s3_local_path, env.s3_remote_path)
cmd = (
"export AWS_ACCESS_KEY_ID=%(AWS_ACCESS_KEY_ID)s; "
"export AWS_SECRET_ACCESS_KEY=%(AWS_SECRET_ACCESS_KEY)s; "
"s3sync --recursive --verbose --progress --public-read "
"%(s3_local_path)s %(s3_remote_path)s"
) % env
print cmd
if not int(dryrun):
if is_local:
rets.append(local(cmd, capture=True)) # can't see progress
# rets.append(run(cmd))
else:
rets.append(run(cmd))
if auto_invalidate:
for ret in rets:
print "s3sync:", ret
paths = re.findall(
"(?:Create|Update)\s+node\s+([^\n]+)", ret, flags=re.DOTALL | re.MULTILINE | re.IGNORECASE
)
print "paths:", paths
# TODO:handle more than 1000 paths?
invalidate(*paths)