def migrate(app='', migration='', site=None, fake=0, ignore_errors=0, skip_databases=None, database=None, migrate_apps='', delete_ghosts=1):
"""
Runs the standard South migrate command for one or more sites.
"""
ignore_errors = int(ignore_errors)
delete_ghosts = int(delete_ghosts)
skip_databases = (skip_databases or '')
if isinstance(skip_databases, basestring):
skip_databases = [_.strip() for _ in skip_databases.split(',') if _.strip()]
migrate_apps = migrate_apps or ''
migrate_apps = [
_.strip().split('.')[-1]
for _ in migrate_apps.strip().split(',')
if _.strip()
]
if app:
migrate_apps.append(app)
render_remote_paths()
_env = type(env)(env)
_env.django_migrate_migration = migration or ''
_env.django_migrate_fake_str = '--fake' if int(fake) else ''
_env.django_migrate_database = '--database=%s' % database if database else ''
_env.delete_ghosts = '--delete-ghost-migrations' if delete_ghosts else ''
for site, site_data in iter_unique_databases(site=site):
print('-'*80, file=sys.stderr)
print('site:', site, file=sys.stderr)
if env.available_sites_by_host:
hostname = common.get_current_hostname()
sites_on_host = env.available_sites_by_host.get(hostname, [])
if sites_on_host and site not in sites_on_host:
print('skipping site:', site, sites_on_host, file=sys.stderr)
continue
print('migrate_apps:', migrate_apps, file=sys.stderr)
if migrate_apps:
_env.django_migrate_app = ' '.join(migrate_apps)
else:
_env.django_migrate_app = ''
_env.SITE = site
cmd = (
'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; '
'%(django_manage)s migrate --noinput --traceback %(django_migrate_database)s %(delete_ghosts)s %(django_migrate_app)s %(django_migrate_migration)s '
'%(django_migrate_fake_str)s'
) % _env
cmd = cmd.strip()
with settings(warn_only=ignore_errors):
run_or_dryrun(cmd)
def createsuperuser(username='******', email=None, password=None, site=None):
"""
Runs the Django createsuperuser management command.
"""
from burlap.dj import render_remote_paths
set_site(site)
render_remote_paths()
env.db_createsuperuser_username = username
env.db_createsuperuser_email = email or username
run_or_dryrun('export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s createsuperuser --username=%(db_createsuperuser_username)s --email=%(db_createsuperuser_email)s' % env)
def sync_media(self, sync_set=None, clean=0, iter_local_paths=0):
"""
Uploads select media to an Apache accessible directory.
"""
apache.get_apache_settings()
from burlap.dj import render_remote_paths
apache_specifics = apache.set_apache_specifics()
render_remote_paths()
clean = int(clean)
print('Getting site data for %s...' % self.genv.SITE)
site_data = self.genv.sites[self.genv.SITE]
self.genv.update(site_data)
sync_sets = self.genv.apache_sync_sets
if sync_set:
sync_sets = [sync_set]
ret_paths = []
for sync_set in sync_sets:
for paths in self.genv.apache_sync_sets[sync_set]:
#print 'paths:',paths
self.genv.apache_sync_local_path = os.path.abspath(paths['local_path'] % self.genv)
if paths['local_path'].endswith('/') and not self.genv.apache_sync_local_path.endswith('/'):
self.genv.apache_sync_local_path += '/'
if iter_local_paths:
ret_paths.append(self.genv.apache_sync_local_path)
continue
self.genv.apache_sync_remote_path = paths['remote_path'] % self.genv
if clean:
self.sudo_or_dryrun('rm -Rf %(apache_sync_remote_path)s' % self.genv)
print('Syncing %s to %s...' % (self.genv.apache_sync_local_path, self.genv.apache_sync_remote_path))
self.genv.apache_tmp_chmod = paths.get('chmod', self.genv.apache_chmod)
#with settings(warn_only=True):
self.sudo_or_dryrun('mkdir -p %(apache_sync_remote_path)s' % self.genv, user=self.genv.apache_user)
self.sudo_or_dryrun('chmod -R %(apache_tmp_chmod)s %(apache_sync_remote_path)s' % self.genv, user=self.genv.apache_user)
cmd = ('rsync -rvz --progress --recursive --no-p --no-g --rsh "ssh -o StrictHostKeyChecking=no -i %(key_filename)s" %(apache_sync_local_path)s %(user)s@%(host_string)s:%(apache_sync_remote_path)s') % self.genv
self.local_or_dryrun(cmd)
self.sudo_or_dryrun('chown -R %(apache_user)s:%(apache_group)s %(apache_sync_remote_path)s' % self.genv)
if iter_local_paths:
return ret_paths
def uninstall(package):
from burlap.dj import render_remote_paths
render_remote_paths()
if env.pip_virtual_env_dir_template:
env.pip_virtual_env_dir = env.pip_virtual_env_dir_template % env
env.pip_local_cache_dir = env.pip_local_cache_dir_template % env
env.pip_package = package
if env.is_local:
run_or_dryrun(env.pip_uninstall_command % env)
else:
sudo_or_dryrun(env.pip_uninstall_command % env)
def update_install(clean=0,
pip_requirements_fn=None,
virtualenv_dir=None,
user=None,
group=None,
perms=None):
try:
from burlap.dj import render_remote_paths
except ImportError:
render_remote_paths = None
_env = type(env)(env)
pip_requirements_fn = pip_requirements_fn or env.pip_requirements_fn
bootstrap(force=clean)
init(clean=clean, virtualenv_dir=virtualenv_dir, check_permissions=False)
req_fn = find_template(pip_requirements_fn)
assert req_fn, 'Could not find file: %s' % pip_requirements_fn
_env.pip_remote_requirements_fn = '/tmp/pip-requirements.txt'
put_or_dryrun(
local_path=req_fn,
remote_path=_env.pip_remote_requirements_fn,
)
if render_remote_paths:
render_remote_paths()
if virtualenv_dir:
_env.virtualenv_dir = virtualenv_dir
elif _env.pip_virtual_env_dir_template:
_env.virtualenv_dir = _env.pip_virtual_env_dir_template % _env
_env.pip_update_install_command = "%(virtualenv_dir)s/bin/pip install -r %(pip_remote_requirements_fn)s"
if _env.is_local:
run_or_dryrun(_env.pip_update_install_command % _env)
else:
sudo_or_dryrun(_env.pip_update_install_command % _env)
if not _env.is_local and (_env.pip_check_permissions or user or group
or perms):
set_virtualenv_permissions(
user=user,
group=group,
perms=perms,
virtualenv_dir=virtualenv_dir,
)
def createsuperuser(username='******', email=None, password=None, site=None):
"""
Runs the Django createsuperuser management command.
"""
from burlap.dj import render_remote_paths
set_site(site)
render_remote_paths()
env.db_createsuperuser_username = username
env.db_createsuperuser_email = email or username
run_or_dryrun(
'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s createsuperuser --username=%(db_createsuperuser_username)s --email=%(db_createsuperuser_email)s'
% env)
def database_files_dump(site=None):
"""
Runs the Django management command to export files stored in the database to the filesystem.
Assumes the app django_database_files is installed.
"""
from burlap.dj import render_remote_paths
set_site(site or env.SITE)
render_remote_paths()
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s database_files_dump' % env
if env.is_local:
local_or_dryrun(cmd)
else:
run_or_dryrun(cmd)
def database_files_dump(site=None):
"""
Runs the Django management command to export files stored in the database to the filesystem.
Assumes the app django_database_files is installed.
"""
from burlap.dj import render_remote_paths
set_site(site or env.SITE)
render_remote_paths()
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s database_files_dump' % env
if env.is_local:
local_or_dryrun(cmd)
else:
run_or_dryrun(cmd)
def get_conceptnet_data(dryrun=0):
dryrun = int(dryrun)
from burlap.dj import render_remote_paths
render_remote_paths()
cn_path = os.path.join(
env.remote_app_dir,
'.env/lib/python2.7/site-packages/conceptnet5/support_data')
cmd = 'mkdir -p %s' % cn_path
print cmd
if not dryrun:
run(cmd)
cmd = 'wget -P %s https://raw.githubusercontent.com/commonsense/conceptnet5/master/conceptnet5/support_data/iso639.txt' % cn_path
print cmd
if not dryrun:
run(cmd)
def upgrade_pip():
from burlap.dj import render_remote_paths
render_remote_paths()
if env.pip_virtual_env_dir_template:
env.pip_virtual_env_dir = env.pip_virtual_env_dir_template % env
sudo_or_dryrun(
"chown -R %(pip_user)s:%(pip_group)s %(pip_virtual_env_dir)s" % env)
run_or_dryrun(
". %(pip_virtual_env_dir)s/bin/activate; pip install --upgrade setuptools"
% env)
run_or_dryrun(
". %(pip_virtual_env_dir)s/bin/activate; pip install --upgrade distribute"
% env)
with settings(warn_only=True):
run_or_dryrun(
". %(pip_virtual_env_dir)s/bin/activate; pip install --upgrade pip"
% env)
def loaddata(path, site=None):
"""
Runs the Dango loaddata management command.
By default, runs on only the current site.
Pass site=all to run on all sites.
"""
render_remote_paths()
site = site or env.SITE
env._loaddata_path = path
for site, site_data in common.iter_sites(site=site, no_secure=True):
try:
set_db(site=site)
env.SITE = site
cmd = ('export SITE=%(SITE)s; export ROLE=%(ROLE)s; '
'cd %(shell_default_dir)s; '
'./manage loaddata %(_loaddata_path)s') % env
sudo_or_dryrun(cmd)
except KeyError:
pass
def loaddata(path, site=None):
"""
Runs the Dango loaddata management command.
By default, runs on only the current site.
Pass site=all to run on all sites.
"""
render_remote_paths()
site = site or env.SITE
env._loaddata_path = path
for site, site_data in common.iter_sites(site=site, no_secure=True):
try:
set_db(site=site)
env.SITE = site
cmd = ('export SITE=%(SITE)s; export ROLE=%(ROLE)s; '
'cd %(shell_default_dir)s; '
'./manage loaddata %(_loaddata_path)s') % env
sudo_or_dryrun(cmd)
except KeyError:
pass
def set_virtualenv_permissions(user=None,
group=None,
perms=None,
virtualenv_dir=None):
from burlap.dj import render_remote_paths
_env = type(env)(env)
_env.pip_user = user or _env.pip_user
_env.pip_group = group or _env.pip_group
_env.pip_chmod = perms or _env.pip_chmod
_env.pip_virtual_env_dir = virtualenv_dir or _env.pip_virtual_env_dir
render_remote_paths()
if virtualenv_dir:
_env.pip_virtual_env_dir = virtualenv_dir
elif _env.pip_virtual_env_dir_template:
_env.pip_virtual_env_dir = _env.pip_virtual_env_dir_template % _env
sudo_or_dryrun(
'chown -R %(pip_user)s:%(pip_group)s %(pip_virtual_env_dir)s' % _env)
sudo_or_dryrun('chmod -R %(pip_chmod)s %(pip_virtual_env_dir)s' % _env)
def install_fixtures(name, site=None):
"""
Installs a set of Django fixtures.
"""
from burlap.dj import render_remote_paths
set_site(site)
render_remote_paths()
fixtures_paths = env.db_fixture_sets.get(name, [])
for fixture_path in fixtures_paths:
env.db_fq_fixture_path = os.path.join(env.remote_app_src_package_dir, fixture_path)
print('Loading %s...' % (env.db_fq_fixture_path,))
if not env.is_local and not files.exists(env.db_fq_fixture_path):
put_or_dryrun(
local_path=env.db_fq_fixture_path,
remote_path='/tmp/data.json',
use_sudo=True,
)
env.db_fq_fixture_path = env.put_remote_path
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s loaddata %(db_fq_fixture_path)s' % env
print(cmd)
run_or_dryrun(cmd)
def manage(cmd, *args, **kwargs):
"""
A generic wrapper around Django's manage command.
"""
render_remote_paths()
environs = kwargs.pop('environs', '').strip()
if environs:
environs = ' '.join('export %s=%s;' % tuple(_.split('=')) for _ in environs.split(','))
environs = ' ' + environs + ' '
env.dj_cmd = cmd
env.dj_args = ' '.join(map(str, args))
env.dj_kwargs = ' '.join(
('--%s' % _k if _v in (True, 'True') else '--%s=%s' % (_k, _v))
for _k, _v in kwargs.iteritems())
env.dj_environs = environs
cmd = (
'export SITE=%(SITE)s; export ROLE=%(ROLE)s;%(dj_environs)scd %(remote_manage_dir)s; '
'%(django_manage)s %(dj_cmd)s %(dj_args)s %(dj_kwargs)s') % env
run_or_dryrun(cmd)
def manage(cmd, *args, **kwargs):
"""
A generic wrapper around Django's manage command.
"""
render_remote_paths()
environs = kwargs.pop('environs', '').strip()
if environs:
environs = ' '.join('export %s=%s;' % tuple(_.split('='))
for _ in environs.split(','))
environs = ' ' + environs + ' '
env.dj_cmd = cmd
env.dj_args = ' '.join(map(str, args))
env.dj_kwargs = ' '.join(
('--%s' % _k if _v in (True, 'True') else '--%s=%s' % (_k, _v))
for _k, _v in kwargs.iteritems())
env.dj_environs = environs
cmd = (
'export SITE=%(SITE)s; export ROLE=%(ROLE)s;%(dj_environs)scd %(remote_manage_dir)s; '
'%(django_manage)s %(dj_cmd)s %(dj_args)s %(dj_kwargs)s') % env
run_or_dryrun(cmd)
def install_fixtures(name, site=None):
"""
Installs a set of Django fixtures.
"""
from burlap.dj import render_remote_paths
set_site(site)
render_remote_paths()
fixtures_paths = env.db_fixture_sets.get(name, [])
for fixture_path in fixtures_paths:
env.db_fq_fixture_path = os.path.join(env.remote_app_src_package_dir,
fixture_path)
print('Loading %s...' % (env.db_fq_fixture_path, ))
if not env.is_local and not files.exists(env.db_fq_fixture_path):
put_or_dryrun(
local_path=env.db_fq_fixture_path,
remote_path='/tmp/data.json',
use_sudo=True,
)
env.db_fq_fixture_path = env.put_remote_path
cmd = 'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; %(django_manage)s loaddata %(db_fq_fixture_path)s' % env
print(cmd)
run_or_dryrun(cmd)
def render_paths(e=None):
from burlap.dj import render_remote_paths
_global_env = e is None
e = e or env
e = type(e)(e)
e.pip_path_versioned = e.pip_path % e
e = render_remote_paths(e)
if e.pip_virtual_env_dir_template:
e.pip_virtual_env_dir = e.pip_virtual_env_dir_template % e
if e.is_local:
e.pip_virtual_env_dir = os.path.abspath(e.pip_virtual_env_dir)
if _global_env:
env.update(e)
return e
def syncdb(site=None, all=0, database=None):
"""
Runs the standard Django syncdb command for one or more sites.
"""
#print 'Running syncdb...'
_env = type(env)(env)
_env.db_syncdb_all_flag = '--all' if int(all) else ''
_env.db_syncdb_database = ''
if database:
_env.db_syncdb_database = ' --database=%s' % database
_env = render_remote_paths(e=_env)
for site, site_data in iter_unique_databases(site=site):
cmd = (
'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; '
'%(django_manage)s syncdb --noinput %(db_syncdb_all_flag)s %(db_syncdb_database)s') % _env
run_or_dryrun(cmd)
def syncdb(site=None, all=0, database=None):
"""
Runs the standard Django syncdb command for one or more sites.
"""
#print 'Running syncdb...'
_env = type(env)(env)
_env.db_syncdb_all_flag = '--all' if int(all) else ''
_env.db_syncdb_database = ''
if database:
_env.db_syncdb_database = ' --database=%s' % database
_env = render_remote_paths(e=_env)
for site, site_data in iter_unique_databases(site=site):
cmd = (
'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; '
'%(django_manage)s syncdb --noinput %(db_syncdb_all_flag)s %(db_syncdb_database)s'
) % _env
run_or_dryrun(cmd)
def render_paths():
from burlap.dj import render_remote_paths
render_remote_paths()
def install(package='', clean=0, no_deps=1, all=0, upgrade=1):
"""
Installs the local cache of pip packages.
"""
from burlap.dj import render_remote_paths
print('Installing pip requirements...')
assert env[ROLE]
require('is_local')
# Delete any pre-existing environment.
if int(clean):
clean_virtualenv()
render_remote_paths()
if env.pip_virtual_env_dir_template:
env.pip_virtual_env_dir = env.pip_virtual_env_dir_template % env
env.pip_local_cache_dir = env.pip_local_cache_dir_template % env
env.pip_path_versioned = env.pip_path % env
if env.is_local:
env.pip_cache_dir = os.path.abspath(env.pip_local_cache_dir % env)
else:
env.pip_cache_dir = env.pip_remote_cache_dir % env
print('env.host_string:', env.host_string)
print('env.key_filename:', env.key_filename)
run_or_dryrun('mkdir -p %(pip_cache_dir)s' % env)
if not env.pip_cache_dir.endswith('/'):
env.pip_cache_dir = env.pip_cache_dir + '/'
env.pip_key_filename = os.path.abspath(env.key_filename)
local_or_dryrun(
'rsync -avz --progress --rsh "ssh -o StrictHostKeyChecking=no -i %(pip_key_filename)s" %(pip_local_cache_dir)s/* %(user)s@%(host_string)s:%(pip_cache_dir)s'
% env)
env.pip_upgrade_flag = ''
if int(upgrade):
env.pip_upgrade_flag = ' -U '
env.pip_no_deps = ''
if int(no_deps):
env.pip_no_deps = '--no-deps'
if int(all):
packages = list(iter_pip_requirements())
elif package:
packages = [package]
else:
packages = [k for k, v in check()]
env.pip_build_dir = tempfile.mkdtemp()
for package in packages:
env.pip_package = package
if env.is_local:
run_or_dryrun(env.pip_install_command % env)
else:
sudo_or_dryrun(env.pip_install_command % env)
if not env.is_local:
sudo_or_dryrun(
'chown -R %(pip_user)s:%(pip_group)s %(remote_app_dir)s' % env)
sudo_or_dryrun('chmod -R %(pip_chmod)s %(remote_app_dir)s' % env)
def migrate(app='',
migration='',
site=None,
fake=0,
ignore_errors=0,
skip_databases=None,
database=None,
migrate_apps='',
delete_ghosts=1):
"""
Runs the standard South migrate command for one or more sites.
"""
ignore_errors = int(ignore_errors)
delete_ghosts = int(delete_ghosts)
skip_databases = (skip_databases or '')
if isinstance(skip_databases, basestring):
skip_databases = [
_.strip() for _ in skip_databases.split(',') if _.strip()
]
migrate_apps = migrate_apps or ''
migrate_apps = [
_.strip().split('.')[-1] for _ in migrate_apps.strip().split(',')
if _.strip()
]
if app:
migrate_apps.append(app)
render_remote_paths()
_env = type(env)(env)
_env.django_migrate_migration = migration or ''
_env.django_migrate_fake_str = '--fake' if int(fake) else ''
_env.django_migrate_database = '--database=%s' % database if database else ''
_env.delete_ghosts = '--delete-ghost-migrations' if delete_ghosts else ''
for site, site_data in iter_unique_databases(site=site):
print('-' * 80, file=sys.stderr)
print('site:', site, file=sys.stderr)
if env.available_sites_by_host:
hostname = common.get_current_hostname()
sites_on_host = env.available_sites_by_host.get(hostname, [])
if sites_on_host and site not in sites_on_host:
print('skipping site:', site, sites_on_host, file=sys.stderr)
continue
print('migrate_apps:', migrate_apps, file=sys.stderr)
if migrate_apps:
_env.django_migrate_app = ' '.join(migrate_apps)
else:
_env.django_migrate_app = ''
_env.SITE = site
cmd = (
'export SITE=%(SITE)s; export ROLE=%(ROLE)s; cd %(remote_manage_dir)s; '
'%(django_manage)s migrate --noinput --traceback %(django_migrate_database)s %(delete_ghosts)s %(django_migrate_app)s %(django_migrate_migration)s '
'%(django_migrate_fake_str)s') % _env
cmd = cmd.strip()
with settings(warn_only=ignore_errors):
run_or_dryrun(cmd)
def check_remote_paths(verbose=1):
if 'django_settings_module' in env:
return
render_remote_paths()
def render_paths(self):
from burlap.dj import render_remote_paths
render_remote_paths()
if self.env.erlang_cookie_template:
self.env.erlang_cookie = self.env.erlang_cookie_template % self.genv
def check_remote_paths(verbose=1):
if 'django_settings_module' in env:
return
render_remote_paths()
def sync(sync_set, force=0):
"""
Uploads media to an Amazon S3 bucket using s3sync.
Requires the s3sync gem: sudo gem install s3sync
"""
from burlap.dj import get_settings, render_remote_paths
force = int(force)
env.s3_sync_force_flag = ' --force ' if force else ''
# print'env.SITE:',env.SITE
_settings = get_settings(verbose=1)
assert _settings, 'Unable to import settings.'
for k in _settings.__dict__.iterkeys():
if k.startswith('AWS_'):
env[k] = _settings.__dict__[k]
#local_or_dryrun('which s3sync')
#print 'AWS_STATIC_BUCKET_NAME:',_settings.AWS_STATIC_BUCKET_NAME
render_remote_paths()
site_data = env.sites[env.SITE]
env.update(site_data)
rets = []
for paths in env.s3_sync_sets[sync_set]:
is_local = paths.get('is_local', True)
local_path = paths['local_path'] % env
remote_path = paths['remote_path']
remote_path = remote_path.replace(':/', '/')
if not remote_path.startswith('s3://'):
remote_path = 's3://' + remote_path
local_path = local_path % env
if is_local:
#local_or_dryrun('which s3sync')#, capture=True)
env.s3_local_path = os.path.abspath(local_path)
else:
#run('which s3sync')
env.s3_local_path = local_path
if local_path.endswith('/') and not env.s3_local_path.endswith('/'):
env.s3_local_path = env.s3_local_path + '/'
env.s3_remote_path = remote_path % env
print('Syncing %s to %s...' % (env.s3_local_path, env.s3_remote_path))
# Old buggy Ruby version.
# cmd = ('export AWS_ACCESS_KEY_ID=%(AWS_ACCESS_KEY_ID)s; '\
# 'export AWS_SECRET_ACCESS_KEY=%(AWS_SECRET_ACCESS_KEY)s; '\
# 's3sync --recursive --verbose --progress --public-read '\
# '%(s3_local_path)s %(s3_remote_path)s') % env
# Superior Python version.
if force:
env.s3_sync_cmd = 'put'
else:
env.s3_sync_cmd = 'sync'
cmd = (
'export AWS_ACCESS_KEY_ID=%(AWS_ACCESS_KEY_ID)s; '\
'export AWS_SECRET_ACCESS_KEY=%(AWS_SECRET_ACCESS_KEY)s; '\
's3cmd %(s3_sync_cmd)s --progress --acl-public --guess-mime-type --no-mime-magic '\
'--delete-removed --cf-invalidate --recursive %(s3_sync_force_flag)s '\
'%(s3_local_path)s %(s3_remote_path)s') % env
if is_local:
local_or_dryrun(cmd)
else:
run_or_dryrun(cmd)
def sync(sync_set, force=0):
"""
Uploads media to an Amazon S3 bucket using s3sync.
Requires the s3sync gem: sudo gem install s3sync
"""
from burlap.dj import get_settings, render_remote_paths
force = int(force)
env.s3_sync_force_flag = ' --force ' if force else ''
# print'env.SITE:',env.SITE
_settings = get_settings(verbose=1)
assert _settings, 'Unable to import settings.'
for k in _settings.__dict__.iterkeys():
if k.startswith('AWS_'):
env[k] = _settings.__dict__[k]
#local_or_dryrun('which s3sync')
#print 'AWS_STATIC_BUCKET_NAME:',_settings.AWS_STATIC_BUCKET_NAME
render_remote_paths()
site_data = env.sites[env.SITE]
env.update(site_data)
rets = []
for paths in env.s3_sync_sets[sync_set]:
is_local = paths.get('is_local', True)
local_path = paths['local_path'] % env
remote_path = paths['remote_path']
remote_path = remote_path.replace(':/', '/')
if not remote_path.startswith('s3://'):
remote_path = 's3://' + remote_path
local_path = local_path % env
if is_local:
#local_or_dryrun('which s3sync')#, capture=True)
env.s3_local_path = os.path.abspath(local_path)
else:
#run('which s3sync')
env.s3_local_path = local_path
if local_path.endswith('/') and not env.s3_local_path.endswith('/'):
env.s3_local_path = env.s3_local_path + '/'
env.s3_remote_path = remote_path % env
print('Syncing %s to %s...' % (env.s3_local_path, env.s3_remote_path))
# Old buggy Ruby version.
# cmd = ('export AWS_ACCESS_KEY_ID=%(AWS_ACCESS_KEY_ID)s; '\
# 'export AWS_SECRET_ACCESS_KEY=%(AWS_SECRET_ACCESS_KEY)s; '\
# 's3sync --recursive --verbose --progress --public-read '\
# '%(s3_local_path)s %(s3_remote_path)s') % env
# Superior Python version.
if force:
env.s3_sync_cmd = 'put'
else:
env.s3_sync_cmd = 'sync'
cmd = (
'export AWS_ACCESS_KEY_ID=%(AWS_ACCESS_KEY_ID)s; '\
'export AWS_SECRET_ACCESS_KEY=%(AWS_SECRET_ACCESS_KEY)s; '\
's3cmd %(s3_sync_cmd)s --progress --acl-public --guess-mime-type --no-mime-magic '\
'--delete-removed --cf-invalidate --recursive %(s3_sync_force_flag)s '\
'%(s3_local_path)s %(s3_remote_path)s') % env
if is_local:
local_or_dryrun(cmd)
else:
run_or_dryrun(cmd)
def render_paths():
from burlap.dj import render_remote_paths
render_remote_paths()
def sync_media(self, sync_set=None, clean=0, iter_local_paths=0):
"""
Uploads select media to an Apache accessible directory.
"""
apache.get_apache_settings()
from burlap.dj import render_remote_paths
apache_specifics = apache.set_apache_specifics()
render_remote_paths()
clean = int(clean)
print('Getting site data for %s...' % self.genv.SITE)
site_data = self.genv.sites[self.genv.SITE]
self.genv.update(site_data)
sync_sets = self.genv.apache_sync_sets
if sync_set:
sync_sets = [sync_set]
ret_paths = []
for sync_set in sync_sets:
for paths in self.genv.apache_sync_sets[sync_set]:
#print 'paths:',paths
self.genv.apache_sync_local_path = os.path.abspath(
paths['local_path'] % self.genv)
if paths['local_path'].endswith(
'/'
) and not self.genv.apache_sync_local_path.endswith('/'):
self.genv.apache_sync_local_path += '/'
if iter_local_paths:
ret_paths.append(self.genv.apache_sync_local_path)
continue
self.genv.apache_sync_remote_path = paths[
'remote_path'] % self.genv
if clean:
self.sudo_or_dryrun('rm -Rf %(apache_sync_remote_path)s' %
self.genv)
print('Syncing %s to %s...' %
(self.genv.apache_sync_local_path,
self.genv.apache_sync_remote_path))
self.genv.apache_tmp_chmod = paths.get('chmod',
self.genv.apache_chmod)
#with settings(warn_only=True):
self.sudo_or_dryrun('mkdir -p %(apache_sync_remote_path)s' %
self.genv,
user=self.genv.apache_user)
self.sudo_or_dryrun(
'chmod -R %(apache_tmp_chmod)s %(apache_sync_remote_path)s'
% self.genv,
user=self.genv.apache_user)
cmd = (
'rsync -rvz --progress --recursive --no-p --no-g --rsh "ssh -o StrictHostKeyChecking=no -i %(key_filename)s" %(apache_sync_local_path)s %(user)s@%(host_string)s:%(apache_sync_remote_path)s'
) % self.genv
self.local_or_dryrun(cmd)
self.sudo_or_dryrun(
'chown -R %(apache_user)s:%(apache_group)s %(apache_sync_remote_path)s'
% self.genv)
if iter_local_paths:
return ret_paths
def render_paths(self):
from burlap.dj import render_remote_paths
render_remote_paths()
if self.env.erlang_cookie_template:
self.env.erlang_cookie = self.env.erlang_cookie_template % self.genv
