def devserver_conf():
"""
Render and update invenio-devserver configuration
The task will look for the template ``config_local.py.tpl``, and render and
write it to ``config_local.py`` in the virtual environments site-packages.
The invenio-devserver install two commands:
* ``serve`` - Invenio development server based on Werkzeug.
* ``mailserve`` - Debug mail server which will print all emails to the console.
.. note::
The invenio-devserver works with the non-Flask based versions of Invenio.
Also, the invenio-devserver is only installed if ``env.WITH_DEVSERVER`` is
``True``.
.. seealso::
See also invenio-devserver for further information on the content of
``config_local.py.tpl``: https://bitbucket.org/osso/invenio-devserver
"""
puts(cyan(">>> Configuring invenio-devserver..." % env))
pyver = python_version()
local_file = 'config_local.py.tpl'
local_remote = os.path.join(env.CFG_INVENIO_PREFIX, 'lib/python%s/site-packages/config_local.py' % pyver)
puts(">>> Writing config_local.py to %s ..." % local_remote)
try:
write_template(local_remote, env, tpl_file=local_file)
except TemplateNotFound:
warn(red("Could not find template %s" % local_file))
def run_migrations():
if env.environment == 'live':
with prefix('source venv/bin/activate'):
run('cd datalabsupport/datalabsupport/ && python manage.py migrate --settings=datalabsupport.settings'
)
else:
warn("Refusing to run migrations in staging environment")
def devserver_install_flask():
"""
Install a Flask devserver
The task will look for the template ``rundevserver.py.tpl``, render it and
write it to ``bin/rundevserver.py``.
To start the Flask development server, run::
(venv)$ rundevserver.py
.. note::
``rundevserver.py`` only works with Flask based versions of Invenio.
"""
puts(cyan(">>> Configuring Flask devserver..." % env))
local_file = 'rundevserver.py.tpl'
local_remote = os.path.join(env.CFG_INVENIO_PREFIX, 'bin/rundevserver.py')
puts(">>> Writing rundevserver.py to %s ..." % local_remote)
try:
write_template(local_remote, env, tpl_file=local_file)
local("chmod a+x %s" % local_remote)
except TemplateNotFound:
warn(red("Could not find template %s" % local_file))
def _param(name, adict):
"Get parameter value"
val = adict.get(name, '')
if val == '':
warn('Parameter %s is undefined' % name)
return val
def _config_nginx(site_name, sdir, settings):
if settings=='dev':
return
sudo("mkdir -p /etc/nginx/conf.d/{0}".format(site_name))
if settings=='production':
template_name = 'nginx.ssl_template.conf'
elif settings=='staging':
template_name = 'nginx.http_template.conf'
else:
warn(yellow(
'Not configuring nginx based on settings=={0}'.format(settings)))
return
if not exists("/etc/nginx/sites-available/{0}".format(site_name)):
nginx_template_path = '{0}/deploy_tools/{1}'.format(
sdir, template_name)
sed_cmd = "sed \"s/SITENAME/{0}/g\" {1} | tee /etc/nginx/sites-avail"\
"able/{0}".format(site_name, nginx_template_path)
sudo(sed_cmd)
if not exists("/etc/nginx/sites-enabled/{0}".format(site_name)):
sudo ("ln -s /etc/nginx/sites-available/{0} /etc/nginx/sites-enabl"\
"ed/{0}".format(site_name))
if settings=='production':
_set_up_SSL_key_and_cert(site_name)
def clean():
with settings(warn_only=True):
sudo('service nginx stop')
result = sudo('service %s stop' % PROJECT_NAME)
if result.failed:
warn( "%s was not running." % PROJECT_NAME)
with settings(warn_only=True):
result = sudo('service %s stop' % PROJECT_NAME_STAGING)
if result.failed:
warn( "%s was not running." % PROJECT_NAME_STAGING)
for name in (PROJECT_NAME, PROJECT_NAME_STAGING):
sudo('rmvirtualenv %s' % name)
with settings(warn_only=True):
for dir in (PROJECT_DIR, PROJECT_DIR_STAGING, PROJECT_LOGDIR, PROJECT_LOGDIR_STAGING):
sudo('rm -rf %s' % dir)
sudo('rm /home/%s/%s' % (PROJECT_USER, PROJECT_SCRIPT_NAME))
sudo('rm /etc/nginx/sites-enabled/%s' % PROJECT_NAME)
sudo('rm /etc/nginx/sites-available/%s' % PROJECT_NAME)
sudo('rm /etc/init/%s.conf' % PROJECT_NAME)
sudo('rm /etc/init.d/%s' % PROJECT_NAME)
sudo('rm /home/%s/%s' % (PROJECT_USER, PROJECT_SCRIPT_NAME_STAGING))
sudo('rm /etc/nginx/sites-enabled/%s' % PROJECT_NAME_STAGING)
sudo('rm /etc/nginx/sites-available/%s' % PROJECT_NAME_STAGING)
sudo('rm /etc/init/%s.conf' % PROJECT_NAME_STAGING)
sudo('rm /etc/init.d/%s' % PROJECT_NAME_STAGING)
def maintenancemode():
"""If using the maintenancemode app
(https://github.com/jezdez/django-maintenancemode), this command will toggle
it on and off. It finds the `MAINTENANCE_MODE` variable in your
`settings.py` on the remote server, toggles its value and restarts the web
server.
Requires the env keys:
toggle - set by enable() or disable(), indicates whether we should turn
maintenance mode on or off.
settings - relative path from the project root to the settings.py file
current_release_path - path to the current release on the remote server
"""
require('toggle', provided_by=[enable, disable])
require('settings')
require('current_release_path')
settings_file = os.path.join(utils.absolute_release_path(), env.settings)
if exists(settings_file):
sed(settings_file, '(MAINTENANCE_MODE = )(False|True)',
'\\1%(toggle)s' % env)
restart_webserver()
else:
warn('Settings file %s could not be found' % settings_file)
def locales(names):
"""
Require the list of locales to be available.
"""
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
if distrib_id() == "Archlinux":
run_as_root('locale-gen')
else:
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
def fetch(self):
if self.fetched:
return
self.hosts = _fetch_hosts()
for host in self.hosts:
try:
name, vdc, _ = host.split('.', 3)
except ValueError:
warn("discarding badly formatted hostname '{0}'".format(host))
continue
# Don't refer to foo.bar.production, as it's confusing when doing
# things in preview or staging. Refer to the machines exclusively by
# short name.
short_host = '{0}.{1}'.format(name, vdc)
cls = name.rstrip('-1234567890').replace('-', '_')
self.roledefs['all'].append(short_host)
self.roledefs['class-%s' % cls].append(short_host)
self.roledefs['vdc-%s' % vdc].append(short_host)
self.classes.add(cls)
self.vdcs.add(vdc)
self.fetched = True
def apache_conf():
""" Upload and update Apache configuration """
puts(cyan(">>> Configuring Apache..." % env))
conf_files = ['etc/apache/invenio-apache-vhost.conf', 'etc/apache/invenio-apache-vhost-ssl.conf']
conf_files = [(p, os.path.join(env.CFG_INVENIO_PREFIX, p)) for p in conf_files]
for local_file, remote_file in conf_files:
puts(">>> Writing %s ..." % remote_file)
try:
if not exists_local(os.path.dirname(remote_file)):
sudo_local("mkdir -p %s" % os.path.dirname(remote_file), user=env.CFG_INVENIO_USER)
write_template(remote_file, env, tpl_file=local_file, use_sudo=True)
except TemplateNotFound:
abort(red("Could not find template %s" % local_file))
apache_conf = env.get('CFG_APACHE_CONF', '/etc/httpd/conf/httpd.conf')
if confirm("Include created files in %s?" % apache_conf):
if exists_local(apache_conf, use_sudo=True):
lines = ["Include %s" % r for (l, r) in conf_files]
if is_local():
with settings(host_string="localhost"):
append(apache_conf, lines, use_sudo=True)
else:
append(apache_conf, lines, use_sudo=True)
else:
warn(red("File %s does not exists" % apache_conf))
sudo_local("%(CFG_INVENIO_APACHECTL)s configtest" % env)
def clean():
with settings(warn_only=True):
sudo('service nginx stop')
result = sudo('service %s stop' % PROJECT_NAME)
if result.failed:
warn("%s was not running." % PROJECT_NAME)
with settings(warn_only=True):
result = sudo('service %s stop' % PROJECT_NAME_STAGING)
if result.failed:
warn("%s was not running." % PROJECT_NAME_STAGING)
for name in (PROJECT_NAME, PROJECT_NAME_STAGING):
sudo('rmvirtualenv %s' % name)
with settings(warn_only=True):
for dir in (PROJECT_DIR, PROJECT_DIR_STAGING, PROJECT_LOGDIR,
PROJECT_LOGDIR_STAGING):
sudo('rm -rf %s' % dir)
sudo('rm /home/%s/%s' % (PROJECT_USER, PROJECT_SCRIPT_NAME))
sudo('rm /etc/nginx/sites-enabled/%s' % PROJECT_NAME)
sudo('rm /etc/nginx/sites-available/%s' % PROJECT_NAME)
sudo('rm /etc/init/%s.conf' % PROJECT_NAME)
sudo('rm /etc/init.d/%s' % PROJECT_NAME)
sudo('rm /home/%s/%s' % (PROJECT_USER, PROJECT_SCRIPT_NAME_STAGING))
sudo('rm /etc/nginx/sites-enabled/%s' % PROJECT_NAME_STAGING)
sudo('rm /etc/nginx/sites-available/%s' % PROJECT_NAME_STAGING)
sudo('rm /etc/init/%s.conf' % PROJECT_NAME_STAGING)
sudo('rm /etc/init.d/%s' % PROJECT_NAME_STAGING)
def copy_configs():
for config, data in env.configs.items():
per_host_file = env.per_host_config_scheme % {"host": env.host, "config": config}
generic_file = env.config_scheme % {"host": env.host, "config": config}
if os.path.exists(per_host_file):
config_file = per_host_file
elif os.path.exists(generic_file):
config_file = generic_file
else:
warn("No config file found for %s (tried %s and %s)" % (config, per_host_file, generic_file))
continue
with open(config_file, "r") as config_file:
if callable(data):
data = data()
config_content = config_file.read()
if "context" in data:
config_content = config_content % data["context"]
if not "target" in data:
warn("No target specified for config %s" % config)
continue
put(StringIO.StringIO(config_content), data["target"], use_sudo=True)
if "chown" in data:
chown = data["chown"]
else:
chown = "root:root"
sudo("chown %s %s" % (chown, data["target"]))
def clear_cloudflare(purge_all=False):
url = 'https://api.cloudflare.com/client/v4/zones/%s'
headers = {
"Content-Type": "application/json",
"X-Auth-Key": os.environ['CF_API_KEY'],
"X-Auth-Email": os.environ['CF_API_EMAIL']
}
if purge_all:
data = {'purge_everything': True}
else:
# XXX need to think about these. If we're looking at files
# that have changed since the deployment started, do we need
# to bother with files that have changed according to git?
changed_files_from_git = env.changed_files.copy()
data = {'files': purge_urls(changed_files_from_git,
find_changed_static_files())}
print "Purging from Cloudflare:"
print data
result = json.loads(
requests.delete(url % ZONE_ID + '/purge_cache',
headers=headers, data=json.dumps(data)).text)
if result['success']:
print "Cloudflare clearing succeeded: %s" % \
json.dumps(result, indent=2)
else:
warn("Cloudflare clearing failed: %s" %
json.dumps(result, indent=2))
def md5sum(filename, use_sudo=False):
"""
Compute the MD5 sum of a file.
"""
func = use_sudo and run_as_root or run
with settings(hide('running', 'stdout', 'stderr', 'warnings'),
warn_only=True):
# Linux (LSB)
if exists(u'/usr/bin/md5sum'):
res = func(u'/usr/bin/md5sum %(filename)s' % locals())
# BSD / OS X
elif exists(u'/sbin/md5'):
res = func(u'/sbin/md5 -r %(filename)s' % locals())
# SmartOS Joyent build
elif exists(u'/opt/local/gnu/bin/md5sum'):
res = func(u'/opt/local/gnu/bin/md5sum %(filename)s' % locals())
else:
abort('No MD5 utility was found on this system.')
if res.succeeded:
parts = res.split()
_md5sum = len(parts) > 0 and parts[0] or None
else:
warn(res)
_md5sum = None
return _md5sum
def _get_commits_for_release(commits: List[Commit], auto: bool = False) -> List[Commit]:
candidate_commits = OrderedDict([
(commit.sha, commit)
for commit in commits
])
# do not prompt for release sha in auto mode
if not auto and len(commits) > 1:
commits_log = '\n'.join([
f'{commit.sha} {commit.msg}'
for i, commit in enumerate(commits)
])
release_commit_sha = None
fastprint(f'You are about to release commits (not tip of master branch):\n{commits_log}\n')
while not release_commit_sha:
commit_text_user_prompt = prompt('Type full commit hash you are releasing >',
validate=lambda n: n if len(n) == 40 else False)
if not commit_text_user_prompt:
warn('There should be 40 characters in commit hash')
elif commit_text_user_prompt not in candidate_commits:
warn(f'Your input "{commit_text_user_prompt}" does not match any commit hash:\n{commits_log}')
else:
release_commit_sha = commit_text_user_prompt
# drop all the top commits not matching the chosen one
for candidate_commit_sha in list(candidate_commits):
if candidate_commit_sha == release_commit_sha:
break
else:
candidate_commits.pop(candidate_commit_sha)
# reverse the order of commits
return list(candidate_commits.values())
def install_deps():
"""Install OS-level (e.g. C library) dependencies
Requires that the OS provides APT.
This task respects the roles under which it is invoked, (and defaults to
"dev"). "Base" dependencies are always installed, as well as role-specific
dependencies found in the dependencies/ directory. For example:
fab -R staging dependencies
will install dependencies specified by base.dependencies and
staging.dependencies, given that they are both discovered in the
dependencies/ directory.
"""
# Check for apt-get:
if not which_binary('apt-get'):
# warn & return rather than abort so as not to raise SystemExit:
fab.warn("No path to APT, cannot install OS dependencies")
return
# Install APT packages specified in dependencies dir:
roles = fab.env.roles or ['dev'] # Default to just dev
deps_paths = (join(BASEDIR, 'dependencies', '{}.dependencies'.format(role))
for role in itertools.chain(['base'], roles))
deps = itertools.chain.from_iterable(open(deps_path).readlines()
for deps_path in deps_paths
if os.path.exists(deps_path))
l('sudo apt-get install -y {}'.format(
' '.join(dep.strip() for dep in deps)))
def prepare_deployment_host(bootstrap_path=None, release=None, use_branch=None,
skip_clone=None, bootstrap_branch=None,
skip_pip_download=None, skip_brew_download=None,
specific_tag=None):
"""Prepares the deployment host.
"""
bootstrap_env(
path=bootstrap_path,
filename='bootstrap.conf',
bootstrap_branch=bootstrap_branch)
if release:
env.project_release = release
prepare_deployment_dir()
prepare_deployment_repo(skip_clone=skip_clone, use_branch=use_branch,
specific_tag=specific_tag)
with cd(env.project_repo_root):
result = run('git status', warn_only=True)
results = result.split('\n')
if results[0] != 'On branch {bootstrap_branch}'.format(
bootstrap_branch=bootstrap_branch):
warn(results[0])
if not exists(env.fabric_config_path):
abort('Missing fabric config file. Expected {}'.format(
env.fabric_config_path))
update_fabric_env()
if env.target_os == MACOSX and not skip_pip_download:
pip_download_cache()
if env.target_os == MACOSX and not skip_brew_download:
update_deployment_brew_dir()
def clear_cloudflare(purge_all=False):
url = 'https://api.cloudflare.com/client/v4/zones/%s'
headers = {
"Content-Type": "application/json",
"X-Auth-Key": os.environ['CF_API_KEY'],
"X-Auth-Email": os.environ['CF_API_EMAIL']
}
if purge_all:
data = {'purge_everything': True}
else:
# XXX need to think about these. If we're looking at files
# that have changed since the deployment started, do we need
# to bother with files that have changed according to git?
changed_files_from_git = env.changed_files.copy()
data = {
'files':
purge_urls(changed_files_from_git, find_changed_static_files())
}
print "Purging from Cloudflare:"
print data
result = json.loads(
requests.delete(url % ZONE_ID + '/purge_cache',
headers=headers,
data=json.dumps(data)).text)
if result['success']:
print "Cloudflare clearing succeeded: %s" % \
json.dumps(result, indent=2)
else:
warn("Cloudflare clearing failed: %s" % json.dumps(result, indent=2))
def configure(self):
# Prepare configuration file contents
with open(self.template_path, "rb") as template:
config = template.read() % self._conf_args
# Deploy configuration file on server
with settings(hide('running', 'stdout'), warn_only=True):
result = put(
local_path=StringIO(config),
remote_path=self._conf_destination,
use_sudo=self._conf_needs_sudo)
if result.succeeded:
print green("Configured service %s" % self._service_name)
else:
print red("Failed to configure service %s" % self._service_name)
abort(red("Cannot continue"))
# Execute extra post-config commands
if self._post_config_commands:
fab_func = self._conf_needs_sudo and sudo or run
with settings(hide('running', 'stdout'), warn_only=True):
for cmd in self._post_config_commands:
result = fab_func(cmd)
if result.failed:
break
if result.succeeded:
print green("Applied post-configuration for service %s" % self._service_name)
else:
warn(yellow("Unable to apply post-configuration for service %s" % self._service_name))
def revert_left(self, other):
"""
Reverts all migrations which are present in this deployment but not in the other.
"""
delta = self.compare(other)
failures = dict()
for app in delta:
to_revert = delta[app]["left"] # filenames
if to_revert:
common_migrations = delta[app]["common"]
if common_migrations:
latest_common_migration = max([self.__migration_nr(fn) for fn in common_migrations])
else:
latest_common_migration = None
if latest_common_migration is not None:
revert_to_nr = "%04d" % latest_common_migration
else:
revert_to_nr = "zero"
revert_str = " ".join(["%04d" % n for n in sorted([self.__migration_nr(fn) for fn in to_revert])])
print "Reverting migrations for app %s: %s ..." % (app, revert_str)
with settings(hide('running', 'stdout'), warn_only=True):
result = self.__manage_cmd("migrate %s %s" % (app, revert_to_nr))
if result.failed:
failures[app] = "(stdout): %s\n(stderr): %s" % (result.stdout, result.stderr)
for app in failures:
warn(red("Failed to revert superfluous migrations for app %s" % app))
print "Details:\n%s" % failures[app]
if failures:
warn(red("This needs your attention!"))
def run_migrations():
if env.environment == 'production':
with prefix('source .venv/bin/activate'):
run('cd openprescribing/ && python manage.py migrate '
'--settings=openprescribing.settings.production')
else:
warn("Refusing to run migrations in staging environment")
def clean(*args, **kwargs):
"""
Clean before reinstalling. It can be called for multiple environments and there's an optional clean_nginx argument at the end.
fab -H user@host clean:production,staging,development,clean_nginx=y
"""
project_settings = get_settings()
projects = build_projects_vars()
with settings(hide('warnings'), warn_only=True):
sudo('service nginx stop')
for key in args:
print "CLEANING CONFIGURATION FILES AND STOPPING SERVICES FOR %s..." % key
result = sudo('service %(name)s stop' % projects[key])
if result.failed:
warn( "%(name)s was not running." % projects[key])
for app in project_settings.EXTRA_APPS:
run('workon %s && pip uninstall -y %s' % (projects[key]['name'], app['name']))
sudo('rm -rf %(dir)s' % projects[key])
sudo('rm -rf %(logdir)s' % projects[key])
sudo('rmvirtualenv %(name)s' % projects[key])
sudo('rm /home/%(user)s/%(script_name)s' % projects[key])
sudo('rm /etc/nginx/sites-enabled/%(name)s' % projects[key])
sudo('rm /etc/nginx/sites-available/%(name)s' % projects[key])
sudo('rm /etc/init/%(name)s.conf' % projects[key])
sudo('rm /etc/init.d/%(name)s' % projects[key])
if kwargs.get('clean_nginx','n') == 'y':
sed('/etc/nginx/nginx.conf', 'types_hash_max_size.*', '# types_hash_max_size 2048;', use_sudo=True)
sed('/etc/nginx/nginx.conf', 'server_names_hash_bucket_size.*', '# server_names_hash_bucket_size 64;', use_sudo=True)
fix_venv_permission()
def maintenancemode():
"""If using the maintenancemode app
(https://github.com/jezdez/django-maintenancemode), this command will toggle
it on and off. It finds the `MAINTENANCE_MODE` variable in your
`settings.py` on the remote server, toggles its value and restarts the web
server.
Requires the env keys:
toggle - set by enable() or disable(), indicates whether we should turn
maintenance mode on or off.
settings - relative path from the project root to the settings.py file
current_release_path - path to the current release on the remote server
"""
require('toggle', provided_by=[enable, disable])
require('settings')
require('current_release_path')
settings_file = os.path.join(utils.absolute_release_path(), env.settings)
if exists(settings_file):
sed(settings_file, '(MAINTENANCE_MODE = )(False|True)',
'\\1%(toggle)s' % env)
restart_webserver()
else:
warn('Settings file %s could not be found' % settings_file)
def deploy():
if not contains("/etc/passwd", env.deploy_user):
sudo("useradd %(user)s -d %(home)s -U" % {"user": env.deploy_user, "home": env.project_dir})
if exists(env.project_dir, use_sudo=True):
warn("Project dir %s already exists" % env.project_dir)
with cd(env.project_dir):
sudo("git reset --hard", user=env.deploy_user)
sudo("git pull", user=env.deploy_user)
else:
with cd(env.project_base_dir):
sudo("git clone %s" % env.project_repository)
for dir in ("logs", "public", "media/static", "media/dynamic"):
full_dir = os.path.join(env.project_dir, dir)
sudo("mkdir -p %s" % full_dir)
sudo("chown -R %(user)s:%(user)s %(directory)s" % {"user": env.deploy_user, "directory": env.project_dir})
sudo("chmod 750 %s" % env.project_dir)
if not exists(env.virtualenv_dir, use_sudo=True):
sudo("virtualenv --no-site-packages %s" % env.virtualenv_dir, user=env.deploy_user)
sudo(
"pip install -E %(env)s -r %(req)s"
% {"env": env.virtualenv_dir, "req": os.path.join(env.project_dir, "deploy/requirements.txt")},
user=env.deploy_user,
)
copy_configs()
django_command("syncdb")
django_command("migrate")
django_command("collectstatic --noinput")
sudo("service uwsgi reload")
sudo("service nginx reload")
def main(host=None, user=None):
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--user', help="user to connect")
parser.add_argument('-H', '--host', help="host to deploy", required=True)
parser.add_argument('--refresh', help="whether to refresh service",
action='store_true', default=False)
parser.add_argument('--remote-path', help="remote service path",
required=True)
parser.add_argument("--remote-user", help="remote service user",
default="root:root")
parser.add_argument("--only-static", help="deploy only static files",
action='store_true', default=False)
args = parser.parse_args()
if not args.user:
warn("Using default user: {}".format(env.user))
else:
env.user = args.user
if not args.remote_user:
env.remote_user = "******"
warn("Using default remote user: root:root")
else:
env.remote_user = args.remote_user
env.remote_path = args.remote_path
env.only_static = args.only_static
env.refresh = args.refresh
env.use_ssh_config = True
execute(deploy, hosts=[args.host])
def locales(names):
"""
Require the list of locales to be available.
"""
if distrib_id() == "Ubuntu":
config_file = '/var/lib/locales/supported.d/local'
if not is_file(config_file):
run_as_root('touch %s' % config_file)
else:
config_file = '/etc/locale.gen'
# Regenerate locales if config file changes
with watch(config_file, use_sudo=True) as config:
# Add valid locale names to the config file
supported = dict(supported_locales())
for name in names:
if name in supported:
charset = supported[name]
locale = "%s %s" % (name, charset)
uncomment(config_file, escape(locale), use_sudo=True, shell=True)
append(config_file, locale, use_sudo=True, partial=True, shell=True)
else:
warn('Unsupported locale name "%s"' % name)
if config.changed:
family = distrib_family()
if family == 'debian':
run_as_root('dpkg-reconfigure --frontend=noninteractive locales')
elif family in ['arch', 'gentoo']:
run_as_root('locale-gen')
else:
raise UnsupportedFamily(supported=['debian', 'arch', 'gentoo'])
def md5sum(filename, use_sudo=False):
"""
Compute the MD5 sum of a file.
"""
func = use_sudo and run_as_root or run
with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
# Linux (LSB)
if exists(u'/usr/bin/md5sum'):
res = func(u'/usr/bin/md5sum %(filename)s' % locals())
# BSD / OS X
elif exists(u'/sbin/md5'):
res = func(u'/sbin/md5 -r %(filename)s' % locals())
# SmartOS Joyent build
elif exists(u'/opt/local/gnu/bin/md5sum'):
res = func(u'/opt/local/gnu/bin/md5sum %(filename)s' % locals())
else:
abort('No MD5 utility was found on this system.')
if res.succeeded:
parts = res.split()
_md5sum = len(parts) > 0 and parts[0] or None
else:
warn(res)
_md5sum = None
return _md5sum
def write_template_hook(ctx):
try:
write_template(filename % ctx, ctx, tpl_file=tpl_file)
except TemplateNotFound, e:
if warn_only:
warn(red("Couldn't find template %s" % tpl_file))
else:
abort(red("Couldn't find template %s" % tpl_file))
def wrapper(*args, **kwds):
if all(test(name) for name in names):
return f(*args, **kwds)
else:
for name in names:
if not test(name):
warn("{0:s} not found".format(name))
abort("requires({0:s}) failed".format(repr(names)))
def write_template_hook(ctx):
try:
write_template(filename % ctx, ctx, tpl_file=tpl_file)
except TemplateNotFound:
if warn_only:
warn(red("Couldn't find template %s" % tpl_file))
else:
abort(red("Couldn't find template %s" % tpl_file))
def generate_new_secret_key(key_length=32):
"""
Generate new secret key
"""
symbols = string.ascii_letters + str(string.digits) + '!@#$%^&*()+-'
secret_key = "".join(random.sample(symbols*2, key_length))
warn("New secret key: " + green(secret_key))
return secret_key
def rollback():
"""Rollback to the previous release."""
rm = ReleaseManager(env.remote_deploy_base)
previous = rm.get_previous_release()
if not previous:
warn("No previous release, cannot rollback!")
return
rm.set_current_release(previous)
def package(version=None, pkgtype='deb', env=defenv):
'Creates deployable packages'
if not version:
version = prompt("What version did you want packaged there, hotshot?")
if not env.exists('builddir'):
warn('Implicitly invoking build')
build(pkgtype, env)
pkg = packager.Package('package.ini', env, pkgtype)
pkg.build(version)
def stop_site(env='development', **kwargs):
sudo('service nginx stop')
projects = build_projects_vars()
project = projects[env]
with settings(hide('warnings'), warn_only=True):
result = sudo('service %s stop' % project['name'])
if result.failed:
warn( "%s was not running." % project['name'])
def __control_service(self, cmd):
fab_func = self._run_needs_sudo and sudo or run
with settings(hide('running', 'stdout'), warn_only=True):
result = fab_func("service %s %s" % (self._service_name, cmd))
if result.succeeded:
cmd += cmd == "stop" and "ped" or "ed"
print green("%s service %s" % (cmd, self._service_name))
else:
print red("Failed to %s service %s" % (cmd, self._service_name))
warn(red("This needs your attention!"))
def _param(name, adict):
"Get parameter value"
val = adict.get(name, '')
if val == '':
val = config.TPCONF_variable_defaults.get(name, '')
if val == '':
warn('Parameter %s is undefined' % name)
return val
def setup_db(env=None, force='0', testdata='1'):
"""Initialize the database
Requires that a virtual environment has been created, and is either
already activated, or specified, e.g.:
db:MY-ENV
To force initialization during development, by tearing down any existing
database, specify "force":
db:force=[1|true|yes|y]
In development, a test data fixture is loaded into the database by default; disable
this by specifying "testdata":
db:testdata=[0|false|no|n]
"""
roles = fab.env.roles or ['dev']
sql_path = join(BASEDIR, 'edgeflip', 'sql')
sql_context = {'DATABASE': 'edgeflip', 'USER': '******'}
password = None
# Database teardown
if 'dev' in roles:
password = os.environ.get('MYSQL_PWD') or fab.prompt("Enter mysql password:"******"{}"'.format(
password,
teardown_sql.format(**sql_context),
))
elif true(force):
fab.warn("Cannot force database set-up outside of development role {!r}"
.format(fab.env.roles))
return
# Database initialization
setup_sql = open(join(sql_path, 'setup.sql')).read()
setup_prepped = setup_sql.format(**sql_context)
if password is None:
l('mysql --user=root -p --execute="{}"'.format(setup_prepped))
else:
l('mysql --user=root --password={} --execute="{}"'.format(
password,
setup_prepped,
))
# Application schema initialization
manage('syncdb', flags=['migrate', 'noinput'], env=env)
# Load test data (dev):
if 'dev' in roles and true(testdata):
manage('loaddata', ['test_data'], env=env)
def deps():
if not env.is_remote:
warn('Can\'t execute task "install.dependencies" locally.')
return
pkgs = ['virtualenvwrapper', 'git', 'python-dev',
'postgresql-9.1', 'postgresql-9.1-postgis',
'postgresql-server-dev-9.1',
'libev4', 'libev-dev', 'libevent-2.0-5', 'libevent-dev'
]
sudo('apt-get update')
sudo('apt-get install {} -y'.format(' '.join(pkgs)))
def _check_repo_age():
if not os.path.exists(REPO_OUTDATED_FILE):
return
if time.time() - os.path.getmtime(REPO_OUTDATED_FILE) > REPO_OUTDATED_TIME:
repo = Repo(os.getcwd())
current_branch = repo.active_branch.name
if current_branch == 'master':
repo.remotes.origin.pull()
else:
warn('Your fabric-scripts may be out-of-date. Please `git pull` the repo')
def provision():
""" Install required software for EduDuck.
This will install global package requirements using apt and pip.
Note that vhost specific python packages will be installed via deploy().
(This will be done in the _update_virtualenv function via requirements file).
Run provision(), then deploy(), then possibly restore().
"""
# If MySQL is already installed, skip.
try:
run("dpkg -s mysql-server")
except:
passwd = prompt('Please enter MySQL root password, leave blank for auto generated password:'******'mysql-server mysql-server/root_password password {0}'".format(passwd)
sudo(debconf_cmd)
debconf_cmd = "debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password {0}'".format(passwd)
sudo(debconf_cmd)
if not exists('/home/chris/.my.cnf'):
run("touch /home/chris/.my.cnf")
run("chmod 640 /home/chris/.my.cnf")
run("echo [client] > /home/chris/.my.cnf")
run("echo user=root >> /home/chris/.my.cnf")
run("echo password='******' >> /home/chris/.my.cnf".format(passwd))
apt_packages = [
'python-virtualenv',
'python-pip',
'mysql-server',
'libmysqlclient-dev',
'python-dev',
'nginx',
'git',
]
pip_packages = [
'virtualenvwrapper',
]
apt_cmd = "apt-get install -y " + " ".join([pkg for pkg in apt_packages])
sudo(apt_cmd)
pip_cmd = "pip install " + " ".join([pkg for pkg in pip_packages])
sudo(pip_cmd)
try:
warn(yellow("NB MySQL root pw {0}".format(passwd)))
except:
print(green("Condition: Emerald"))
def manage(self, command):
"""
Runs django management command. Example::
fab manage:createsuperuser
"""
command_name = command.split()[0]
if not self.command_is_available(command_name):
warn('Management command "%s" is not available' % command_name)
else:
run('python manage.py ' + command)
