def _change_cwd(which, path):
path = path.replace(' ', '\ ')
if env.get(which) and not path.startswith('/'):
new_cwd = env.get(which) + '/' + path
else:
new_cwd = path
return _setenv(**{which: new_cwd})
def rq_dashboard():
"""Starts the RQ Dashboard webserver"""
# Stop the web process
stop('web')
env = _read_env()
with lcd(root_dir):
local('rq-dashboard -p %s -u %s' % (env.get('PORT', '9181'), env.get('REDIS_URL', 'redis://localhost:6379')))
def run(self):
self.JENKINS_URL = env.get('JENKINS_URL', '').strip('/')
if not self.JENKINS_URL:
abort(red('Provide fabric env variable JENKINS_URL pointing to your jenkins instance.'))
JENKINS_LOGIN = env.get('JENKINS_LOGIN', '')
JENKINS_PASSWORD = env.get('JENKINS_PASSWORD', '')
if not all([JENKINS_LOGIN, JENKINS_PASSWORD]):
abort(red('Provide both JENKINS_LOGIN and JENKINS_PASSWORD to connect to api'))
self.api = jenkins.Jenkins(self.JENKINS_URL, JENKINS_LOGIN, JENKINS_PASSWORD)
def notify_hipchat():
"""
Send notification to a HipChat room
"""
if env.get('hipchat_token', False) and env.get('hipchat_room_id', False) and not env.dry_run:
hp = HipChatNotifier(env.hipchat_token)
name = helpers.capture('git config user.name')
if name is '':
name = 'Someone'
message = '%s just deployed "%s" (branch: %s) to %s' % (
name, env.project_name, env.branch, env.settings)
hp.message(env.hipchat_room_id, 'Deployment', message, True)
def connect(self, *args, **kwargs):
REDMINE_URL = env.get('REDMINE_URL', '').strip('/')
if not REDMINE_URL:
abort(red('Provide REDMINE_URL pointing to your redmine instance'))
REDMINE_API_KEY = env.get('REDMINE_API_KEY', '')
if not REDMINE_API_KEY:
abort(red('Go to %s/my/account and grab yourself a api key' % REDMINE_URL))
self.api = Hammock(env.REDMINE_URL,
headers={'Content-Type': 'application/json',
'X-Redmine-API-Key': env.REDMINE_API_KEY})
def run(cls, cmd, *args, **kwargs):
"""
Unified command execution - calls fabric's local(), run() or sudo operation() as required.
Caller needs to set env via settings().
:param cmd: the command to execute
"""
from fabric.state import env
from fabric.operations import run, sudo, local
if env.get("host_string").endswith('@localhost'):
return local(cmd, capture=True)
if env.get("sudo_user") in (None, env.get("user")):
return run(cmd, *args, **kwargs)
else:
return sudo(cmd, *args, **kwargs)
def __getitem__(self, item):
"""
Returns value of configuration key
based on current host or global section
For example, if current task is connected to 10.12.43.2
and it can be found from config::
[DEFAULT]
mykey = 'defaultval'
[setup:testenv]
fe_hosts = 10.12.43.2
db_hosts = 10.12.43.2
mc_hosts = 10.12.43.2
mykey = 'myvalue'
Then:
>>> config = Config()
>>> config['mykey']
'myvalue'
"""
# Default section where to read
section = "DEFAULT"
# Try to determine the section based on current host name
host_string = env.get("host_string", "")
if host_string:
default = {}
# Iterate roledefs and match hostname with them
for rolename, hosts in env.get("roledefs", default).items():
# Skip the layer specific roles
if rolename[-2:] in ["fe", "db", "mc"]:
pass
elif host_string in hosts:
section = "setup:%s" % rolename
# Check if option can be found and fallback to 'DEFAULT'
if not self.config.has_option(section, item):
logger.debug("Option cannot be found: %s, falling back" % item)
section = "DEFAULT"
value = self.config.get(section, item)
logger.debug("Reading config [%s] %s = %s" % (section, item, value))
return value
def _render_graphics(paths):
"""
Render a set of graphics
"""
# Fake out deployment target
app_config.configure_targets(env.get('settings', None))
for path in paths:
slug = path.split('%s/' % app_config.GRAPHICS_PATH)[1].split('/')[0]
with flat_app.app.test_request_context(path='graphics/%s/' % slug):
view = flat_app.__dict__['_graphics_detail']
content = view(slug).data
with open('%s/index.html' % path, 'w') as writefile:
writefile.write(content)
# Fallback for legacy projects w/o child templates
if not os.path.exists('%s/child_template.html' % path):
continue
download_copy(slug)
with flat_app.app.test_request_context(path='graphics/%s/child.html' % slug):
view = flat_app.__dict__['_graphics_child']
content = view(slug).data
with open('%s/child.html' % path, 'w') as writefile:
writefile.write(content)
# Un-fake-out deployment target
app_config.configure_targets(app_config.DEPLOYMENT_TARGET)
def render_all():
"""
Render HTML templates and compile assets.
"""
from flask import g
require('slug', provided_by=['post'])
less()
app_config_js()
copytext_js(env.slug)
compiled_includes = {}
app_config.configure_targets(env.get('settings', None))
with app.app.test_request_context():
path = 'posts/%s/www/index.html' % env.slug
with app.app.test_request_context(path=env.static_path):
print 'Rendering %s' % path
g.compile_includes = True
g.compiled_includes = compiled_includes
view = app.__dict__['_post']
content = view(env.slug).data
with open(path, 'w') as f:
f.write(content)
def host_prompting_wrapper(*args, **kwargs):
handle_prompt_abort()
while not env.get('host_string', False):
host_string = raw_input("No hosts found. Please specify (single)"
" host string for connection: ")
interpret_host_string(host_string)
return func(*args, **kwargs)
def render():
"""
Render the Tumblr theme.
"""
from flask import g
less()
app_config_js()
compiled_includes = {}
app_config.configure_targets(env.get('settings', None))
path = 'theme/www/index.html'
with app.app.test_request_context(path='/theme'):
print 'Rendering %s' % path
if env.settings not in ['staging', 'production']:
g.compile_includes = False
else:
g.compile_includes = True
g.compiled_includes = compiled_includes
view = static_theme.__dict__['_theme']
content = view()
with open(path, 'w') as f:
f.write(content.encode('utf-8'))
local('pbcopy < theme/www/index.html')
print 'The Tumblr theme HTML has been copied to your clipboard.'
local('open https://www.tumblr.com/customize/%s' % app_config.TUMBLR_NAME)
def notify_event(commits=None):
"""
Send a message to slack about a successful deployment
:return str: formatted message
"""
from .project import project_name, git_repository_path, github_link
msg = u'*{project}* (*{state}*)'
if commits:
msg += u' deployed `<{base_url}/compare/{old}...{new}|{old} → {new}>`'
else:
msg += u' reset to `<{base_url}/commit/{commit}|{commit}>`'
msg += u' on `{host}`'
old_commit, new_commit = commits or (None, None)
commit = commits or git.get_commit(repository_path=git_repository_path(), short=True)
msg = msg.format(
project=project_name(),
base_url=github_link(),
state=env.get('state', 'unknown'),
old=old_commit,
new=new_commit,
commit=commit,
host=env['host_string']
)
slack.notify(msg)
return msg
def _deploy_summary(title, revision):
from hashlib import md5
from time import time
from .project import project_name
deployer = git.get_local_commiter()
email = git.get_local_email()
project = project_name()
state = env.get('state', 'Unknown')
avatar_hash = md5(email.strip().lower()).hexdigest()
avatar_url = 'https://www.gravatar.com/avatar/{}?s=16'.format(avatar_hash)
fallback = "Deploy: {} ({}) by {}".format(project, state, deployer)
summary = {
'fallback': fallback,
'color': '#439FE0',
'author_name': deployer,
'author_icon': avatar_url,
# 'ts': int(time()),
'title': u'{} ({})'.format(project, state),
'fields': [
{'title': 'Label', 'value': title, 'short': True}
],
'mrkdwn_in': ['text', 'fields']
}
if revision:
summary['fields'].append({
'title': 'revision',
'value': '_{}_'.format(revision),
'short': True
})
return summary
def sitemap():
"""
Render and deploy sitemap.
"""
require('settings', provided_by=[staging, production])
app_config.configure_targets(env.get('settings', None))
with flat_app.app.test_request_context(path='sitemap.xml'):
print 'Rendering sitemap.xml'
view = flat_app.__dict__['_sitemap']
content = view().data
with open('.sitemap.xml', 'w') as f:
f.write(content)
s3 = boto.connect_s3()
flat.deploy_file(
s3,
'.sitemap.xml',
app_config.PROJECT_SLUG,
app_config.DEFAULT_MAX_AGE
)
def init():
"""
Setup the server for the first time
:return:
"""
banner("init")
with show("output"):
if not env.get('no_apt_update'):
sudo('apt-get update')
require.directory(env.path, mode="777", use_sudo=True)
require.directory('/var/run/jianguo/', owner='www-data', group='www-data', mode='770', use_sudo=True)
require.directory('/var/log/jianguo/', owner='www-data', group='www-data', mode='770', use_sudo=True)
require.directory('/var/log/supervisord/', owner='www-data', group='www-data', mode='770', use_sudo=True)
require.directory('/var/run/supervisord/', owner='www-data', group='www-data', mode='770', use_sudo=True)
require.directory('~/.ssh', mode='700')
put('deployment', '~/.ssh/id_rsa')
run('chmod 600 ~/.ssh/id_rsa')
require.deb.packages([
'gcc', 'python-all-dev', 'libpq-dev', 'libjpeg-dev', 'libxml2-dev', 'libxslt1-dev',
'libfreetype6-dev', 'libevent-dev', 'supervisor'
])
require.python.pip(version="1.0")
new_virtualenv()
me = run('whoami')
sudo('adduser %s www-data' % me)
install_nginx()
install_postgres()
def run(self, origin=None, version=None, build_dir=None, dest_dir=None, excludes=None):
if excludes is not None:
self.excludes = excludes
elif self.excludes is None:
self.excludes = env.get('%s_excludes' % self.prefix)
super(BuildJqueryTask, self).run(origin, version, build_dir, dest_dir)
def render():
from flask import g
require('static_path', provided_by=['tumblr'])
require('settings', provided_by=['staging', 'production', 'development'])
less()
app_config_js()
copytext_js('theme')
compiled_includes = {}
app_config.configure_targets(env.get('settings', None))
with app.app.test_request_context():
path = 'tumblr/www/index.html'
with app.app.test_request_context(path=env.static_path):
print 'Rendering %s' % path
if env.settings == 'development':
g.compile_includes = False
else:
g.compile_includes = True
g.compiled_includes = compiled_includes
view = static_theme.__dict__['_theme']
content = view().data
with open(path, 'w') as f:
f.write(content)
local('pbcopy < tumblr/www/index.html')
print 'The Tumblr theme HTML has been copied to your clipboard.'
local('open https://www.tumblr.com/customize/%s' % app_config.TUMBLR_NAME)
def host_prompting_wrapper(*args, **kwargs):
while not env.get('host_string', False):
handle_prompt_abort("the target host connection string")
host_string = raw_input("No hosts found. Please specify (single)"
" host string for connection: ")
env.update(to_dict(host_string))
return func(*args, **kwargs)
def local(command, capture=True):
"""
Run a command on the local system.
`local` is simply a convenience wrapper around the use of the builtin
Python ``subprocess`` module with ``shell=True`` activated. If you need to
do anything special, consider using the ``subprocess`` module directly.
`local` will, by default, capture and return the contents of the command's
stdout as a string, and will not print anything to the user (the command's
stderr is captured but discarded.)
.. note::
This differs from the default behavior of `run` and `sudo` due to the
different mechanisms involved: it is difficult to simultaneously
capture and print local commands, so we have to choose one or the
other. We hope to address this in later releases.
If you need full interactivity with the command being run (and are willing
to accept the loss of captured stdout) you may specify ``capture=False`` so
that the subprocess' stdout and stderr pipes are connected to your terminal
instead of captured by Fabric.
When ``capture`` is False, global output controls (``output.stdout`` and
``output.stderr`` will be used to determine what is printed and what is
discarded.
"""
# Handle cd() context manager
cwd = env.get('cwd', '')
if cwd:
cwd = 'cd %s && ' % _shell_escape(cwd)
# Construct real command
real_command = cwd + command
if output.debug:
print("[localhost] run: %s" % (real_command))
elif output.running:
print("[localhost] run: " + command)
# By default, capture both stdout and stderr
PIPE = subprocess.PIPE
out_stream = PIPE
err_stream = PIPE
# Tie in to global output controls as best we can; our capture argument
# takes precedence over the output settings.
if not capture:
if output.stdout:
out_stream = None
if output.stderr:
err_stream = None
p = subprocess.Popen([real_command], shell=True, stdout=out_stream,
stderr=err_stream)
(stdout, stderr) = p.communicate()
# Handle error condition (deal with stdout being None, too)
out = _AttributeString(stdout or "")
out.failed = False
if p.returncode != 0:
out.failed = True
msg = "local() encountered an error (return code %s) while executing '%s'" % (p.returncode, command)
_handle_failure(message=msg)
# If we were capturing, this will be a string; otherwise it will be None.
return out
def fabcast(command):
"""
Actually run specified commands on the server specified
by staging() or production().
"""
require('settings', provided_by=['production', 'staging'])
if not app_config.DEPLOY_TO_SERVERS:
print 'You must set DEPLOY_TO_SERVERS = True in your app_config.py and setup a server before fabcasting.'
if env.get('branch'):
branch = 'branch:%s' % env.get('branch')
else:
branch = ''
run('cd %s && bash run_on_server.sh fab %s $DEPLOYMENT_TARGET %s' % (app_config.SERVER_REPOSITORY_PATH, branch, command))
def run(command, shell=True, pty=True, combine_stderr=None, use_sudo=False, user=None):
use_sudo = use_sudo or user is not None or env.get('use_sudo')
if not use_sudo:
return fabric_run(command, shell=shell, pty=pty, combine_stderr=combine_stderr)
else:
user = user or env.get('sudo_user', env.user)
# Make SSH agent socket available to the sudo user
with silent():
fabric_sudo('chown -R {}: $(dirname $SSH_AUTH_SOCK)'.format(user), user='******')
if user == env.user:
user = None
return fabric_sudo(command, shell=shell, pty=pty, combine_stderr=combine_stderr, user=user)
def run(self, origin=None, version=None, build_dir=None, dest_dir=None, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = env.get('%s_config' % self.prefix)
super(BuildBootstrapTask, self).run(origin, version, build_dir, dest_dir)
def host_prompting_wrapper(*args, **kwargs):
env.prompt or abort(
"Needed to prompt for host, but env.prompt is False. "
"(You specified --abort-on-prompt)")
while not env.get('host_string', False):
host_string = raw_input("No hosts found. Please specify (single) host string for connection: ")
interpret_host_string(host_string)
return func(*args, **kwargs)
def run(self, job_name, project_slug, template_job=None):
super(CreateJob, self).run()
template_job = template_job or env.get('JENKINS_TEMPLATE_JOB', 'example-tests')
job = self.api.copy_job(template_job, job_name)
job.enable()
self.setup_job_config(job, context={'project': project_slug})
job.invoke()
puts("Created job %s/job/%s/" % (self.JENKINS_URL, job_name))
def jasmine():
"""Run the jasmine test server. Stops the web process first to open the port."""
stop('web')
env = _read_env()
with lcd(root_dir):
local('jasmine -p %s' % env.get('PORT', ''))
def app(port='8000'):
"""
Serve app.py.
"""
if env.get('settings'):
local("DEPLOYMENT_TARGET=%s bash -c 'gunicorn -b 0.0.0.0:%s --timeout 3600 --debug --reload --log-file=logs/app.log app:wsgi_app'" % (env.settings, port))
else:
local('gunicorn -b 0.0.0.0:%s --timeout 3600 --debug --reload --log-file=logs/app.log app:wsgi_app' % port)
def cd(path):
"""
Context manager that keeps directory state when calling `run`/`sudo`.
Any calls to `run` or `sudo` within the wrapped block will implicitly have
a string similar to ``"cd <path> && "`` prefixed in order to give the sense
that there is actually statefulness involved.
Because use of `cd` affects all `run` and `sudo` invocations, any code
making use of `run` and/or `sudo`, such as much of the ``contrib`` section,
will also be affected by use of `cd`. However, at this time, `get` and
`put` do not honor `cd`; we expect this to be fixed in future releases.
Like the actual 'cd' shell builtin, `cd` may be called with relative paths
(keep in mind that your default starting directory is your remote user's
``$HOME``) and may be nested as well.
Below is a "normal" attempt at using the shell 'cd', which doesn't work due
to how shell-less SSH connections are implemented -- state is **not** kept
between invocations of `run` or `sudo`::
run('cd /var/www')
run('ls')
The above snippet will list the contents of the remote user's ``$HOME``
instead of ``/var/www``. With `cd`, however, it will work as expected::
with cd('/var/www'):
run('ls') # Turns into "cd /var/www && ls"
Finally, a demonstration (see inline comments) of nesting::
with cd('/var/www'):
run('ls') # cd /var/www && ls
with cd('website1'):
run('ls') # cd /var/www/website1 && ls
.. note::
This context manager is currently implemented by appending to (and, as
always, restoring afterwards) the current value of an environment
variable, ``env.cwd``. However, this implementation may change in the
future, so we do not recommend manually altering ``env.cwd`` -- only
the *behavior* of `cd` will have any guarantee of backwards
compatibility.
.. note::
Space characters will be escaped automatically to make dealing with
such directory names easier.
"""
path = path.replace(' ', '\ ')
if env.get('cwd') and not path.startswith('/'):
new_cwd = env.cwd + '/' + path
else:
new_cwd = path
return _setenv(cwd=new_cwd)
def install_minimal():
"""Installs minimal dependencies"""
sudo("apt-get update")
sudo("apt-get install --yes --force-yes {0}".format(MINIMAL_PACKAGES))
if env.get("extra_packages"):
sudo("apt-get install --yes --force-yes {0}".format(env.extra_packages))
def run(self, project_slug, developer):
config_name = self.get_server_name(project_slug, developer)
config_available_path = '/etc/uwsgi/apps-available/%s.ini' % config_name
config_enabled_path = '/etc/uwsgi/apps-enabled-%s/%s.ini' % (env.get('PYTHON_VERSION', '2.7'), config_name)
upload_template(
template_dir=self.get_template_path(),
filename=env.get('UWSGI_CONFIG_TEMPLATE', 'uwsgi.config.tmpl'),
destination=config_available_path,
context=self.get_context(project_slug, developer),
use_sudo=True,
use_jinja=True,
backup=False
)
sudo('ln -fs %(available)s %(enabled)s' % {
'available': config_available_path,
'enabled': config_enabled_path
})
def dev_web():
"""Runs the Django development webserver"""
# Stop the gunicorn process
stop('web')
env = _read_env()
manage('runserver', env.get('PORT', ''))
