def npm():
with env.cd(env.dashboard):
env.run('npm install')
with env.cd(env.impressions):
env.run('npm install')
with env.cd(env.emails):
env.run('npm install')
def _install_ruby():
'''install ruby'''
ruby_file = _download_ruby()
if os.path.isfile('%s/bin/ruby/bin/ruby' % _get_pwd()):
print 'ruby Already installed.'
else:
with env.cd(os.path.dirname(ruby_file)):
env.run('tar xf %s' % os.path.basename(ruby_file))
with env.cd('ruby-%s' % ruby_version):
ruby_install_folder = '%s/bin/ruby' % _get_pwd()
if not os.path.isdir(ruby_install_folder):
os.makedirs(ruby_install_folder)
env.run('./configure --prefix="%s"' % ruby_install_folder)
env.run('make')
env.run('make install')
env.run('make clean')
assert os.path.isfile('%s/bin/ruby/bin/ruby' % _get_pwd())
## now lets install task juggler
print 'Installing taskjuggler'
if os.path.isfile('%s/bin/ruby/bin/tj3' % _get_pwd()):
print 'taskjuggler Already installed.'
else:
env.run('%s/bin/ruby/bin/gem install taskjuggler' % _get_pwd())
assert os.path.isfile('%s/bin/ruby/bin/tj3' % _get_pwd())
def gulp():
with env.cd(env.dashboard):
env.run('gulp build')
with env.cd(env.impressions):
env.run('gulp aware --%(conf_path)s' % env)
env.run('gulp guages --%(conf_path)s' % env)
with env.cd(env.emails):
env.run('gulp emails')
def bootstrap():
upload()
env.cd(env.path)
env.run('rm -rf {0}'.format(env.environment))
env.run('mkdir -p {0}'.format(env.environment))
env.run('{0} {1} --no-site-packages'.format(env.virtualenv,
env.environment))
update_requirements()
def _install_ffmpeg():
'''install ffmpeg'''
ffmpeg_file = _download_ffmpeg()
if os.path.isfile('%s/bin/ffmpeg/ffmpeg' % _get_pwd()):
print 'ffmpeg Already installed.'
else:
with env.cd(os.path.dirname(ffmpeg_file)):
ffmpeg_install_folder = '%s/bin/ffmpeg' % _get_pwd()
if not os.path.isdir(ffmpeg_install_folder):
os.makedirs(ffmpeg_install_folder)
env.run('tar xvfJ %s' % ffmpeg_file)
with env.cd('ffmpeg*'):
env.run('cp -rf * %s' % ffmpeg_install_folder)
assert os.path.isfile('%s/bin/ffmpeg/ffmpeg' % _get_pwd())
def _virtualenv():
"""
Changes to the proj_dir and activates the virtualenv
"""
with env.cd(env.proj_dir):
with prefix(env.activate):
yield
def manage(cmd=''):
"""
Remote plug of django manage.py
"""
_dynamic_env()
with env.cd(env.source_path):
env.run('%s manage.py %s' % (env.python, cmd))
def deploy(commit='HEAD'):
"""
Setup container and virtualenv if not exists
Upload configuration files for services
Reload services
"""
_dynamic_env()
setup()
archive_name = 'build.tar'
local('git archive %s > %s' % (commit, archive_name))
remote_archive_path = os.path.join(env.source_path, archive_name)
env.run('rm -Rf %(source_path)s' % env)
env.run("mkdir -p %(source_path)s" % env)
put(archive_name, remote_archive_path)
local('rm -f %s' % archive_name)
_stamp_commit(commit)
with env.cd(env.source_path):
env.run('tar xf %s' % archive_name)
env.run('rm -f %s' % archive_name)
env.run('cp %(settings_file)s %(local_settings)s' % env)
env.run('%(pip)s install -r requirements.txt' % env)
env.run('%(python)s manage.py syncdb --noinput' % env)
env.run('%(python)s manage.py migrate' % env)
env.run('%(python)s manage.py collectstatic --noinput' % env)
env.run('%(python)s manage.py compilemessages' % env)
reload()
def format():
"""Format all the code"""
with env.cd(env.base_path):
env.run(
'yapf -i -r --style config/style.yapf -e "*/migrations/*" -p muckrock'
)
env.run('isort -sp config -rc muckrock')
def compress_static_files():
"""
Run django compressor.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('python rnacentral/manage.py compress')
def debug_mode():
sudo('supervisorctl stop fearless-api')
with env.cd(os.path.join(_get_pwd(), 'src/api')):
#env.run('ls -la')
env.run('../../pyenv/bin/python flib/app.py')
sudo('supervisorctl start fearless-api')
def install_django_requirements():
"""
Run pip install.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('pip install --upgrade -r rnacentral/requirements.txt')
def collect_static_files():
"""
Run django `collectstatic` command.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('python rnacentral/manage.py collectstatic --noinput')
def _install_redis():
'''install redis'''
redis_file = _download_redis()
if os.path.isfile('%s/bin/redis/bin/redis-server' % _get_pwd()):
print 'redis Already installed.'
else:
with env.cd(os.path.dirname(redis_file)):
env.run('tar xf %s' % os.path.basename(redis_file))
with env.cd('redis-%s' % redis_version):
redis_install_folder = '%s/bin/redis' % _get_pwd()
if not os.path.isdir(redis_install_folder):
os.makedirs(redis_install_folder)
env.run('make')
env.run('make PREFIX="%s" install' % redis_install_folder)
env.run('make clean')
assert os.path.isfile('%s/bin/redis/bin/redis-server' % _get_pwd())
def celerybeat():
"""Run celery beat"""
with env.cd(env.base_path):
env.run(
DOCKER_COMPOSE_RUN_OPT.format(opt='--use-aliases',
service='celerybeat',
cmd=''))
def _virtualenv():
"""
Changes to the proj_dir and activates the virtualenv
"""
with env.cd(env.proj_dir):
with prefix(env.activate):
yield
def collect_static_files():
"""
Run django `collectstatic` command.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('python rnacentral/manage.py collectstatic --noinput')
def compress_static_files():
"""
Run django compressor.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('python rnacentral/manage.py compress')
def install_django_requirements():
"""
Run pip install.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('pip install --upgrade -r rnacentral/requirements.txt')
def runserver():
"""Run the test server"""
with env.cd(env.base_path):
env.run(
DOCKER_COMPOSE_RUN_OPT.format(opt='--service-ports --use-aliases',
service='django',
cmd=''))
def start_supervisor():
"""
Start supervisord and memcached on production machine.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('supervisord -c supervisor/supervisor.conf')
env.run('supervisorctl -c supervisor/supervisor.conf start memcached')
def restart_django(restart_url=None):
"""
Restart django process and visit the website.
"""
with env.cd(settings.PROJECT_PATH):
env.run('touch rnacentral/rnacentral/wsgi.py')
if restart_url:
requests.get(restart_url)
def start_supervisor():
"""
Start supervisord and memcached on production machine.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('supervisord -c supervisor/supervisor.conf')
env.run('supervisorctl -c supervisor/supervisor.conf start memcached')
def install_requirements():
"""
install the environment python packages
Usage:
fab <env> install_requirements
"""
with env.cd(env.project_root):
env.run('%(venv)s pip install -r requirements.txt' % env)
def virtualenv_setup():
"""
The third step
"""
env.run("/usr/bin/virtualenv --no-site-packages %(venv_root)s" % env)
with env.cd(env.project_root):
env.run("mkdir logs")
env.run("touch logs/error-django.log")
def update_envs():
"""
Updates local environment settings to the default repo settings, useful for parallel programming
Usage:
fab local update_envs
"""
with env.cd(env.project_root):
env.run('cp adomattic/conf/%(conf_path)s/settings.py adomattic/settings/local.py' % env)
def coverage(settings='test'):
"""Run the tests and generate a coverage report"""
with env.cd(env.base_path):
env.run('coverage erase')
env.run(
'coverage run --branch --source muckrock manage.py test --settings=muckrock.settings.%s'
% settings)
env.run('coverage html')
def restart_django(restart_url=None):
"""
Restart django process and visit the website.
"""
with env.cd(settings.PROJECT_PATH):
env.run('touch rnacentral/rnacentral/wsgi.py')
if restart_url:
requests.get(restart_url)
def coverage(settings='test', reuse='0'):
"""Run the tests and generate a coverage report"""
with env.cd(env.base_path):
env.run('coverage erase')
env.run(
'REUSE_DB=%s coverage run --branch --source muckrock --omit="*/migrations/*" manage.py test --settings=muckrock.settings.%s'
% (reuse, settings))
env.run('coverage html')
def test(base_url='http://localhost:8000/'):
"""
Single entry point for all tests.
"""
with env.cd(settings.PROJECT_PATH):
env.run('python rnacentral/apiv1/tests.py --base_url=%s' % base_url)
env.run('python rnacentral/portal/tests/selenium_tests.py --base_url %s --driver=phantomjs' % base_url) # pylint: disable=C0301
env.run('python rnacentral/apiv1/search/sequence/tests.py --base_url %s' % base_url) # pylint: disable=C0301
def get_ipdb():
"""
gets the latest ipdb file from maxmind
"""
with env.cd(env.project_root):
env.run('mkdir adomattic/ipdb')
env.run('wget -P adomattic/ipdb/ http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz')
env.run('gunzip adomattic/ipdb/GeoLite2-City.mmdb.gz')
def update_npm():
"""
Navigate to the folder with `package.json` and run npm update to install
static content dependencies.
"""
path = os.path.join(settings.PROJECT_PATH, 'rnacentral', 'portal', 'static')
with env.cd(path):
env.run('rm -f package-lock.json')
env.run('npm update --loglevel info')
def git_updates(git_branch=None):
"""
Perform git updates.
"""
with env.cd(settings.PROJECT_PATH):
if git_branch:
env.run('git checkout {branch}'.format(branch=git_branch))
env.run('git pull')
env.run('git submodule update')
def create_sitemaps():
"""
Create sitemaps cache in sitemaps folder.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('rm rnacentral/sitemaps/*')
env.run('python rnacentral/manage.py create_sitemaps')
slack("Created sitemaps at ves-oy-a4")
def git_updates(git_branch=None):
"""
Perform git updates.
"""
with env.cd(settings.PROJECT_PATH):
if git_branch:
env.run('git checkout {branch}'.format(branch=git_branch))
env.run('git pull')
env.run('git submodule update')
def update_npm():
"""
Navigate to the folder with `package.json` and run npm update to install
static content dependencies.
"""
path = os.path.join(settings.PROJECT_PATH, 'rnacentral', 'portal',
'static')
with env.cd(path):
env.run('npm update --loglevel info')
def create_sitemaps():
"""
Create sitemaps cache in sitemaps folder.
"""
with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \
prefix(COMMANDS['activate_virtualenv']):
env.run('rm rnacentral/sitemaps/*')
env.run('python rnacentral/manage.py create_sitemaps')
slack("Created sitemaps at ves-oy-a4")
def git_pull():
"""
pull from git
Usage:
fab <env> git_pull
"""
with env.cd(env.project_root):
env.run('git fetch' % env)
env.run('git checkout %(branch)s; git pull' % env)
env.run('git submodule update --init --recursive' % env)
def pylint():
"""Run pylint"""
with env.cd(env.base_path):
excludes = ['migrations', '__init__.py', 'manage.py',
'vendor', 'fabfile', 'static', 'node_modules']
stmt = ('find ./muckrock -name "*.py"' +
''.join(' | grep -v %s' % e for e in excludes) +
' | xargs pylint --load-plugins=pylint_django '
'--rcfile=config/pylint.conf -r n')
env.run(stmt)
def test(test_path='', reuse='0', capture=False):
"""Run all tests, or a specific subset of tests"""
cmd = ('REUSE_DB=%(reuse)s ./manage.py test %(test_path)s %(capture)s '
'--settings=muckrock.settings.test' % {
'reuse': reuse,
'test_path': test_path,
'capture': '--nologcapture' if not capture else ''
})
with env.cd(env.base_path):
env.run(cmd)
def stop_task(force=False):
"""
obi stop
"""
with env.cd(env.project_dir):
# fall-back to on-stop-cmds for backwards compatibility
# TODO(jshrake): remove support for the amiguous on-stop-cmds key
for cmd in env.config.get("pre-stop-cmds",
env.config.get("on-stop-cmds", [])):
env.run(cmd)
with env.cd(env.project_dir):
for cmd in env.config.get("local-pre-stop-cmds", []):
local(cmd)
# target_regex = find_launch_target()
# why this funny construct? to be extremely specific about what our regex is
# when run on a remote room, we want to match only `obi go room` invocations
# of our current project, but launched by any user
# if target_regex.startswith(default_remote_project_folder()):
# target_regex = target_regex.replace(default_remote_project_folder(),
# default_remote_project_folder().replace(
# env.local_user,
# "[a-z_][a-z0-9_]{0,30}"))
signal = "SIGTERM"
if force:
signal = "SIGKILL"
# temporarily ignore above code and issue signal to env.target_name because
# target_regex won't hit webthing-enabled projects due to shell wrapper
if env.target_name:
default_stop = "pkill -{0} -f '[a-z/]+{1}([[:space:]]|$)' || true".format(
signal, env.target_name)
else:
default_stop = "echo 'no pkill command issued because target=\"\"'"
stop_cmd = env.config.get("stop-cmd", default_stop)
env.run(stop_cmd)
with env.cd(env.project_dir):
for cmd in env.config.get("post-stop-cmds", []):
env.run(cmd)
with env.cd(env.project_dir):
for cmd in env.config.get("local-post-stop-cmds", []):
local(cmd)
def coverage_(settings='test', reuse='0'):
"""Run the tests and generate a coverage report"""
cmd = DOCKER_COMPOSE_RUN_OPT_USER.format(
opt='-e REUSE_DB={reuse}'.format(reuse=reuse),
service='django',
cmd='sh -c \'coverage erase && '
'coverage run --branch --source muckrock --omit="*/migrations/*" '
'manage.py test --settings=muckrock.settings.{settings} && '
'coverage html -i\''.format(settings=settings))
with env.cd(env.base_path):
env.run(cmd)
def stop_task():
"""
obi stop
"""
with env.cd(env.project_dir):
# fall-back to on-stop-cmds for backwards compatibility
# TODO(jshrake): remove support for the amiguous on-stop-cmds key
for cmd in env.config.get("pre-stop-cmds", env.config.get("on-stop-cmds", [])):
env.run(cmd)
with env.cd(env.project_dir):
for cmd in env.config.get("local-pre-stop-cmds", []):
local(cmd)
default_stop = "pkill -SIGINT -f '[a-z/]+{0} .*' || true".format(env.target_name)
stop_cmd = env.config.get("stop-cmd", default_stop)
env.run(stop_cmd)
with env.cd(env.project_dir):
for cmd in env.config.get("post-stop-cmds", []):
env.run(cmd)
with env.cd(env.project_dir):
for cmd in env.config.get("local-post-stop-cmds", []):
local(cmd)
def launch_task(debugger, extras):
"""
Handles launching the application in obi go
"""
target = ""
# Did the user specify a target?
config_target = env.config.get("target", None)
if config_target:
target = os.path.join(env.project_dir, config_target)
# TODO(jshrake): Consider nesting these conditionals
# Look for a binary with name env.target_name in the build directory
if not env.file_exists(target):
target = os.path.join(env.build_dir, env.target_name)
# Look for a binary with name env.target_name in the binary directory
if not env.file_exists(target):
target = os.path.join(env.project_dir, "bin", env.target_name)
# Just give up -- can't find the target name
if not env.file_exists(target):
abort("Cannot find target binary to launch. Please specify the relative path to the binary via the target key")
launch_args = env.config.get("launch-args", [])
formatted_launch = "{0} {1} {2}".format(
env.relpath(target), # {0}
" ".join(extras), # {1}
" ".join(launch_args) # {2}
)
env_vars = env.config.get("env-vars", {})
env_vars = " ".join(["{0}={1}".format(key, val) for key, val in env_vars.items()])
with env.cd(env.project_dir):
# Process pre-launch commands
for cmd in env.config.get("pre-launch-cmds", []):
env.run(cmd)
if debugger:
debug_cmd = debugger
debuggers = env.config.get("debuggers", None)
if debuggers:
debug_cmd = debuggers.get(debugger, debug_cmd)
default_launch = env.debug_launch_format_str.format(env_vars, debug_cmd, formatted_launch)
launch_cmd = env.config.get("debug-launch-cmd", default_launch)
env.background_run(launch_cmd)
else:
log_file = env.relpath(os.path.join(env.project_dir, env.target_name + ".log"))
default_launch = env.launch_format_str.format(env_vars, formatted_launch, log_file)
launch_cmd = env.config.get("launch-cmd", default_launch)
env.background_run(launch_cmd)
# Process the post-launch commands
for cmd in env.config.get("post-launch-cmds", []):
env.run(cmd)
def fetch_task(fetch_files_to_dir, files):
"""
obi fetch
"""
fetch_dir = fetch_files_to_dir + '/%(host)s/%(path)s'
files_to_fetch = files or env.config.get("fetch", [])
with env.cd(env.project_dir):
for f in files_to_fetch:
try:
fabric.operations.get(f, fetch_dir)
# dont fail when f doesn't exist on the remote machines
except:
continue
def deploy_locally(git_branch=None, restart_url='https://rnacentral.org', quick=False, compress=True):
"""
Run deployment locally.
"""
slack("Starting deployment of '%s' at ves-hx-a4" % git_branch)
git_updates(git_branch)
if not quick:
update_npm()
collect_static_files()
if compress:
compress_static_files()
if not quick:
install_django_requirements()
flush_memcached()
restart_django(restart_url)
if not git_branch:
with env.cd(settings.PROJECT_PATH):
git_branch = env.run('git rev-parse --abbrev-ref HEAD', capture=True) # env.run == local, takes capture arg
slack("Deployed '%s' at ves-hx-a4: <https://test.rnacentral.org|test.rnacentral.org>" % git_branch)
def deploy_remotely(git_branch=None, restart_url='https://rnacentral.org', quick=False, compress=True):
"""
Run deployment remotely.
"""
slack("Starting deployment of '%s' at %s" % (git_branch, env.host))
git_updates(git_branch)
# on PG machine npm was unable to download node_modules, so we rsync them from OY
if not quick:
if env.host == 'ves-pg-a4':
cmd = 'rsync -av {path}/ {host}:{path}'.format(
path=os.path.join(
settings.PROJECT_PATH,
'rnacentral',
'portal',
'static',
'node_modules'
),
host='ves-pg-a4',
)
local(cmd)
else:
update_npm()
if not quick:
rsync_local_files()
collect_static_files()
if compress:
compress_static_files()
flush_memcached()
restart_django(restart_url)
if not git_branch:
with env.cd(settings.PROJECT_PATH):
git_branch = env.run('git rev-parse --abbrev-ref HEAD')
slack("Deployed '%s' at %s: <https://rnacentral.org|rnacentral.org>" % (git_branch, env.host))
def update():
with env.cd(env.directory):
env.run('hg pull')
env.run('hg update')
def virtualenv():
with env.cd(env.directory):
with prefix(env.activate):
yield
def restart():
"restarts the apache django daemon"
with env.cd(env.directory):
env.run('touch ./freieit/apache2/django.wsgi')
def make_venv():
with env.cd(env.directory):
env.run('virtualenv --system-site-packages -p %s %s' % (env.pyexecutable, env.venvpath))
def clean():
with env.cd(env.directory):
env.run('rm -f _freieit.db')
env.run('rm -rf _media/')
def update_code():
"""
Pulls changes from the central repo and checks out the right branch
"""
with env.cd(env.proj_dir):
env.run('git pull && git checkout %s' % env.branch)
def make_images():
with env.cd(op.join(PROJECT_ROOT, 'tools')):
env.run('docker build -t %s apache/' % imgname('apache'))
env.run('docker pull mariadb')
