def npm(): with env.cd(env.dashboard): env.run('npm install') with env.cd(env.impressions): env.run('npm install') with env.cd(env.emails): env.run('npm install')
def _install_ruby(): '''install ruby''' ruby_file = _download_ruby() if os.path.isfile('%s/bin/ruby/bin/ruby' % _get_pwd()): print 'ruby Already installed.' else: with env.cd(os.path.dirname(ruby_file)): env.run('tar xf %s' % os.path.basename(ruby_file)) with env.cd('ruby-%s' % ruby_version): ruby_install_folder = '%s/bin/ruby' % _get_pwd() if not os.path.isdir(ruby_install_folder): os.makedirs(ruby_install_folder) env.run('./configure --prefix="%s"' % ruby_install_folder) env.run('make') env.run('make install') env.run('make clean') assert os.path.isfile('%s/bin/ruby/bin/ruby' % _get_pwd()) ## now lets install task juggler print 'Installing taskjuggler' if os.path.isfile('%s/bin/ruby/bin/tj3' % _get_pwd()): print 'taskjuggler Already installed.' else: env.run('%s/bin/ruby/bin/gem install taskjuggler' % _get_pwd()) assert os.path.isfile('%s/bin/ruby/bin/tj3' % _get_pwd())
def gulp(): with env.cd(env.dashboard): env.run('gulp build') with env.cd(env.impressions): env.run('gulp aware --%(conf_path)s' % env) env.run('gulp guages --%(conf_path)s' % env) with env.cd(env.emails): env.run('gulp emails')
def bootstrap(): upload() env.cd(env.path) env.run('rm -rf {0}'.format(env.environment)) env.run('mkdir -p {0}'.format(env.environment)) env.run('{0} {1} --no-site-packages'.format(env.virtualenv, env.environment)) update_requirements()
def _install_ffmpeg(): '''install ffmpeg''' ffmpeg_file = _download_ffmpeg() if os.path.isfile('%s/bin/ffmpeg/ffmpeg' % _get_pwd()): print 'ffmpeg Already installed.' else: with env.cd(os.path.dirname(ffmpeg_file)): ffmpeg_install_folder = '%s/bin/ffmpeg' % _get_pwd() if not os.path.isdir(ffmpeg_install_folder): os.makedirs(ffmpeg_install_folder) env.run('tar xvfJ %s' % ffmpeg_file) with env.cd('ffmpeg*'): env.run('cp -rf * %s' % ffmpeg_install_folder) assert os.path.isfile('%s/bin/ffmpeg/ffmpeg' % _get_pwd())
def _virtualenv(): """ Changes to the proj_dir and activates the virtualenv """ with env.cd(env.proj_dir): with prefix(env.activate): yield
def manage(cmd=''): """ Remote plug of django manage.py """ _dynamic_env() with env.cd(env.source_path): env.run('%s manage.py %s' % (env.python, cmd))
def deploy(commit='HEAD'): """ Setup container and virtualenv if not exists Upload configuration files for services Reload services """ _dynamic_env() setup() archive_name = 'build.tar' local('git archive %s > %s' % (commit, archive_name)) remote_archive_path = os.path.join(env.source_path, archive_name) env.run('rm -Rf %(source_path)s' % env) env.run("mkdir -p %(source_path)s" % env) put(archive_name, remote_archive_path) local('rm -f %s' % archive_name) _stamp_commit(commit) with env.cd(env.source_path): env.run('tar xf %s' % archive_name) env.run('rm -f %s' % archive_name) env.run('cp %(settings_file)s %(local_settings)s' % env) env.run('%(pip)s install -r requirements.txt' % env) env.run('%(python)s manage.py syncdb --noinput' % env) env.run('%(python)s manage.py migrate' % env) env.run('%(python)s manage.py collectstatic --noinput' % env) env.run('%(python)s manage.py compilemessages' % env) reload()
def format(): """Format all the code""" with env.cd(env.base_path): env.run( 'yapf -i -r --style config/style.yapf -e "*/migrations/*" -p muckrock' ) env.run('isort -sp config -rc muckrock')
def compress_static_files(): """ Run django compressor. """ with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \ prefix(COMMANDS['activate_virtualenv']): env.run('python rnacentral/manage.py compress')
def debug_mode(): sudo('supervisorctl stop fearless-api') with env.cd(os.path.join(_get_pwd(), 'src/api')): #env.run('ls -la') env.run('../../pyenv/bin/python flib/app.py') sudo('supervisorctl start fearless-api')
def install_django_requirements(): """ Run pip install. """ with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \ prefix(COMMANDS['activate_virtualenv']): env.run('pip install --upgrade -r rnacentral/requirements.txt')
def collect_static_files(): """ Run django `collectstatic` command. """ with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \ prefix(COMMANDS['activate_virtualenv']): env.run('python rnacentral/manage.py collectstatic --noinput')
def _install_redis(): '''install redis''' redis_file = _download_redis() if os.path.isfile('%s/bin/redis/bin/redis-server' % _get_pwd()): print 'redis Already installed.' else: with env.cd(os.path.dirname(redis_file)): env.run('tar xf %s' % os.path.basename(redis_file)) with env.cd('redis-%s' % redis_version): redis_install_folder = '%s/bin/redis' % _get_pwd() if not os.path.isdir(redis_install_folder): os.makedirs(redis_install_folder) env.run('make') env.run('make PREFIX="%s" install' % redis_install_folder) env.run('make clean') assert os.path.isfile('%s/bin/redis/bin/redis-server' % _get_pwd())
def celerybeat(): """Run celery beat""" with env.cd(env.base_path): env.run( DOCKER_COMPOSE_RUN_OPT.format(opt='--use-aliases', service='celerybeat', cmd=''))
def runserver(): """Run the test server""" with env.cd(env.base_path): env.run( DOCKER_COMPOSE_RUN_OPT.format(opt='--service-ports --use-aliases', service='django', cmd=''))
def start_supervisor(): """ Start supervisord and memcached on production machine. """ with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \ prefix(COMMANDS['activate_virtualenv']): env.run('supervisord -c supervisor/supervisor.conf') env.run('supervisorctl -c supervisor/supervisor.conf start memcached')
def restart_django(restart_url=None): """ Restart django process and visit the website. """ with env.cd(settings.PROJECT_PATH): env.run('touch rnacentral/rnacentral/wsgi.py') if restart_url: requests.get(restart_url)
def install_requirements(): """ install the environment python packages Usage: fab <env> install_requirements """ with env.cd(env.project_root): env.run('%(venv)s pip install -r requirements.txt' % env)
def virtualenv_setup(): """ The third step """ env.run("/usr/bin/virtualenv --no-site-packages %(venv_root)s" % env) with env.cd(env.project_root): env.run("mkdir logs") env.run("touch logs/error-django.log")
def update_envs(): """ Updates local environment settings to the default repo settings, useful for parallel programming Usage: fab local update_envs """ with env.cd(env.project_root): env.run('cp adomattic/conf/%(conf_path)s/settings.py adomattic/settings/local.py' % env)
def coverage(settings='test'): """Run the tests and generate a coverage report""" with env.cd(env.base_path): env.run('coverage erase') env.run( 'coverage run --branch --source muckrock manage.py test --settings=muckrock.settings.%s' % settings) env.run('coverage html')
def coverage(settings='test', reuse='0'): """Run the tests and generate a coverage report""" with env.cd(env.base_path): env.run('coverage erase') env.run( 'REUSE_DB=%s coverage run --branch --source muckrock --omit="*/migrations/*" manage.py test --settings=muckrock.settings.%s' % (reuse, settings)) env.run('coverage html')
def test(base_url='http://localhost:8000/'): """ Single entry point for all tests. """ with env.cd(settings.PROJECT_PATH): env.run('python rnacentral/apiv1/tests.py --base_url=%s' % base_url) env.run('python rnacentral/portal/tests/selenium_tests.py --base_url %s --driver=phantomjs' % base_url) # pylint: disable=C0301 env.run('python rnacentral/apiv1/search/sequence/tests.py --base_url %s' % base_url) # pylint: disable=C0301
def get_ipdb(): """ gets the latest ipdb file from maxmind """ with env.cd(env.project_root): env.run('mkdir adomattic/ipdb') env.run('wget -P adomattic/ipdb/ http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz') env.run('gunzip adomattic/ipdb/GeoLite2-City.mmdb.gz')
def update_npm(): """ Navigate to the folder with `package.json` and run npm update to install static content dependencies. """ path = os.path.join(settings.PROJECT_PATH, 'rnacentral', 'portal', 'static') with env.cd(path): env.run('rm -f package-lock.json') env.run('npm update --loglevel info')
def git_updates(git_branch=None): """ Perform git updates. """ with env.cd(settings.PROJECT_PATH): if git_branch: env.run('git checkout {branch}'.format(branch=git_branch)) env.run('git pull') env.run('git submodule update')
def create_sitemaps(): """ Create sitemaps cache in sitemaps folder. """ with env.cd(settings.PROJECT_PATH), prefix(COMMANDS['set_environment']), \ prefix(COMMANDS['activate_virtualenv']): env.run('rm rnacentral/sitemaps/*') env.run('python rnacentral/manage.py create_sitemaps') slack("Created sitemaps at ves-oy-a4")
def update_npm(): """ Navigate to the folder with `package.json` and run npm update to install static content dependencies. """ path = os.path.join(settings.PROJECT_PATH, 'rnacentral', 'portal', 'static') with env.cd(path): env.run('npm update --loglevel info')
def git_pull(): """ pull from git Usage: fab <env> git_pull """ with env.cd(env.project_root): env.run('git fetch' % env) env.run('git checkout %(branch)s; git pull' % env) env.run('git submodule update --init --recursive' % env)
def pylint(): """Run pylint""" with env.cd(env.base_path): excludes = ['migrations', '__init__.py', 'manage.py', 'vendor', 'fabfile', 'static', 'node_modules'] stmt = ('find ./muckrock -name "*.py"' + ''.join(' | grep -v %s' % e for e in excludes) + ' | xargs pylint --load-plugins=pylint_django ' '--rcfile=config/pylint.conf -r n') env.run(stmt)
def test(test_path='', reuse='0', capture=False): """Run all tests, or a specific subset of tests""" cmd = ('REUSE_DB=%(reuse)s ./manage.py test %(test_path)s %(capture)s ' '--settings=muckrock.settings.test' % { 'reuse': reuse, 'test_path': test_path, 'capture': '--nologcapture' if not capture else '' }) with env.cd(env.base_path): env.run(cmd)
def stop_task(force=False): """ obi stop """ with env.cd(env.project_dir): # fall-back to on-stop-cmds for backwards compatibility # TODO(jshrake): remove support for the amiguous on-stop-cmds key for cmd in env.config.get("pre-stop-cmds", env.config.get("on-stop-cmds", [])): env.run(cmd) with env.cd(env.project_dir): for cmd in env.config.get("local-pre-stop-cmds", []): local(cmd) # target_regex = find_launch_target() # why this funny construct? to be extremely specific about what our regex is # when run on a remote room, we want to match only `obi go room` invocations # of our current project, but launched by any user # if target_regex.startswith(default_remote_project_folder()): # target_regex = target_regex.replace(default_remote_project_folder(), # default_remote_project_folder().replace( # env.local_user, # "[a-z_][a-z0-9_]{0,30}")) signal = "SIGTERM" if force: signal = "SIGKILL" # temporarily ignore above code and issue signal to env.target_name because # target_regex won't hit webthing-enabled projects due to shell wrapper if env.target_name: default_stop = "pkill -{0} -f '[a-z/]+{1}([[:space:]]|$)' || true".format( signal, env.target_name) else: default_stop = "echo 'no pkill command issued because target=\"\"'" stop_cmd = env.config.get("stop-cmd", default_stop) env.run(stop_cmd) with env.cd(env.project_dir): for cmd in env.config.get("post-stop-cmds", []): env.run(cmd) with env.cd(env.project_dir): for cmd in env.config.get("local-post-stop-cmds", []): local(cmd)
def coverage_(settings='test', reuse='0'): """Run the tests and generate a coverage report""" cmd = DOCKER_COMPOSE_RUN_OPT_USER.format( opt='-e REUSE_DB={reuse}'.format(reuse=reuse), service='django', cmd='sh -c \'coverage erase && ' 'coverage run --branch --source muckrock --omit="*/migrations/*" ' 'manage.py test --settings=muckrock.settings.{settings} && ' 'coverage html -i\''.format(settings=settings)) with env.cd(env.base_path): env.run(cmd)
def stop_task(): """ obi stop """ with env.cd(env.project_dir): # fall-back to on-stop-cmds for backwards compatibility # TODO(jshrake): remove support for the amiguous on-stop-cmds key for cmd in env.config.get("pre-stop-cmds", env.config.get("on-stop-cmds", [])): env.run(cmd) with env.cd(env.project_dir): for cmd in env.config.get("local-pre-stop-cmds", []): local(cmd) default_stop = "pkill -SIGINT -f '[a-z/]+{0} .*' || true".format(env.target_name) stop_cmd = env.config.get("stop-cmd", default_stop) env.run(stop_cmd) with env.cd(env.project_dir): for cmd in env.config.get("post-stop-cmds", []): env.run(cmd) with env.cd(env.project_dir): for cmd in env.config.get("local-post-stop-cmds", []): local(cmd)
def launch_task(debugger, extras): """ Handles launching the application in obi go """ target = "" # Did the user specify a target? config_target = env.config.get("target", None) if config_target: target = os.path.join(env.project_dir, config_target) # TODO(jshrake): Consider nesting these conditionals # Look for a binary with name env.target_name in the build directory if not env.file_exists(target): target = os.path.join(env.build_dir, env.target_name) # Look for a binary with name env.target_name in the binary directory if not env.file_exists(target): target = os.path.join(env.project_dir, "bin", env.target_name) # Just give up -- can't find the target name if not env.file_exists(target): abort("Cannot find target binary to launch. Please specify the relative path to the binary via the target key") launch_args = env.config.get("launch-args", []) formatted_launch = "{0} {1} {2}".format( env.relpath(target), # {0} " ".join(extras), # {1} " ".join(launch_args) # {2} ) env_vars = env.config.get("env-vars", {}) env_vars = " ".join(["{0}={1}".format(key, val) for key, val in env_vars.items()]) with env.cd(env.project_dir): # Process pre-launch commands for cmd in env.config.get("pre-launch-cmds", []): env.run(cmd) if debugger: debug_cmd = debugger debuggers = env.config.get("debuggers", None) if debuggers: debug_cmd = debuggers.get(debugger, debug_cmd) default_launch = env.debug_launch_format_str.format(env_vars, debug_cmd, formatted_launch) launch_cmd = env.config.get("debug-launch-cmd", default_launch) env.background_run(launch_cmd) else: log_file = env.relpath(os.path.join(env.project_dir, env.target_name + ".log")) default_launch = env.launch_format_str.format(env_vars, formatted_launch, log_file) launch_cmd = env.config.get("launch-cmd", default_launch) env.background_run(launch_cmd) # Process the post-launch commands for cmd in env.config.get("post-launch-cmds", []): env.run(cmd)
def fetch_task(fetch_files_to_dir, files): """ obi fetch """ fetch_dir = fetch_files_to_dir + '/%(host)s/%(path)s' files_to_fetch = files or env.config.get("fetch", []) with env.cd(env.project_dir): for f in files_to_fetch: try: fabric.operations.get(f, fetch_dir) # dont fail when f doesn't exist on the remote machines except: continue
def deploy_locally(git_branch=None, restart_url='https://rnacentral.org', quick=False, compress=True): """ Run deployment locally. """ slack("Starting deployment of '%s' at ves-hx-a4" % git_branch) git_updates(git_branch) if not quick: update_npm() collect_static_files() if compress: compress_static_files() if not quick: install_django_requirements() flush_memcached() restart_django(restart_url) if not git_branch: with env.cd(settings.PROJECT_PATH): git_branch = env.run('git rev-parse --abbrev-ref HEAD', capture=True) # env.run == local, takes capture arg slack("Deployed '%s' at ves-hx-a4: <https://test.rnacentral.org|test.rnacentral.org>" % git_branch)
def deploy_remotely(git_branch=None, restart_url='https://rnacentral.org', quick=False, compress=True): """ Run deployment remotely. """ slack("Starting deployment of '%s' at %s" % (git_branch, env.host)) git_updates(git_branch) # on PG machine npm was unable to download node_modules, so we rsync them from OY if not quick: if env.host == 'ves-pg-a4': cmd = 'rsync -av {path}/ {host}:{path}'.format( path=os.path.join( settings.PROJECT_PATH, 'rnacentral', 'portal', 'static', 'node_modules' ), host='ves-pg-a4', ) local(cmd) else: update_npm() if not quick: rsync_local_files() collect_static_files() if compress: compress_static_files() flush_memcached() restart_django(restart_url) if not git_branch: with env.cd(settings.PROJECT_PATH): git_branch = env.run('git rev-parse --abbrev-ref HEAD') slack("Deployed '%s' at %s: <https://rnacentral.org|rnacentral.org>" % (git_branch, env.host))
def update(): with env.cd(env.directory): env.run('hg pull') env.run('hg update')
def virtualenv(): with env.cd(env.directory): with prefix(env.activate): yield
def restart(): "restarts the apache django daemon" with env.cd(env.directory): env.run('touch ./freieit/apache2/django.wsgi')
def make_venv(): with env.cd(env.directory): env.run('virtualenv --system-site-packages -p %s %s' % (env.pyexecutable, env.venvpath))
def clean(): with env.cd(env.directory): env.run('rm -f _freieit.db') env.run('rm -rf _media/')
def update_code(): """ Pulls changes from the central repo and checks out the right branch """ with env.cd(env.proj_dir): env.run('git pull && git checkout %s' % env.branch)
def make_images(): with env.cd(op.join(PROJECT_ROOT, 'tools')): env.run('docker build -t %s apache/' % imgname('apache')) env.run('docker pull mariadb')