def ping():
'''
Test command
'''
env.run('echo pong!')
env.run("cat /etc/*-release") # List linux dist info
def updateMigrations():
import sys
sys.path.append(env.local_source_root)
from mysite.mysettings.common import installed_apps
for a in installed_apps:
if a in migrated_apps:
env.run('%s manage.py schemamigration %s --auto' % (env.python, a))
def loadfixtures(type_='system'):
"""
load fixtures (system and test).
usage:
fab load_fixtures -> loads all files which name ends with
'_fixtures.json' inside the fixtures folder
(except for 'test_fixtures.json')
fab load_fixtures:test -> load only the fixtures/test_fixtures.json file
"""
if type_ == 'test':
fixtures = ""
folder = 'fixtures/test'
for fixture in os.listdir(folder):
if (fixture.endswith('.json')
and fixture != 'contenttypes_fixtures.json'):
fixtures += "{}/{} ".format(folder, fixture)
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py loaddata {} --settings={}'.format(
fixtures, env.komoo_django_settings))
else:
for fixture in os.listdir('fixtures'):
if fixture.endswith('_fixtures.json'):
with virtualenv(), env.cd('mootiro_maps'):
env.run(
'python manage.py loaddata fixtures/{} --settings={}'.
format(fixture, env.komoo_django_settings))
def clean_cache():
"""
clears system cache
"""
env.run("free -m | awk 'FNR == 2 { print $4 }'")
env.sudo("echo 3 > /proc/sys/vm/drop_caches")
env.run("free -m | awk 'FNR == 2 { print $4 }'")
def compilemessages():
"""
compile messages file
"""
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py compilemessages --settings={}'
.format(env.komoo_django_settings))
def java():
"""Install java."""
r1 = env.run(SUDO_ADD_APT + "ppa:webupd8team/java -y")
r2 = env.run(SUDO_UPDATE)
r3 = env.run(SUDO_INSTALL + "echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections")
r4 = env.run(SUDO_INSTALL + "oracle-java8-installer")
return r1, r2, r3, r4
def copy():
branch = None
repro_url = None
if "repro_url" in env:
repro_url = env.repro_url
if not repro_url:
sys.exit("repro_url is missing in configuration")
if "branch" in env:
branch = env.branch
if not branch and gitext.get_reverse_path():
path = gitext.get_reverse_path()
branch = gitext.get_git_branch(path)
if not branch:
logger.warn("Git branch not set, using master instead")
branch = "master"
with apply_settings(), env.cd(env.app_path):
env.run("git clone --depth 1 -b %(branch)s %(repro)s %(path)s" % {
"branch": branch,
"repro": repro_url,
"path": env.current_release,
})
def stop():
'''Stop remote application server.'''
_setup_supervisor()
with virtualenv(), env.cd('mootiro_maps'):
env.run(
'supervisorctl -c {dir}/supervisor/{env}.conf stop {env}:'.format(
env=env.komoo_env, dir=env.komoo_project_folder))
def compilemessages():
"""
compile messages file
"""
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py compilemessages --settings={}'.format(
env.komoo_django_settings))
def dev_server(address=settings.HORIZON_DEV_ADDRESS,
horizon_path=settings.HORIZON_ROOT):
"""Run horizon server for development purposes"""
with env.cd(horizon_path):
env.run(('sudo tools/with_venv.sh python manage.py runserver '
'{0}').format(address))
def clean_cache():
"""
clears system cache
"""
env.run("free -m | awk 'FNR == 2 { print $4 }'")
env.sudo("echo 3 > /proc/sys/vm/drop_caches")
env.run("free -m | awk 'FNR == 2 { print $4 }'")
def backup_db():
"""
Performs a database backup and saves file to /shared/muteparrot_db.sql
"""
env.run("docker exec container pg_dump -U postgres example_db "
"-f /shared/example.sql")
puts("Backup done!")
def copy():
branch = None
repro_url = None
if "repro_url" in env:
repro_url = env.repro_url
if not repro_url:
sys.exit("repro_url is missing in configuration")
if "branch" in env:
branch = env.branch
if not branch and gitext.get_reverse_path():
path = gitext.get_reverse_path()
branch = gitext.get_git_branch(path)
if not branch:
logger.warn("Git branch not set, using master instead")
branch = "master"
with apply_settings(), env.cd(env.app_path):
env.run("git clone --depth 1 -b %(branch)s %(repro)s %(path)s" % {
"branch": branch,
"repro": repro_url,
"path": env.current_release,
})
def ping():
'''
Test command
'''
env.run('echo pong!')
env.run("cat /etc/*-release") # List linux dist info
def migrate_database(script):
'''Dumps database to a file, runs migration script and loads it back.'''
input_file = backup()
output_file = input_file + ' (migrated)'
env.run('python {script} {inp} {out}'.format(script=script, inp=input_file,
out=output_file))
loaddata(output_file)
def restore_db(release=None):
"""
Restores backup back to version, uses current version by default.
"""
assert "mysql_user" in env, "Missing mysqL_user in env"
assert "mysql_password" in env, "Missing mysql_password in env"
assert "mysql_host" in env, "Missing mysql_host in env"
assert "mysql_db" in env, "Missing mysql_db in env"
if not release:
release = paths.get_current_release_name()
if not release:
raise Exception("Release %s was not found" % release)
backup_file = "mysql/%s.sql.gz" % release
backup_path = paths.get_backup_path(backup_file)
if not env.exists(backup_path):
raise Exception("Backup file %s not found" % backup_path)
env.run("gunzip < %s | mysql -u %s -p%s -h %s %s" %
(backup_path, env.mysql_user, env.mysql_password, env.mysql_host,
env.mysql_db))
def loadfixtures(type_='system'):
"""
load fixtures (system and test).
usage:
fab load_fixtures -> loads all files which name ends with
'_fixtures.json' inside the fixtures folder
(except for 'test_fixtures.json')
fab load_fixtures:test -> load only the fixtures/test_fixtures.json file
"""
if type_ == 'test':
fixtures = ""
folder = 'fixtures/test'
for fixture in os.listdir(folder):
if (fixture.endswith('.json') and
fixture != 'contenttypes_fixtures.json'):
fixtures += "{}/{} ".format(folder, fixture)
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py loaddata {} --settings={}'.format(
fixtures, env.komoo_django_settings))
else:
for fixture in os.listdir('fixtures'):
if fixture.endswith('_fixtures.json'):
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py loaddata fixtures/{} --settings={}'.format(
fixture, env.komoo_django_settings))
def google_chrome_stable():
"""Install google-chrome-stable."""
r1 = env.run(SUDO_INSTALL + "libxss1 libappindicator1 libindicator7")
r2 = env.run("wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb")
r3 = env.run("sudo dpkg -i google-chrome*.deb")
r4 = env.run("sudo rm google-chrome*.deb")
return r1, r2, r3, r4
def ipython():
"""Install and upgrade ipython."""
r1 = env.run(PIP_INSTALL + "--upgrade ipython[all]")
r2 = env.run(PIP_INSTALL + "nose")
print(green('Run iptest for ipython.'))
r3 = env.run("iptest")
return r1, r2, r3
def pycharm():
"""Install pycharm."""
with lcd("/opt"):
r1 = env.run("wget http://download.jetbrains.com/python/pycharm-professional-3.4.1.tar.gz")
r2 = env.run("sudo tar -xzvf pycharm-professional-3.4.1.tar.gz")
r3 = env.run("sudo rm pycharm-professional-3.4.1.tar.gz")
r4 = env.run("sudo ln -s /opt/pycharm-professional-3.4.1/bin/pycharm.sh /usr/bin/pycharm")
return r1, r2, r3, r4
def test():
"""
Prints environment information.
"""
init_tasks()
env.run("cat /etc/*-release") # List linux dist info
def drop():
"""Drops komoo database """
dbname = env.komoo_dbname
logging.info("Droping database '{}'".format(dbname))
if confirm('Do you really want to drop the database "{}"? '
'(Please, make sure you have a backup)'.format(dbname),
default=False):
env.run('dropdb -U {} {}'.format(env.komoo_dbuser, dbname))
def _setup_supervisor():
# make sure supervisord is running
sock_file = ('{}/supervisor/supervisor.sock'
).format(env.komoo_project_folder)
if not exists(sock_file):
with virtualenv(), env.cd('mootiro_maps'):
env.run('supervisord -c {}/supervisor/{}.conf'.format(
env.komoo_project_folder, env.komoo_env))
def initial_revisions():
"""
load initial revisions for django-revisions module
should run only once when installed/or when loaded a new app/model
"""
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py createinitialrevisions --settings={}'
.format(env.komoo_django_settings))
def test():
"""
Prints environment information.
"""
init_tasks()
env.run("cat /etc/*-release") # List linux dist info
def _setup_supervisor():
# make sure supervisord is running
sock_file = ('{}/supervisor/supervisor.sock').format(
env.komoo_project_folder)
if not exists(sock_file):
with virtualenv(), env.cd('mootiro_maps'):
env.run('supervisord -c {}/supervisor/{}.conf'.format(
env.komoo_project_folder, env.komoo_env))
def initial_revisions():
"""
load initial revisions for django-revisions module
should run only once when installed/or when loaded a new app/model
"""
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py createinitialrevisions --settings={}'.format(
env.komoo_django_settings))
def makemessages(lang='pt_BR'):
"""create translations messages file"""
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py makemessages --all -l {} --settings={}'.format(
lang, env.komoo_django_settings))
env.run('python manage.py makemessages'
' --all -d djangojs -l {} --settings={}'.format(
lang, env.komoo_django_settings))
def drop():
"""Drops komoo database """
dbname = env.komoo_dbname
logging.info("Droping database '{}'".format(dbname))
if confirm('Do you really want to drop the database "{}"? '
'(Please, make sure you have a backup)'.format(dbname),
default=False):
env.run('dropdb -U {} {}'.format(env.komoo_dbuser, dbname))
def migrate_database(script):
'''Dumps database to a file, runs migration script and loads it back.'''
input_file = backup()
output_file = input_file + ' (migrated)'
env.run('python {script} {inp} {out}'.format(script=script,
inp=input_file,
out=output_file))
loaddata(output_file)
def empty_copy():
"""
A stub copy method that does nothing more then create a .txt file.
"""
source_path = os.path.join(env.current_release, "src")
env.run("mkdir -p %s" % source_path)
env.run("touch %s/app.txt" % source_path)
def empty_copy():
"""
A stub copy method that does nothing more then create a .txt file.
"""
source_path = os.path.join(env.current_release, "src")
env.run("mkdir -p %s" % source_path)
env.run("touch %s/app.txt" % source_path)
def makemessages(lang='pt_BR'):
"""create translations messages file"""
with virtualenv(), env.cd('mootiro_maps'):
env.run(
'python manage.py makemessages --all -l {} --settings={}'.format(
lang, env.komoo_django_settings))
env.run('python manage.py makemessages'
' --all -d djangojs -l {} --settings={}'.format(
lang, env.komoo_django_settings))
def backup(forced=False):
'''Dumps remote database and stores it in backups folder.'''
remote_path = '{}/backups/{}'.format(env.komoo_project_folder, DBFILE())
if env.exists(remote_path):
if not forced:
return remote_path
env.run('pg_dump -U {} --no-privileges --no-owner {} > {}'.format(
env.komoo_dbuser, env.komoo_dbname, remote_path))
return remote_path
def elasticsearch():
""" configure elasticsearch from scratch """
run_elasticsearch(bg='true')
env.run('sleep 10s')
setup_django()
from search.utils import reset_index, create_mapping
reset_index()
create_mapping()
kill_tasks('elasticsearch')
def patch():
"""
build env_ironment: pip install everything + patch django for postgis
encoding problem on postgres 9.1
"""
with virtualenv_cm():
env.run("patch -p0 `which python | "
"sed -e 's/bin\/python$/lib\/python2.7\/site-packages\/django\/"
"contrib\/gis\/db\/backends\/postgis\/adapter.py/'` "
"docs/postgis-adapter-2.patch")
def elasticsearch():
""" download and place elastic search on the lib folder """
# FIXME: Hard coded version is evil.
with virtualenv_cm():
env.run(''
'wget -P lib/ http://download.elasticsearch.org/elasticsearch/'
'elasticsearch/elasticsearch-0.20.5.tar.gz;'
'tar xzvf lib/elasticsearch-0.20.5.tar.gz -C lib/;'
'mv lib/elasticsearch-0.20.5 lib/elasticsearch;'
'rm lib/elasticsearch-0.20.5.tar.gz;')
def create_services(tag='unstable'):
for name, _, _ in SERVICES:
command = 'docker service create --replicas 2 ' + \
'--name ' + name + ' --network ' + SWARM101_NETWORK + \
' ' + name + ':' + tag
env.run(command)
time.sleep(5)
command = 'docker service update --publish-add 8000:8000 front_gateway'
env.run(command)
def report():
"""
checks disk usage
"""
disk_usage_io = StringIO()
varnish_usage_io = StringIO()
disk_usage = env.run("time df | awk 'FNR == 2 { print $5 }'", stdout=disk_usage_io)
varnish_usage = env.run("varnishstat", stdout=varnish_usage_io)
print("Current disk usage is {}".format(disk_usage))
print("Current varnish usage is {}".format(varnish_usage))
def kill_tasks(*tasks):
""" Kill background tasks given a list o task names """
if not tasks:
tasks = ['coffee', 'sass', 'elasticsearch', 'manage.py']
for task in tasks:
try:
env.run("ps -eo pid,args | grep %s | grep -v grep | "
"cut -c1-6 | xargs kill" % task)
except Exception:
logging.warning('cannot execut kill on taks: {}'.format(task))
def update(install_path=None):
if not install_path:
install_path = paths.get_source_path(env.current_release)
flags = default_flags
if "composer_flags" in env:
flags = env.composer_flags
with(env.cd(install_path)):
env.run("php composer.phar install {}".format(" ".join(flags)))
def elasticsearch():
""" download and place elastic search on the lib folder """
# FIXME: Hard coded version is evil.
with virtualenv_cm():
env.run(''
'wget -P lib/ http://download.elasticsearch.org/elasticsearch/'
'elasticsearch/elasticsearch-0.20.5.tar.gz;'
'tar xzvf lib/elasticsearch-0.20.5.tar.gz -C lib/;'
'mv lib/elasticsearch-0.20.5 lib/elasticsearch;'
'rm lib/elasticsearch-0.20.5.tar.gz;'
)
def elasticsearch(bg='false'):
"""
run elastic search
usage:
fab run_elasticsearch #runs on foreground
fab run_elasticsearch:bg=true #runs on background
"""
background = '&' if bg == 'true' else ''
with virtualenv():
env.run('./lib/elasticsearch/bin/elasticsearch -f {}'.format(
background))
def patch():
"""
build env_ironment: pip install everything + patch django for postgis
encoding problem on postgres 9.1
"""
with virtualenv_cm():
env.run(
"patch -p0 `which python | "
"sed -e 's/bin\/python$/lib\/python2.7\/site-packages\/django\/"
"contrib\/gis\/db\/backends\/postgis\/adapter.py/'` "
"docs/postgis-adapter-2.patch")
def backup(forced=False):
'''Dumps remote database and stores it in backups folder.'''
remote_path = '{}/backups/{}'.format(env.komoo_project_folder, DBFILE())
if env.exists(remote_path):
if not forced:
return remote_path
env.run('pg_dump -U {} --no-privileges --no-owner {} > {}'.format(
env.komoo_dbuser,
env.komoo_dbname,
remote_path))
return remote_path
def kill_tasks(*tasks):
""" Kill background tasks given a list o task names """
if not tasks:
tasks = ['coffee', 'sass', 'elasticsearch', 'manage.py']
for task in tasks:
try:
env.run(
"ps -eo pid,args | grep %s | grep -v grep | "
"cut -c1-6 | xargs kill" % task)
except Exception:
logging.warning('cannot execut kill on taks: {}'.format(task))
def elasticsearch(bg='false'):
"""
run elastic search
usage:
fab run_elasticsearch #runs on foreground
fab run_elasticsearch:bg=true #runs on background
"""
background = '&' if bg == 'true' else ''
with virtualenv():
env.run(
'./lib/elasticsearch/bin/elasticsearch -f {}'.format(background))
def set_up_as_service(absolute_keystone_path=None):
if not absolute_keystone_path:
absolute_keystone_path = os.getcwd() + '/' + settings.KEYSTONE_ROOT
in_file = open('conf/keystone_idm.conf')
src = string.Template(in_file.read())
out_file = open("tmp_keystone_idm.conf", "w")
out_file.write(src.substitute({
'absolute_keystone_path': absolute_keystone_path}))
out_file.close()
env.run('sudo cp tmp_keystone_idm.conf /etc/init/keystone_idm.conf')
env.run('sudo rm tmp_keystone_idm.conf')
def report():
"""
checks disk usage
"""
disk_usage_io = StringIO()
varnish_usage_io = StringIO()
disk_usage = env.run("time df | awk 'FNR == 2 { print $5 }'",
stdout=disk_usage_io)
varnish_usage = env.run("varnishstat", stdout=varnish_usage_io)
print("Current disk usage is {}".format(disk_usage))
print("Current varnish usage is {}".format(varnish_usage))
def add_network(name, subnet=None, gateway=None, driver=None):
cmd = "docker network create %s --label \"STACK_ID=%s\" " % (name,
env.stack_id)
if driver:
cmd += "--driver %s " % driver
if subnet:
cmd += "--subnet %s " % subnet
if gateway:
cmd += "--gateway %s " % gateway
env.run(cmd, show=True)
def skype_call_recorder():
"""Install skype call recorder.
Source file:
skype-call-recorder-ubuntu_0.10_amd64.deb
"""
r1 = env.run("sudo wget http://atdot.ch/scr/files/0.10/skype-call-recorder-ubuntu_0.10_amd64.deb")
r2 = env.run("sudo dpkg --install skype-call-recorder-ubuntu_0.10_amd64.deb")
# clean up, delete downloaded file.
r3 = env.run("sudo rm skype-call-recorder-ubuntu_0.10_amd64.deb")
return r1, r2, r3
def cleanup_releases(limit=5):
"""
Removes older releases.
"""
init_tasks()
max_versions = limit + 1
env.run("ls -dt %s/*/ | tail -n +%s | xargs rm -rf" %
(paths.get_releases_path(), max_versions))
def get_containers():
with hide("running"):
container_list = _parse_docker_list(
env.run(
"docker ps -a --format \"{{ .Names }}\" --filter Label=\"STACK_ID=%s\""
% env.stack_id).stdout)
return {
container:
json.loads(env.run("docker inspect %s" % container).stdout)[0]
for container in container_list
}
def django(port=False):
if not port:
port = env.komoo_port
if 'dev' not in env.komoo_django_settings:
with virtualenv(), env.cd('mootiro_maps'):
env.run('python manage.py run_gunicorn --workers=2 '
'--bind=127.0.0.1:{} --settings={}'.format(
port, env.komoo_django_settings))
else:
with virtualenv(), env.cd('mootiro_maps'):
env.run(
'python manage.py runserver 0.0.0.0:{} --settings={}'.format(
port, env.komoo_django_settings))
def create_venv():
if "venv_path" not in env:
raise Exception("No env.venv_path has been specified")
# Checks if venv exist and prompts user if reinstalling is an option
if env.exists(Path(get_path(), "bin")):
logger.warn("Virtualenv is already installed")
prompt_result = prompt("Install anyways:", default="no")
if prompt_result == "no":
return
env.run("virtualenv %s" % get_path())
def delete_k8s():
services = [
'bangkok',
'tokyo',
'nyc',
'munich',
'front-gateway',
]
for each in services:
command = f'kubectl delete -f k8s/{each}-deployment.yml'
env.run(command)
command = f'kubectl delete -f k8s/{each}-svc.yml'
env.run(command)
