def remove_pyc_files(dev, branch):
"""
Clean *.pyc
"""
branch_object = retrieve_branch(dev=dev, branch=branch)
return run_commands(['find . -name "*.pyc" -exec rm -f {} \;', ],
directory=branch_object.code_dir)
def restart_services(dev, branch):
"""
Reload nginx and the supervisor
"""
command_list = ['sudo supervisorctl reload',
'sudo /etc/init.d/nginx reload', ]
# NOTE: don't forget to allow www-data run this in sudoers file
return run_commands(command_list)
def update_repo(dev, branch):
"""
Update the branch's repository
"""
branch_object = retrieve_branch(dev=dev, branch=branch)
return run_commands(['git fetch origin',
'git pull origin %s' % branch_object.branch, ],
directory=branch_object.code_dir)
def install_branch(dev, branch):
"""
If we want a branch different of master, download it.
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
if 'master' == branch:
return '{nothing_to_do, "branch == master"}'
command_list = ['git fetch origin',
'git checkout -b %s origin/%s' % (branch, branch), ]
return run_commands(command_list, directory=branch_object.code_dir)
def install_repo(dev, branch):
"""
Create the directory and clone the repo. if it doesn't exist
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
command_list = ['mkdir -p %s' % branch_object.directory,
'git clone %s %s' % (branch_object.git_repo,
branch_object.code_dir), ]
# NOTE: git clone mkdir and rmdir by itself
return run_commands(command_list)
def install_rabbit_vhost(dev, branch):
"""
Create a RabbitMQ vhost and set permissions for the lab user
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
rabbit_vhost = 'sudo rabbitmqctl add_vhost %s'
rabbit_perm = "sudo rabbitmqctl set_permissions -p %s lab '.*' '.*' '.*'"
# NOTE: remember to allow www-data to run rabbitmqctl in sudoers file
return run_commands(
[rabbit_vhost % branch_object.broker_vhost,
rabbit_perm % branch_object.broker_vhost, ])
def install_pip_requirements(dev, branch):
"""
Install all the packages cited in requirements.pip in the virtual
environment
NOTE: Sometimes this step crashes because pip can not download the pkgs
NOTE: Sometimes the name of the pkg changes
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
requirements_file = join(branch_object.code_dir, 'dowant/requirements.pip')
return run_commands(
['pip install --timeout=60 -E %s -r %s' % (branch_object.virtualenv_dir,
requirements_file), ])
def install_virtualenv(dev, branch):
"""
Prepare an empty virtualenv
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
# NOTE Var. in postactivate script
activate_path = join(branch_object.virtualenv_dir, 'bin/postactivate')
command_list = [
'virtualenv --no-site-packages %s' % branch_object.virtualenv_dir, ]
for key in django_settings.ENVIRONMENT_VARS.keys():
fields = (key,
eval(django_settings.ENVIRONMENT_VARS[key]),
activate_path)
command_list.append('echo \'export %s=%s\' >> %s' % fields)
return run_commands(command_list)
def install_dir_struct_post_repo(dev, branch):
"""
Create _/data/ directory.
Replace symlinks to global dirs. with different direcories for each branch
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
fields = (branch_object.log_dir,
join(branch_object.directory, 'log'))
command_list = ['rm -rf code/dowant/media/compressed',
'mkdir code/dowant/media/compressed',
'rm -rf code/dowant/media/restaurant_logos',
'mkdir code/dowant/media/restaurant_logos',
'rm -rf code/dowant/media/uploads',
'mkdir code/dowant/media/uploads',
'rm -rf code/dowant/media/img/company_logos',
'mkdir code/dowant/media/img/company_logos',
'ln -s %s %s' % fields, ]
return run_commands(command_list, directory=branch_object.directory)
def use_prefab_database(db_name):
"""
Assigns a databse for the branch.
If there is a prefab. one, it uses it, if not,
it creates a new one, taking dowant_test as template
"""
(prefab_db_name, output) = look_for_prefab_database(db_name)
if prefab_db_name is None:
# no prefab available, create one (sloooow)
sql_command = 'CREATE DATABASE %s WITH TEMPLATE dowant_test;'
sql = sql_command % db_name
else:
# prefab available, change name (fast!)
sql_command = 'ALTER DATABASE %s RENAME TO %s;'
sql = sql_command % (prefab_db_name, db_name)
output += run_commands(['psql -U dowant_test -c \"%s\"' % sql, ])
return output
def install_dir_struct_pre_repo(dev, branch):
"""
Create _branch_/ _branch_/log/ and _branch_/data/ directory
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
command_list = ['mkdir -p %s' % branch_object.directory,
'mkdir -p %s' % branch_object.log_dir, ]
data_dirs = ('data/', 'data/billing/', 'data/bloomsburys/',
'data/bloomsburys/imports/',
'data/bloomsburys/imports/products.2011-02-07_small/',
'data/bloomsburys/imports/products.B-MY.20110207-164827.xml/',
'data/faxes/', 'data/faxes/retarus/',
'data/compressed/', 'data/orders/', )
for one_dir in data_dirs:
command_list.append('mkdir %s' %
join(branch_object.directory, one_dir), )
return run_commands(command_list)
def install_django_stuff(dev, branch):
"""
3 steps: install db schema mods., compress the JS (FIX: sure?) and
compile the *.po i18n files
NOTE: DB migrations are interactive with the user. If there is,
the step crashes
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
dowant_dir = join(branch_object.code_dir, 'dowant/')
environment = env_to_str(branch_object)
migrate = '%s python manage.py migrate --delete-ghost-migrations'
# FIXME list comprenhension
commands_list = [
migrate % environment,
'%s python manage.py lh_synccompress' % environment,
'%s python manage.py compilemessages' % environment, ]
# '%s python manage.py test -s'%environment, ]
return run_commands(commands_list,
directory=dowant_dir)
def look_for_prefab_database(db_name):
prefab_db_name = None
output = ''
# Look for a existing prefab DB
# FIXME don't hardcode the numbers
for no_copy in range(1, 6):
test_name = 'dowant_prefab_%s' % str(no_copy)
# NOTE: api_apikey is faster than django_session
sql_command = 'SELECT * FROM api_apikey LIMIT 1;'
# NOTE: '-o -' is necessary to avoid the pager that
# psql launches and stucks the process
# Another option would be 'psql -l'
command = '/usr/bin/psql -U dowant_test -t -d %s -c \"%s\" -o -'
try:
output += run_commands(
[command % (test_name, sql_command), ], pty=True, shell=False)
prefab_db_name = test_name
break
except RunCommandsException:
continue
return (prefab_db_name, output)
def install_config_files(dev, branch):
"""
Fulfill the templates for config files and push them to the server
"""
branch_object = Branch.objects.get(dev=dev, branch=branch)
# Prepare data for templates
context = fill_in_the_templates(branch_object)
# Create dirs and install templates
supervisor_path = join(branch_object.config_dir, 'supervisor/')
nginx_path = join(branch_object.config_dir, 'nginx/')
command_list = ['mkdir -p %s' % branch_object.config_dir,
'mkdir -p %s' % supervisor_path,
'mkdir -p %s' % nginx_path, ]
output = run_commands(command_list)
with fabric_settings(host_string=django_settings.FABRIC_HOST, warn_only=True):
template_list = django_settings.SUPERVISOR_TEMPLATE_LIST
for (template_path, target, ignore) in template_list:
source = join(django_settings.SUPERVISOR_TEMPLATE_PATH, template_path)
dest = join(branch_object.directory, target)
upload_template(source, dest, context=context, backup=False)
output += '%s is uploaded\n' % dest
return output
def remove_directory(dev, branch):
branch_object = retrieve_branch(dev=dev, branch=branch)
return run_commands(['rm -rf %s' % branch_object.directory, ])
def remove_virtualenv(dev, branch):
branch_object = retrieve_branch(dev=dev, branch=branch)
return run_commands(['rm -rf %s' % branch_object.virtualenv_dir, ])
def remove_database(dev, branch):
branch_object = retrieve_branch(dev=dev, branch=branch)
return run_commands([
'psql -U dowant_test -c "DROP DATABASE %s;"' % branch_object.db_name,])
def remove_rabbit_vhost(dev, branch):
branch_object = retrieve_branch(dev=dev, branch=branch)
return run_commands([
'sudo rabbitmqctl delete_vhost %s' % branch_object.broker_vhost,])