def display_pre_post(choices):
"""
Displays list of pre and post checkout commands,
returns the list of commands in a dictionary
return({
'pre': [ 'cmd1', 'cmd2', ... ],
'post': [ 'cmd1', 'cmd2', ... ]
})
"""
pkg_info = PackageInfo()
pre_post = pkg_info.pre_post_actions(choices)
for stage in ['pre', 'post']:
if pre_post[stage]:
fastprint(green('{0}-checkout commands:\n'.format(stage),
bold=True) + green(' -> ') + green('\n -> '.join(
pre_post[stage])) + white('\n\n'))
else:
fastprint(green('WARNING', bold=True) +
green(' - no {0}-checkout commands for this set '
'of packages : '.format(stage, choices)) +
white('\n\n'))
return pre_post
def compress_venv(self):
fastprint('Compressing virtualenv')
local('tar -C %(build_dir)s/.. -cjf %(tarbz_path)s %(dirname)s' % {
'build_dir': self.deployer.build_dir,
'tarbz_path': self.tarbz_path,
'dirname': os.path.basename(self.deployer.build_dir)
})
def _parse_hash_from_tag(tag):
split_tag = tag.split("-")
if len(split_tag) != 3:
fastprint("not valid tag %s\n" % split_tag)
sys.exit(1)
fastprint("Latest hash is %s \n" % split_tag[2])
return split_tag[2]
def add_deploy_user():
"""Create the deploy user account, one time task.
The deploy user is used for almost all processes.
Your SSH key is pushed so that you can login via ssh keys.
"""
username = '******'
with settings(user='******'):
# Create the user, no password
fastprint('adding the %s user account...' % username)
run('useradd -m -s /bin/bash %s' % username)
run('adduser %s sudo' % username)
# Allow this user to sudo without password
# really should list specific command (the last ALL)
files.append('/etc/sudoers.d/%s' % username,
'%s ALL=(ALL:ALL) NOPASSWD: ALL' % username)
fastprint('setting up SSH')
ssh_path = '/home/%s/.ssh' % username
if not files.exists(ssh_path, verbose=True):
run('mkdir %s' % ssh_path)
run('chmod 700 %s' % ssh_path)
key_text = _read_key_file('~/.ssh/id_rsa.pub')
files.append('%s/authorized_keys' % ssh_path, key_text)
run('chown -R %s:%s %s' % (username, username, ssh_path))
def fake_activity(message):
print green(message)
for x in xrange(10):
fastprint('.')
sys.stdout.flush()
time.sleep(0.5)
print green('OK')
def restart_server():
print green('Restarting app server...')
for x in xrange(10):
fastprint('.')
sys.stdout.flush()
time.sleep(0.5)
print green('OK')
def send_meeting():
date = prompt('Date:', validate=r'(\d{4})-(\d{2})')
subject = prompt('Subject:')
email_from = '*****@*****.**'
email_to = '*****@*****.**'
filepath = 'meetings/{0}.txt'.format(date)
local('git pull')
if not os.path.isfile(filepath):
return abort('There is no meeting at that date.')
with open(filepath, 'r') as f:
msg = MIMEText(f.read())
msg['Subject'] = subject
msg['From'] = email_from
msg['To'] = email_to
preview = "{0}".format(msg.as_string())
fastprint(preview)
send = prompt('Everything look okay?', validate=r'(yes|no)')
if send == 'yes':
s = smtplib.SMTP('127.0.0.1')
s.sendmail(email_from, [email_to], msg.as_string())
s.quit()
def update_requirements():
"""
Update external dependencies on remote host.
"""
require('virtualenv_root',
'domain_root',
'local_repo_root',
provided_by=('staging', 'production'))
ensure_virtualenv()
fastprint("Updating OpenOrdini dependencies..." % env, show_prefix=True)
with hide('commands'):
# upload pip requirements file to the server
source = os.path.join(env.local_repo_root, 'requirements.txt')
requirement_file = dest = os.path.join(env.domain_root, 'private',
'requirements.txt')
put(source, dest, mode=0644)
pip_executable = os.path.join(env.virtualenv_root, 'bin', 'pip')
cmd = [pip_executable] # use pip version provided by virtualenv
cmd += ['install']
# TODO: verify that --upgrade removes files in the src directory
cmd += ['--upgrade'] # upgrade already installed packages
cmd += ['--requirement %s' % requirement_file
] # specify a requirement file
run(' '.join(cmd))
fastprint(" done." % env, end='\n')
def base_build_ami():
"""
Build the base AMI
"""
require('hosts', provided_by=[base_setup_env])
# Make sure yum is up to date
yum_update()
# Perform the base install
base_install()
# When glusterfs-server package is first installed, it creates a node UUID file at /var/lib/glusterd/glusterd.info
# So, when gluster resolves the hostname to a UUID, it creates a conflict.
sudo('service glusterd stop')
sudo('rm /var/lib/glusterd/glusterd.info')
# Save the instance as an AMI
puts("Stopping the instance")
env.ec2_connection.stop_instances(env.ec2_instance.id, force=True)
while not env.ec2_instance.update() == 'stopped':
fastprint('.')
time.sleep(5)
puts(
"The AMI is being created. Don't forget to terminate the instance if not needed"
)
env.ec2_connection.create_image(env.ec2_instance.id,
env.ami_name,
description='The base GlusterFS AMI')
puts('All done')
def upgrade_teravm_agw_AWS(setup, hash, key_filename=DEFAULT_KEY_FILENAME):
"""
Upgrade teravm agw to image with the given hash.
hash: a hash to identify what images to pull from s3 bucket and use
for upgrading. If None, try find the most recent hash.
key_filename: path to where the private key is for authorized-key based
ssh. The public key counterpart needs to in the authorized_keys file on
the remote host. If empty file name is passed, password-based ssh will
work instead. This can be used if the script is run manually.
"""
fastprint("\nUpgrade teraVM AGW through AWSto %s\n" % hash)
_setup_env("magma", VM_IP_MAP[setup]["gateway"], key_filename)
try:
image = _get_gateway_image(hash)
except Exception:
fastprint("Image %s not found. Not updating AGW \n" % hash)
return
_fetch_image("ag", "gateway/%s" % image)
with cd("/tmp/images"):
run("tar -xzf %s" % image)
# --fix-broken to avoid the case where a previous manual
# install didn't leave missing libraries.
sudo("apt --fix-broken -y install -o "
'Dpkg::Options::="--force-confnew" --assume-yes --force-yes')
sudo("apt-get update -y")
sudo("apt-get autoremove -y")
sudo("apt --fix-broken -y install -o "
'Dpkg::Options::="--force-confnew" --assume-yes --force-yes')
sudo("dpkg --force-confnew -i magma*.deb")
sudo("apt-get install -f -y")
sudo("systemctl stop magma@*")
sudo("systemctl restart magma@magmad")
def update_core_conf():
"""
Update Solr core's configuration files on the remote machine(s),
syncing them with local ones.
"""
require('domain_root',
'app_domain',
'local_repo_root',
provided_by=('staging', 'production'))
# update configuration -- on the remote machine -- for the Solr core
# serving this OpenMunicipio instance (via rsync)
# defaults rsync options: -pthrvz
fastprint("Syncing Solr configuration for %(app_domain)s..." % env,
show_prefix=True)
extra_opts = '--omit-dir-times'
with hide('commands'):
rsync_project(
remote_dir=os.path.join(env.domain_root, 'private'),
local_dir=os.path.join(env.local_repo_root, 'solr'),
exclude=('.*', '*~', 'context_*.xml', 'solr.xml',
'solr.xml.remote'),
delete=True,
extra_opts=extra_opts,
)
sudo('chown -R om:www-data %s' %
os.path.join(env.domain_root, 'private', 'solr'))
fastprint(" done." % env, end='\n')
def display_pre_post(choices):
"""
Displays list of pre and post checkout commands,
returns the list of commands in a dictionary
return({
'pre': [ 'cmd1', 'cmd2', ... ],
'post': [ 'cmd1', 'cmd2', ... ]
})
"""
pkg_info = PackageInfo()
pre_post = pkg_info.pre_post_actions(choices)
for stage in ['pre', 'post']:
if pre_post[stage]:
fastprint(
green('{0}-checkout commands:\n'.format(stage), bold=True) +
green(' -> ') + green('\n -> '.join(pre_post[stage])) +
white('\n\n'))
else:
fastprint(
green('WARNING', bold=True) +
green(' - no {0}-checkout commands for this set '
'of packages : '.format(stage, choices)) + white('\n\n'))
return pre_post
def create_db():
if confirm("Attenzione, stai creando il db. Sei sicuro di voler procedere?"):
if "postgresql" in db_local['ENGINE']:
local('createdb -h {} -p {} -U postgres {}'.format(
db_local['HOST'], db_local['PORT'], db_local['NAME']))
fastprint('- Database {} creato. carico il dump'.format(
db_local['NAME']))
def base_build_ami():
"""
Build the base AMI
"""
require('hosts', provided_by=[base_setup_env])
# Make sure yum is up to date
yum_update()
# Make the swap we might need
make_swap()
# Perform the base install
base_install()
# Save the instance as an AMI
puts("Stopping the instance")
env.ec2_connection.stop_instances(env.ec2_instance.id, force=True)
while not env.ec2_instance.update() == 'stopped':
fastprint('.')
time.sleep(5)
puts(
"The AMI is being created. Don't forget to terminate the instance if not needed"
)
env.ec2_connection.create_image(env.ec2_instance.id,
env.ami_name,
description='The base python AMI')
puts('All done')
def boinc_build_ami():
"""
Deploy the single server environment
Deploy the single server system in the AWS cloud with everything running on a single server
"""
require('hosts', provided_by=[boinc_setup_env])
resize_file_system()
yum_pip_update()
# Wait for things to settle down
time.sleep(5)
# Do we need to mount the NFS system?
mount_nfs()
boinc_install()
# Save the instance as an AMI
puts("Stopping the instance")
env.ec2_connection.stop_instances(env.ec2_instance.id, force=True)
while not env.ec2_instance.update() == 'stopped':
fastprint('.')
time.sleep(5)
puts("The AMI is being created. Don't forget to terminate the instance if not needed")
env.ec2_connection.create_image(env.ec2_instance.id, env.ami_name, description='The base BOINC AMI')
puts('All done')
def upload_supervisor_config():
file_name = '{0}/{1}'.format(env.supervisor_config_path,
env.supervisor_config_name)
if exists(file_name):
sudo('rm {0}'.format(file_name))
put(env.supervisor_config_name, env.supervisor_config_path, use_sudo=True)
fastprint(green('supervisor config ready'), end='\n')
def apply(*args):
default_options = [
'db',
'deps',
'all',
]
if not args:
fastprint("Options: %s or none (fab deploy:)" % ','.join(default_options))
return
if len(args) == 1 and not args[0]:
options = []
else:
options = map(methodcaller('lower'), args)
if 'all' in options:
options = default_options
abs_path = settings.main.root_app
if not exists(abs_path):
install_repo(abs_path)
else:
update_repo(abs_path)
if 'deps' in options:
update_deps(abs_path)
prod_config(abs_path)
collect_static(abs_path)
if 'db' in options:
sync_db(abs_path)
migrate_db(abs_path)
def setup_instance_user():
"""
Create a passwordless account for ``OM_USER`` on the remote machine(s).
"""
require('om_user', 'domain_root', 'om_user_hostkey',
provided_by=('staging', 'production'))
with hide('commands'):
# create system user
create_system_user(env.om_user, home_dir=env.domain_root)
fastprint("Adding SSH key for user `%(om_user)s'..." % env, show_prefix=True)
# ensure that ``OM_USER``'s home directory actually exists
# needed if ``create_system_user()`` fails (e.g. because ``OM_USER`` already exists)
run('mkdir -p %(domain_root)s' % env)
# copy SSH keyfile to a temporary remote location
source = env.om_user_hostkey
tmpfile = '/tmp/id_rsa.pub'
put(source, tmpfile, mode=0644)
with cd(env.domain_root):
# create a ``.ssh/authorized_keys`` file, if not already existing
run('mkdir -p .ssh' % env)
auth_keys = os.path.join('.ssh', 'authorized_keys')
run('touch %s' % auth_keys)
# add the new key to the "authorized" ones
run('cat %s >> %s' % (tmpfile, auth_keys))
# adjust filesystem permissions
run('chown -R %(om_user)s:%(om_user)s %(domain_root)s' % env)
fastprint(" done." % env, end='\n')
def base_build_ami():
"""
Build the base AMI
"""
require('hosts', provided_by=[base_setup_env])
# Make sure yum is up to date
yum_update()
# Perform the base install
base_install()
# When glusterfs-server package is first installed, it creates a node UUID file at /var/lib/glusterd/glusterd.info
# So, when gluster resolves the hostname to a UUID, it creates a conflict.
sudo('service glusterd stop')
sudo('rm /var/lib/glusterd/glusterd.info')
# Save the instance as an AMI
puts("Stopping the instance")
env.ec2_connection.stop_instances(env.ec2_instance.id, force=True)
while not env.ec2_instance.update() == 'stopped':
fastprint('.')
time.sleep(5)
puts("The AMI is being created. Don't forget to terminate the instance if not needed")
env.ec2_connection.create_image(env.ec2_instance.id, env.ami_name, description='The base GlusterFS AMI')
puts('All done')
def run_3gpp_tests(setup,
key_filename=DEFAULT_KEY_FILENAME,
test_files=NG40_TEST_FILES):
"""
Run teravm s6a and gxgy test cases. Usage: 'fab run_3gpp_tests:' for
default key filename and default test files.
key_filename: path to where the private key is for authorized-key based
ssh. The public key counterpart needs to in the authorized_keys file on
the remote host. If empty file name is passed, password-based ssh will
work instead. This can be used if the script is run manually.
test_file: a test file to use instead of the s6a and gxgy defaults.
"""
if isinstance(test_files, str):
test_files = [test_files]
test_output = []
for test_file in test_files:
fastprint("Run test for file %s\n" % (test_file))
_setup_env("ng40", VM_IP_MAP[setup]["ng40"], key_filename)
with cd("/home/ng40/magma/automation"):
with hide("warnings", "running",
"stdout"), settings(warn_only=True):
output = run("ng40test %s" % test_file)
test_output.append(output)
fastprint("Done with file %s\n" % (test_file))
verdicts = _parse_stats(test_output)
return verdicts
def boinc_build_ami():
"""
Deploy the single server environment
Deploy the single server system in the AWS cloud with everything running on a single server
"""
require('hosts', provided_by=[boinc_setup_env])
resize_file_system()
yum_pip_update()
# Wait for things to settle down
time.sleep(5)
# Do we need to mount the NFS system?
mount_nfs()
boinc_install()
# Save the instance as an AMI
puts("Stopping the instance")
env.ec2_connection.stop_instances(env.ec2_instance.id, force=True)
while not env.ec2_instance.update() == 'stopped':
fastprint('.')
time.sleep(5)
puts(
"The AMI is being created. Don't forget to terminate the instance if not needed"
)
env.ec2_connection.create_image(env.ec2_instance.id,
env.ami_name,
description='The base BOINC AMI')
puts('All done')
def setup_instance_user():
"""
Create a passwordless account for ``OM_USER`` on the remote machine(s).
"""
require('om_user', 'domain_root', 'om_user_hostkey',
provided_by=('staging', 'production'))
with hide('commands'):
# create system user
create_system_user(env.om_user, home_dir=env.domain_root)
fastprint("Adding SSH key for user `%(om_user)s'..." % env, show_prefix=True)
# ensure that ``OM_USER``'s home directory actually exists
# needed if ``create_system_user()`` fails (e.g. because ``OM_USER`` already exists)
run('mkdir -p %(domain_root)s' % env)
# copy SSH keyfile to a temporary remote location
source = env.om_user_hostkey
tmpfile = '/tmp/id_rsa.pub'
put(source, tmpfile, mode=0644)
with cd(env.domain_root):
# create a ``.ssh/authorized_keys`` file, if not already existing
run('mkdir -p .ssh' % env)
auth_keys = os.path.join('.ssh', 'authorized_keys')
run('touch %s' % auth_keys)
# add the new key to the "authorized" ones
run('cat %s >> %s' % (tmpfile, auth_keys))
# adjust filesystem permissions
run('chown -R %(om_user)s:%(om_user)s %(domain_root)s' % env)
fastprint(" done." % env, end='\n')
def _prep_deploy_dest():
""" Setup the base app directory structure."""
if env.sudo:
fastprint("Preparing Deployment Directory base (using sudo).\n")
sudo('mkdir -p %s' % env.root)
sudo('mkdir -p %s' % env.deploydir)
sudo('mkdir -p %s' % env.virtualenv_dir)
sudo('mkdir -p %s' % os.path.join(env.deploydir, 'appconf'))
sudo('mkdir -p %s' % os.path.join(env.deploydir, 'bootstrap'))
put('bootstrap/*',
os.path.join(env.deploydir, 'bootstrap'),
use_sudo=True)
sudo('mkdir -p %s' % os.path.join(env.deploydir, 'logs'))
sudo('mkdir -p %s' % os.path.join(env.deploydir, 'run'))
sudo('mkdir -p %s' % os.path.join(env.deploydir, 'data'))
sudo('chown -R %s:%s %s' %
(env.chown_user, env.chown_group, env.deploydir))
else:
fastprint("Preparing Deployment Directory base.\n")
run('mkdir -p %s' % env.root)
run('mkdir -p %s' % env.deploydir)
run('mkdir -p %s' % env.virtualenv_dir)
run('mkdir -p %s' % os.path.join(env.deploydir, 'appconf'))
run('mkdir -p %s' % os.path.join(env.deploydir, 'bootstrap'))
put('bootstrap/*', os.path.join(env.deploydir, 'bootstrap'))
run('mkdir -p %s' % os.path.join(env.deploydir, 'logs'))
run('mkdir -p %s' % os.path.join(env.deploydir, 'run'))
run('mkdir -p %s' % os.path.join(env.deploydir, 'data'))
def add_deploy_user():
"""Create the deploy user account, one time task.
The deploy user is used for almost all processes.
Your SSH key is pushed so that you can login via ssh keys.
"""
username = '******'
with settings(user='******'):
# Create the user, no password
fastprint('adding the %s user account...' % username)
run('useradd -m -s /bin/bash %s' % username)
run('adduser %s sudo' % username)
# Allow this user to sudo without password
# really should list specific command (the last ALL)
files.append('/etc/sudoers.d/%s' % username,
'%s ALL=(ALL:ALL) NOPASSWD: ALL' % username)
fastprint('setting up SSH')
ssh_path = '/home/%s/.ssh' % username
if not files.exists(ssh_path, verbose=True):
run('mkdir %s' % ssh_path)
run('chmod 700 %s' % ssh_path)
key_text = _read_key_file('~/.ssh/id_rsa.pub')
files.append('%s/authorized_keys' % ssh_path, key_text)
run('chown -R %s:%s %s' % (username, username, ssh_path))
def base_build_ami():
"""
Build the base AMI
"""
require('hosts', provided_by=[base_setup_env])
# Make sure yum is up to date
yum_update()
# Make the swap we might need
make_swap()
# Perform the base install
base_install()
# Save the instance as an AMI
puts("Stopping the instance")
env.ec2_connection.stop_instances(env.ec2_instance.id, force=True)
while not env.ec2_instance.update() == 'stopped':
fastprint('.')
time.sleep(5)
puts("The AMI is being created. Don't forget to terminate the instance if not needed")
env.ec2_connection.create_image(env.ec2_instance.id, env.ami_name, description='The base python AMI')
puts('All done')
def style_check():
"""Runs Python static code checkers against the code.
Although more for style reasons, these are quite helpful in
identifying problems with the code. A file will be generated at
./.logs/style.log for perusal.
Due to how pylint works it must be invoked manually.
"""
utils.fastprint("Checking Python code style ... ")
with api.settings(api.hide('warnings'), warn_only=True):
pep8 = api.local('pep8 .', True)
pyflakes = api.local('pyflakes .', True)
# Print them out to a file so we can peruse them later.
log = open('./.log/style.log', 'w')
log.write("pep8:\n%s\n\npyflakes:\n%s" % (pep8, pyflakes))
if pep8:
print(colors.magenta("fail", True))
elif pyflakes:
print(colors.magenta("fail", True))
else:
print(colors.green(" ok ", True))
if pep8 or pyflakes:
print(colors.magenta("Please check ./.log/style.log.", True))
print(colors.yellow("Please be sure to run pylint manually.", True))
return (pep8 and pyflakes)
def upgrade_and_run_3gpp_tests(
setup,
hash=None,
key_filename=DEFAULT_KEY_FILENAME,
custom_test_file=NG40_TEST_FILES,
upgrade_agw="True",
upgrade_feg="True",
):
"""
Runs upgrade and s6a and gxgy tests once. This is run in the cron job on
magma-driver:
fab upgrade_and_run_3gpp_tests: 2>&1 | tee /tmp/teravm_cronjob.log
key_filename: path to where the private key is for authorized-key based
ssh. The public key counterpart needs to in the authorized_keys file on
the remote host. If empty file name is passed, password-based ssh will
work instead. This can be used if the script is run manually.
custom_test_file: a 3gpp test file to run. The default uses s6a and gxgy
"""
err = upgrade_teravm(setup, hash, key_filename, upgrade_agw, upgrade_feg)
if err:
sys.exit(1)
fastprint("\nSleeping for 30 seconds to make sure system is read\n\n")
time.sleep(30)
verdicts = run_3gpp_tests(setup, key_filename, custom_test_file)
def update_project():
"""
Update Django project's files on the remote host.
It works by syncing contents of the project dir on the local machine
with the corresponding one on the remote staging/production server(s).
Therefore, to deploy a given version of the Django project, all you need
to do is to checkout the chosen version on the local machine before starting
the deploy process.
Notice that you can prevent some files from being synced to the server via the
``RSYNC_EXCLUDE`` config option.
"""
require('project_root', provided_by=('staging', 'production'))
if env.environment == 'production':
if not console.confirm(
'Are you sure you want to deploy to the production server(s)?',
default=False):
abort('Production deployment aborted.')
with hide('commands'):
fastprint("Updating Django project files..." % env, show_prefix=True)
# defaults rsync options:
# -pthrvz
# -p preserve permissions
# -t preserve times
# -h output numbers in a human-readable format
# -r recurse into directories
# -v increase verbosity
# -z compress file data during the transfer
extra_opts = '--omit-dir-times'
rsync_project(
remote_dir=env.project_root,
local_dir=env.local_project_root + os.path.sep,
exclude=env.rsync_exclude,
delete=True,
extra_opts=extra_opts,
)
fastprint(" done." % env, end='\n')
fastprint("Updating settings & URLconfs modules..." % env,
show_prefix=True)
with lcd(os.path.join(env.local_project_root, env.project)):
with cd(os.path.join(env.project_root, env.project)):
# update Django settings module
settings_file = 'settings_%(environment)s.py' % env
put(settings_file, settings_file, mode=0644)
# update Django main URLconf module
urls_file = 'urls_%(environment)s.py' % env
put(urls_file, urls_file, mode=0644)
fastprint(" done." % env, end='\n')
## fastprint("Updating WSGI script..." % env, show_prefix=True)
## with lcd(os.path.join(env.local_repo_root, env.project)):
## with cd(os.path.join(env.domain_root, 'private')):
## source = '%(environment)s.wsgi' % env
## dest = 'django.wsgi'
## put(source, dest, mode=0644)
fastprint(" done." % env, end='\n')
# trigger code reloading
touch_WSGI_script()
def start():
"""
Start the webserver on the remote host.
"""
with hide('stdout', 'running'):
fastprint("Starting Memcached service..." % env, show_prefix=True)
sudo('/etc/init.d/memcached start')
fastprint(" done." % env, end='\n')
def release(app_name, do_pip):
""" Release latest source and dependent files and packages for the named app """
with cd('/opt/deploy/%s' % app_name):
fastprint('releasing latest source files')
run('git pull')
if do_pip:
with prefix('source /opt/virtualenvs/%s/bin/activate' % app_name):
run("pip install -r requirements.txt")
def create_db():
"""
Create the application's database on the server machine.
"""
require('db_name', provided_by=('staging', 'production'))
fastprint("Creating database `%(db_name)s'... " % env, show_prefix=True)
run('createdb %(db_name)s' % env)
fastprint("Created database %(db_name)s." % env, end='\n')
def db_user_exists():
"""
Check whether the db user role exists
"""
require('db_user', provided_by=('staging', 'production'))
fastprint("Check PostgreSQL user `%(db_user)s' ..." % env, show_prefix=True)
res = run('psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname=\'%(db_user)s\'"' % env)
return res != ""
def compress_venv(self):
fastprint('Compressing virtualenv')
local(
'tar -C %(build_dir)s/.. -cjf %(tarbz_path)s %(dirname)s' % {
'build_dir': self.deployer.build_dir,
'tarbz_path': self.tarbz_path,
'dirname': os.path.basename(self.deployer.build_dir)
})
def restart_tomcat():
"""
Restart Tomcat servlet engine.
"""
require('tomcat_controller', provided_by=('staging', 'production'))
fastprint("Restarting Tomcat...", show_prefix=True)
sudo('%(tomcat_controller)s restart' % env)
fastprint(" done." % env, end='\n')
def restart():
"""
Restart the webserver on the remote host.
"""
with hide('stdout', 'running'):
fastprint("Restarting Nginx webserver...", show_prefix=True)
sudo('/etc/init.d/nginx restart')
fastprint(" done." % env, end='\n')
def test_fastprint():
"""
fastprint() should print its input to sys.stdout
"""
s = 'zoom!'
fastprint(s)
result = sys.stdout.getvalue()
eq_(result, s)
def disable_site(site_name):
"""
Disable a webserver's virtualhost.
"""
with hide('stdout', 'running'):
fastprint("Disabling site %s..." % site_name, show_prefix=True)
sudo('a2dissite %s' % site_name)
fastprint(" done." % env, end='\n')
def test_fastprint():
"""
fastprint() should print its input to sys.stdout
"""
s = 'zoom!'
fastprint(s)
result = sys.stdout.getvalue()
eq_(result, s)
def stop():
"""
Stop the webserver on the remote host.
"""
with hide('stdout', 'running'):
fastprint("Stopping Apache webserver..." % env, show_prefix=True)
sudo('/etc/init.d/apache2 stop')
fastprint(" done." % env, end='\n')
def rebuild_index():
"""
Rebuild Solr index from scratch.
"""
require('settings', provided_by=('staging', 'production'))
fastprint("Rebuilding Solr index... ", show_prefix=True)
run_venv('django-admin.py rebuild_index --noinput --settings=%(settings)s' % env)
fastprint(" done." % env, end='\n')
def reload():
"""
Reload the webserver on the remote host.
"""
with hide('stdout', 'running'):
fastprint("Reloading Apache webserver..." % env, show_prefix=True)
sudo('/etc/init.d/apache2 reload')
fastprint(" done." % env, end='\n')
def start():
"""
Start the webserver on the remote host.
"""
with hide('stdout', 'running'):
fastprint("Starting Memcached service..." % env, show_prefix=True)
sudo('/etc/init.d/memcached start')
fastprint(" done." % env, end='\n')
def start():
"""
Start the webserver on the remote host.
"""
with hide("stdout", "running"):
fastprint("Starting Memcached service..." % env, show_prefix=True)
sudo("/etc/init.d/memcached start")
fastprint(" done." % env, end="\n")
def release(app_name, do_pip):
""" Release latest source and dependent files and packages for the named app """
with cd('/opt/deploy/%s' % app_name):
fastprint('releasing latest source files')
run('git pull')
if do_pip:
with prefix('source /opt/virtualenvs/%s/bin/activate' % app_name):
run("pip install -r requirements.txt")
def status():
"""
Drops {0} which is a json formatted file that contains a
status message that will be displayed to all users on the
on the courseware for a single course or for all courses
if 'global' is set.
Message(s) are entered or removed interactively on the console.
Example usage:
$ fab groups:prod_edx status
""".format(status_file)
with hide('running', 'stdout', 'stderr', 'warnings'):
env_json = sudo("cat /opt/wwc/lms-xml.env.json")
course_listings = json.loads(env_json)['COURSE_LISTINGS']
course_ids = [course_id for course_list in course_listings.itervalues()
for course_id in course_list]
course_ids = ['global'] + course_ids
with no_ts():
course_status = None
with settings(warn_only=True):
cur_status = noopable(sudo)('cat {0}'.format(status_file))
try:
course_status = json.loads(cur_status)
# add empty entries for courses not in the list
empty_entries = set(course_ids) - set(course_status.keys())
course_status.update({entry: '' for entry in list(empty_entries)})
except ValueError:
fastprint(red("Not a valid json file, overwritting\n"))
if course_status is None:
course_status = {course: '' for course in course_ids}
new_status = multi_choose_with_input(
'Set the status message, blank to disable:',
course_status)
if new_status is not None:
# remove empty entries
new_status = {entry: new_status[entry]
for entry in new_status if len(new_status[entry]) > 1}
with unsquelched():
if not console.confirm(
'Setting new status message:\n{0}'.format(
blue(str(new_status), bold=True)),
default=False):
abort('Operation cancelled by user')
with tempfile.NamedTemporaryFile(delete=True) as f:
f.write(json.dumps(new_status))
f.flush()
execute(update_status, f.name)
else:
abort('Operation cancelled by user')
def kick():
"""Give Apache and MongoDB a restart."""
utils.fastprint("Restarting MongoDB ... ")
api.sudo('service mongodb restart')
print(colors.green(" ok ", True))
utils.fastprint("Restarting Apache2 ... ")
api.sudo('service apache2 restart')
print(colors.green(" ok " , True))
def _pre_pip():
pip_version = run('{0} --version'.format(env.pip_basic_cmd)).split(' ')[1]
if pip_version >= env.pip_version:
fastprint(green('pip ready'), end='\n')
else:
warn(red('installing pip'))
run('{0} install --upgrade pip'.format(env.pip_basic_cmd))
warn(red('upgraded pip'))
fastprint(green('pip ready'), end='\n')
def cleanup_containers(**kwargs):
"""
Removes all containers that have finished running. Similar to the ``prune`` functionality in newer Docker versions.
"""
containers = docker_fabric().cleanup_containers(**kwargs)
if kwargs.get('list_only'):
puts('Existing containers:')
for c_id, c_name in containers:
fastprint('{0} {1}'.format(c_id, c_name), end='\n')
def rebuild_index():
"""
Rebuild Solr index from scratch.
"""
require('settings', provided_by=('staging', 'production'))
with hide('commands'):
fastprint("Rebuilding Solr index... ", show_prefix=True)
run_venv('django-admin.py rebuild_index --settings=%(settings)s' % env)
fastprint(" done." % env, end='\n')
def update_index():
"""
Update Solr index.
"""
require('settings', 'virtualenv_root', provided_by=('staging', 'production'))
with hide('stdout', 'running'):
fastprint("Updating Solr index... ", show_prefix=True)
run_venv('django-admin.py update_index --settings=%(settings)s' % env)
fastprint(" done." % env, end='\n')
def install_system_requirements():
"""
Install generic OS-level software required by OpenOrdini.
"""
fastprint("Installing generic software requirements...", show_prefix=True, end='\n')
with hide('commands'):
for pkg_name in env.provision_packages:
fastprint("* %s" % pkg_name, end='\n')
install_package(pkg_name)
def restart_tomcat():
"""
Restart Tomcat servlet engine.
"""
require('tomcat_controller', provided_by=('staging', 'production'))
fastprint("Restarting Tomcat...", show_prefix=True)
with hide('commands'):
sudo('%(tomcat_controller)s restart' % env)
fastprint(" done." % env, end='\n')
def restart_postgres():
"""
Restart PostgreSQL database server.
"""
require('postgres_controller', provided_by=('staging', 'production'))
fastprint("Restarting Postgres...", show_prefix=True)
with hide('commands'):
sudo('%(postgres_controller)s restart' % env)
fastprint(" done." % env, end='\n')
def media_from_server(settings='develop'):
"""
Copia tutti i file media dal server definito dal settings passato come argomento
"""
server = ServerUtil(settings)
if confirm("Stai sovrascrivendo tutti i file contenuti in /media/ in locale con quelli sul server {}. Vuoi procedere?".format(settings.upper())):
local("rsync -av --progress --inplace --rsh='ssh -p{}' {}@{}:{}/media/ ./media/".format(
server.port, server.user, server.ip, server.working_dir))
fastprint("Ricordati che sincronizzando i file media e' necessario sincronizzare anche il database con il comando 'fab db_from_server'.")
def cleanup_containers(**kwargs):
"""
Removes all containers that have finished running. Similar to the ``prune`` functionality in newer Docker versions.
"""
containers = docker_fabric().cleanup_containers(**kwargs)
if kwargs.get('list_only'):
puts('Existing containers:')
for c_id, c_name in containers:
fastprint('{0} {1}'.format(c_id, c_name), end='\n')
def update_index():
"""
Update Solr index.
"""
require('settings', 'virtualenv_root', provided_by=('staging', 'production'))
with hide('stdout', 'running'):
fastprint("Updating Solr index... ", show_prefix=True)
run_venv('django-admin.py update_index --settings=%(settings)s' % env)
fastprint(" done." % env, end='\n')
