def update_ed_db(instance):
""" upgrade the instance database schema
"cd" command is executed manually (not in a context manager)
because it is not good to use global variable with parallel
"""
path = "{env}/{instance}".format(env=env.ed_basedir, instance=instance)
run("cd " + path + " && PYTHONPATH=. alembic upgrade head")
def enable_rabbitmq_kraken():
""" Enable kraken rabbitmq connection through iptables
"""
if env.dry_run is True:
print("iptables --delete OUTPUT --protocol tcp -m tcp --dport 5672 --jump DROP")
else:
run("iptables --delete OUTPUT --protocol tcp -m tcp --dport 5672 --jump DROP")
def mail_init():
print(green("Installing mail server packages."))
run("sudo apt-get update && "
"sudo apt-get -y upgrade && "
"sudo apt-get -y dist-upgrade && "
"sudo apt-get -y autoremove")
run("sudo apt-get -y install exim4")
def remove_at_instance(instance):
"""Remove an at / connector_rt instance entirely
* Remove the cron
* purge logs
"""
# ex.: /var/log/connectors-rt/at_fr-bou
run("rm -f %s/at_%s" % (env.AT_BASE_LOGDIR, instance))
def __install_apache_package( self, path ):
"""
Download the given file from an Apache download mirror.
Some mirrors may be down or serve crap, so we may need to retry this a couple of times.
"""
# TODO: run Fabric tasks with a different manager, so we don't need to catch SystemExit
components = path.split( '/' )
package, tarball = components[ 0 ], components[ -1 ]
tries = iter( xrange( 3 ) )
while True:
try:
mirror_url = self.__apache_s3_mirror_url( path )
if run( "curl -Ofs '%s'" % mirror_url, warn_only=True ).failed:
mirror_url = self.__apache_official_mirror_url( path )
run( "curl -Ofs '%s'" % mirror_url )
try:
sudo( fmt( 'mkdir -p {install_dir}/{package}' ) )
sudo( fmt( 'tar -C {install_dir}/{package} '
'--strip-components=1 -xzf {tarball}' ) )
return
finally:
run( fmt( 'rm {tarball}' ) )
except SystemExit:
if next( tries, None ) is None:
raise
else:
log.warn( "Could not download or extract the package, retrying ..." )
def _setup_package_repos( self ):
super( ToilJenkinsSlave, self )._setup_package_repos( )
sudo( "apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF" )
distro = run( "lsb_release -is | tr '[:upper:]' '[:lower:]'" )
codename = run( "lsb_release -cs" )
run( 'echo "deb http://repos.mesosphere.io/{} {} main"'
'| sudo tee /etc/apt/sources.list.d/mesosphere.list'.format( distro, codename ) )
def __install_mesos_egg( self ):
# FIXME: this is the ubuntu 14.04 version. Wont work with other versions.
run( "wget http://downloads.mesosphere.io/master/ubuntu/14.04/"
"mesos-0.22.0-py2.7-linux-x86_64.egg" )
# we need a newer version of protobuf than comes default on ubuntu
sudo( "pip install --upgrade protobuf", pty=False )
sudo( "easy_install mesos-0.22.0-py2.7-linux-x86_64.egg" )
def __patch_jenkins_config( self ):
"""
A context manager that retrieves the Jenkins configuration XML, deserializes it into an
XML ElementTree, yields the XML tree, then serializes the tree and saves it back to
Jenkins.
"""
config_file = StringIO( )
if run( 'test -f ~/config.xml', quiet=True ).succeeded:
fresh_instance = False
get( remote_path='~/config.xml', local_path=config_file )
else:
# Get the in-memory config as the on-disk one may be absent on a fresh instance.
# Luckily, a fresh instance won't have any configured security.
fresh_instance = True
config_url = 'http://localhost:8080/computer/(master)/config.xml'
with hide( 'output' ):
config_file.write( run( 'curl "%s"' % config_url ) )
config_file.seek( 0 )
config = ElementTree.parse( config_file )
yield config
config_file.truncate( 0 )
config.write( config_file, encoding='utf-8', xml_declaration=True )
if fresh_instance:
self.__service_jenkins( 'stop' )
try:
put( local_path=config_file, remote_path='~/config.xml' )
finally:
if fresh_instance:
self.__service_jenkins( 'start' )
else:
log.warn( 'Visit the Jenkins web UI and click Manage Jenkins - Reload '
'Configuration from Disk' )
def finish_configuring_slave():
parameter_default_values()
with settings(host_string=env.pgslave_user_host):
# upload repmgr.conf on slave server
repmgr_context = dict(cluster_name=env.cluster_name,
node_number=env.slave_node_number,
sync_user=env.sync_user,
sync_db=env.sync_db,
sync_pass=env.sync_pass,
ssh_port=env.master_ssh_port,
)
repmgr_conf_file = 'conf/repmgr/repmgr.conf'
if not isfile(repmgr_conf_file):
repmgr_conf_file = '%s/%s' % (pg_fabrep_path, repmgr_conf_file)
upload_template(repmgr_conf_file, env.master_pgdata_path,
context=repmgr_context, backup=False)
slave_postgresql_conf = "%spostgresql.conf" % env.slave_pgdata_path
slave_postgresql_conf_bck = "%spostgresql.conf.bck" % env.slave_pgdata_path
sudo('mv %s %s' % (slave_postgresql_conf, slave_postgresql_conf_bck))
sudo("sed '/hot_standby =/c hot_standby = on' %s > %s" % \
(slave_postgresql_conf_bck, slave_postgresql_conf))
sudo("mkdir -p %s" % env.slave_pgconf_path)
sudo("cp %spg_hba.conf %s" % (env.slave_pgdata_path, env.slave_pgconf_path))
sudo("cp %spg_ident.conf %s" % (env.slave_pgdata_path, env.slave_pgconf_path))
sudo("cp %spostgresql.conf %s" % (env.slave_pgdata_path, env.slave_pgconf_path))
run("sudo -u postgres pg_ctl -D /var/lib/postgresql/%(postgres_version)s/%(cluster_name)s/ start" % env)
def test_silent_commands_should_not_have_blank_line(self):
"""
Silent commands should not generate an extra trailing blank line
After the move to interactive I/O, it was noticed that while run/sudo
commands which had non-empty stdout worked normally (consecutive such
commands were totally adjacent), those with no stdout (i.e. silent
commands like ``test`` or ``mkdir``) resulted in spurious blank lines
after the "run:" line. This looks quite ugly in real world scripts.
"""
env.password = None
env.no_agent = env.no_keys = True
env.key_filename = CLIENT_PRIVKEY
with password_response(CLIENT_PRIVKEY_PASSPHRASE, silent=False):
run('normal')
run('silent')
run('normal')
with hide('everything'):
run('normal')
run('silent')
expected = """
[%(prefix)s] run: normal
[%(prefix)s] Login password for '%(user)s':
[%(prefix)s] out: foo
[%(prefix)s] run: silent
[%(prefix)s] run: normal
[%(prefix)s] out: foo
""" % {'prefix': env.host_string, 'user': env.user}
eq_(expected[1:], sys.stdall.getvalue())
def kill_and_run(process, command, num_of_attempts=3):
run('pkill %s' % process)
for i in range(num_of_attempts):
sleep(5)
if not run('pgrep -l %s' % process):
run(command)
break
def setup_web():
require.deb.packages(['git', 'npm', 'nodejs-legacy', 'python3-dev', 'libxml2-dev', 'libxslt1-dev', 'libpq-dev'])
require.nginx.server()
require.nginx.site(env.server_name, template_source='nginx-site',
port=80,
server_alias='',
static_path=env.static_path)
# require.nginx.disabled('default')
with settings(warn_only=True):
if run('type bower').return_code:
sudo('npm install -g bower')
update_source()
require.directory(env.home + '/logs')
require.python.packages(['uwsgi', 'virtualenv', 'ipython', 'celery'], use_sudo=True)
require.file('/etc/init/uwsgi.conf', source='uwsgi.conf', use_sudo=True)
# require.file('/etc/init/celery.conf', source='celery.conf', use_sudo=True)
require.directory('/etc/uwsgi', use_sudo=True)
require.files.template_file('/etc/uwsgi/%s.ini' % env.project_name,
template_source='uwsgi.ini',
context=env, use_sudo=True)
require.service.started('uwsgi')
require.directory(env.home + '/bin')
require.files.template_file(env.home + '/bin/djangorc',
template_source='bin/djangorc',
context=env)
require.files.template_file(env.home + '/bin/loadenv',
template_source='bin/loadenv',
context=env)
run('chmod +x ' + env.home + '/bin/loadenv')
def test_password_memory_on_user_switch(self):
"""
Switching users mid-session should not screw up password memory
"""
def _to_user(user):
return join_host_strings(user, env.host, env.port)
user1 = 'root'
user2 = USER
with settings(hide('everything'), password=None):
# Connect as user1 (thus populating both the fallback and
# user-specific caches)
with settings(
password_response(PASSWORDS[user1]),
host_string=_to_user(user1)
):
run("ls /simple")
# Connect as user2: * First cxn attempt will use fallback cache,
# which contains user1's password, and thus fail * Second cxn
# attempt will prompt user, and succeed due to mocked p4p * but
# will NOT overwrite fallback cache
with settings(
password_response(PASSWORDS[user2]),
host_string=_to_user(user2)
):
# Just to trigger connection
run("ls /simple")
# * Sudo call should use cached user2 password, NOT fallback cache,
# and thus succeed. (I.e. p_f_p should NOT be called here.)
with settings(
password_response('whatever', times_called=0),
host_string=_to_user(user2)
):
sudo("ls /simple")
def cleanup_db_backups(params):
"""
Cleanup sql backup files from folder
"""
print yellow("Warning mysql.cleanup_db_backups is deprecated from version 1.0")
params = utils.format_params(params)
if not 'path' in params:
abort(red("No path param set!"))
if not 'max_backup_history' in params:
params['max_backup_history'] = 5
with cd(params['path']):
folder_result = run("ls -tr1 | grep '\.tar.gz$'")
if len(folder_result) > 0:
files = folder_result.split('\n')
current_file_count = len(files)
print("%s backup files found..." % current_file_count)
if len(files) > params['max_backup_history']:
total_to_remove = len(files) - params['max_backup_history']
print("Going to remove `%s` files" % total_to_remove)
for file in files[0:total_to_remove]:
file_path = "%s/%s" % (params['path'], file.strip())
print("- %s" % file_path)
run("rm %s" % (file_path))
else:
print("No sql backup files to remove... limit is set to `%s`" % params['max_backup_history'])
else:
print(green("No sql backup files available..."))
def import_file(params):
"""
Given the database credentials and a import file it will import into a database
"""
print yellow("Warning mysql.import_file is deprecated from version 1.0")
params = utils.format_params(params)
if not exists(params['import_file']):
print(yellow("Mysql file `%s` does not exist, so no import is executed." % params['import_file']))
else:
command = """
mysql -h %(host)s -u %(user)s --password='******' %(database)s < %(import_file)s
"""
# Make params
command_params = {'user': params['user'],
'password': params['password'],
'database': params['database'],
'host': params['host'],
'import_file':params['import_file']}
run(command % command_params)
print(green("Mysql file `%s` successfully imported." % command_params['import_file']))
def hd0():
"""hadoop uninstall => fab sete:2,5 hd0"""
# 0. rm hadoop
file_i = os.path.join(DEPLOY_HOME, 'hadoop')
run('rm -rf {0}'.format(file_i))
file_i = os.path.join(APP_HOME, 'hadoop')
run('rm -rf {0}'.format(file_i))
def test_quiet_sets_warn_only_to_true(self):
# Sanity test to ensure environment
with settings(warn_only=False):
with quiet():
eq_(run("barf").return_code, 1)
# Kwarg test
eq_(run("barf", quiet=True).return_code, 1)
def dump_db(dumpfile="pathagarh_DB_backup.sql"):
"""Dumps the DB as a SQL script and downloads it"""
require('environment', provided_by=[production, staging])
if isdir(dumpfile):
print("dumpfile '%s' is a directory! Aborting." % dumpfile)
elif (not isfile(dumpfile) or
confirm('\n%s already exists locally. Do you want to overwrite it?'
% dumpfile, default=False)):
remote_filename = '%s/%s' % (env['project_path'], dumpfile)
if (not exists(remote_filename) or
confirm('\n%s already exists. Do you want to overwrite it?'
% remote_filename, default=False)):
print('\nDumping DB...')
with settings(hide('stderr')):
run('mysqldump -u %s %s %s > %s' %
(env['db_user'], env['db_password_opt'],
env['db_name'], remote_filename))
get(remote_filename, '.')
run('rm %s' % (remote_filename))
else:
print('\nAborting.')
else:
print('\nAborting.')
def zk0():
"""zookeeper uninstall => fab sete:2,4 zk0"""
# 0. rm zookeeper
file_i = os.path.join(DEPLOY_HOME, 'zookeeper')
run('rm -rf {0}'.format(file_i))
file_i = os.path.join(APP_HOME, 'zookeeper')
run('rm -rf {0}'.format(file_i))
def _putl(source_file, dest_dir):
"""
To be used instead of put, since it doesn't support symbolic links
"""
put(source_file, '/')
run("mv -f /{0} {1}".format(os.path.basename(source_file), dest_dir))
def load_db(dumpfile=None):
"""Loads data from a SQL script to Pootle DB"""
require('environment', provided_by=[production, staging])
if dumpfile is not None:
if isfile(dumpfile):
remote_filename = '%(project_path)s/DB_backup_to_load.sql' % env
if (not exists(remote_filename) or
confirm('\n%s already exists. Do you want to overwrite it?'
% remote_filename, default=False)):
print('\nLoading data into the DB...')
with settings(hide('stderr')):
put(dumpfile, remote_filename)
run('mysql -u %s %s %s < %s' %
(env['db_user'], env['db_password_opt'],
env['db_name'], remote_filename))
run('rm %s' % (remote_filename))
else:
print('\nAborting.')
else:
print('\nERROR: The file "%s" does not exist. Aborting.' % dumpfile)
else:
print('\nERROR: A (local) dumpfile must be provided. Aborting.')
def dump_db(dumpfile="pootle_DB_backup.sql"):
"""Dumps the DB as a SQL script and downloads it"""
require("environment", provided_by=[production, staging])
if isdir(dumpfile):
abort("dumpfile '%s' is a directory! Aborting." % dumpfile)
elif not isfile(dumpfile) or confirm(
"\n%s already exists locally. Do you want to overwrite it?" % dumpfile, default=False
):
remote_filename = "%s/%s" % (env["project_path"], dumpfile)
if not exists(remote_filename) or confirm(
"\n%s already exists. Do you want to overwrite it?" % remote_filename, default=False
):
print("\nDumping DB...")
with settings(hide("stderr")):
run(
"mysqldump -u %s %s %s > %s"
% (env["db_user"], env["db_password_opt"], env["db_name"], remote_filename)
)
get(remote_filename, ".")
run("rm %s" % (remote_filename))
else:
abort("\nAborting.")
else:
abort("\nAborting.")
def _update_requirements():
"""Updates dependencies defined in the requirements file"""
print('\n\nUpdating requirements...')
with prefix('source %(env_path)s/bin/activate' % env):
run('pip install -U -r %(project_repo_path)s/requirements/deploy.txt' % env)
run('chmod -R go=u,go-w %(env_path)s' % env)
def check_war_files():
with settings(warn_only=True):
with hide('output','running'):
for my_war in env.war_list:
if my_war['wartype'] == 'ACM':
for my_env in env.env_list:
#print my_war
if files.exists(env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']):
print "Found " + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']
result = run("jar tvf " + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile'] + " | egrep \"WEB-INF/lib/log4j*.jar WEB-INF/lib/jaxen\-*.jar\"")
if result.return_code == 0:
print result.stdout
elif result.return_code == 1:
print 'SUCCESS, no unwanted jars present in the file: ' + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']
if my_war['wartype'] == 'SOA':
for my_env in env.env_list:
if my_env.find("SOA"):
if files.exists(env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']):
print "Found " + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']
result = run("jar tvf " + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile'] + " | egrep \"WEB-INF/lib/jaxp-ri*\.jar WEB-INF/lib/jaxp-api*\.jar WEB-INF/lib/xercesImpl*\.jar WEB-INF/lib/log4j*\.jar\"")
if result.return_code == 0:
print result.stdout
elif result.return_code == 1:
print 'SUCCESS, no unwanted jars present in the file: ' + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']
if my_war['wartype'] == 'CL':
for my_env in env.env_list:
if my_env.find("CL"):
if files.exists(env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']):
print "Found " + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']
result = run("jar tvf " + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile'] + " | egrep \"WEB-INF/lib/jaxp-ri*\.jar WEB-INF/lib/jaxp-api*\.jar WEB-INF/lib/xercesImpl*\.jar WEB-INF/lib/log4j-*\.jar\"")
if result.return_code == 0:
print result.stdout
elif result.return_code == 1:
print 'SUCCESS, no unwanted jars present in the file: ' + env.jboss_root + "/server/" + my_env + "/deploy/" + my_war['warfile']
def _init_virtualenv():
"""Creates initial virtualenv"""
print('\n\nCreating virtualenv...')
run('virtualenv -p %(python)s --no-site-packages %(env_path)s' % env)
with prefix('source %(env_path)s/bin/activate' % env):
run('easy_install pip')
def test_systemd_config_changes(self):
self.is_ubuntu_18 = True
if self.bundle_type == 'remote_jar':
state.env.user = '******'
state.env.password = '******'
state.env.host_string = 'localhost:2223'
self.do_deploy(host='stage')
pre_config_status = operations.run("ps aux | grep {0} | grep verbose | grep -v grep".format(self.service_name),
warn_only=True)
self.assertTrue(pre_config_status.failed)
# Emulate chef change to systemd config
operations.sudo(
"sed -i 's/\/usr\/bin\/java -jar/\/usr\/bin\/java -verbose -jar/;' /etc/systemd/system/{0}.service".format(
self.service_name)
)
# Have to reload systemd when source config changes on disk
reload = operations.sudo("systemctl daemon-reload")
self.assertTrue(reload.succeeded)
self.do_deploy(host='stage')
config_status = operations.run("ps aux | grep {0} | grep verbose | grep -v grep".format(self.service_name), warn_only=True)
self.assertTrue(config_status.succeeded)
else:
pass
def incoming(remote="origin", branch=None):
"""
Displays incoming commits
"""
if not branch:
branch = env.branch
run(("cd %(path)s; git fetch " + remote + " && git log --oneline .." + remote + "/" + branch) % env)
def _remove_config():
"""Removes server configuration files"""
print('\n\nRemoving server configuration...')
sudo('rm -rf %(vhost_file)s' % env)
run('rm -rf %(wsgi_file)s' % env)
run('rm -rf %(project_settings_path)s/90-%(environment)s-local.conf' % env)
def _install_apache_package( self, remote_path, install_dir ):
"""
Download the given package from an Apache download mirror and extract it to a child
directory of the directory at the given path.
:param str remote_path: the URL path of the package on the Apache download server and its
mirrors.
:param str install_dir: The path to a local directory in which to create the directory
containing the extracted package.
"""
# TODO: run Fabric tasks with a different manager, so we don't need to catch SystemExit
components = remote_path.split( '/' )
package, tarball = components[ 0 ], components[ -1 ]
# Some mirrors may be down or serve crap, so we may need to retry this a couple of times.
tries = iter( xrange( 3 ) )
while True:
try:
mirror_url = self.__apache_s3_mirror_url( remote_path )
if run( "curl -Ofs '%s'" % mirror_url, warn_only=True ).failed:
mirror_url = self.__apache_official_mirror_url( remote_path )
run( "curl -Ofs '%s'" % mirror_url )
try:
sudo( fmt( 'mkdir -p {install_dir}/{package}' ) )
sudo( fmt( 'tar -C {install_dir}/{package} '
'--strip-components=1 -xzf {tarball}' ) )
return
finally:
run( fmt( 'rm {tarball}' ) )
except SystemExit:
if next( tries, None ) is None:
raise
else:
log.warn( "Could not download or extract the package, retrying ..." )
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'):
"""
Remove queue for a kraken
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
if env.rabbitmq_host_api == 'localhost':
host_string = env.roledefs['tyr_master'][0]
else:
host_string = env.rabbitmq_host_api
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host_string):
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_rt"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_task"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
def test_connection():
print(colored('\n\n----- LOCAL -----\n\n', 'yellow'))
local('uname -a')
print(colored('\n\n----- REMOTE -----\n\n', 'yellow'))
run('uname -a')
def ls():
run('cd %(path)s;ls -lAF' % env)
def uname():
print env.host_string
run('uname -a')
def ps():
"""
show processes of this user
"""
run('ps -f -u %(user)s | grep -v "ps -f" | grep -v sshd' % env)
def setup_celery_dir():
''' Setup the celery directory for logging and pid storage '''
with settings(warn_only=True):
# delete old logfile
run("rm -r %s" % LOG_FILE)
run("mkdir -p /tmp/celery/")
def git_clone():
"""
"""
run("cd ~/; git clone {}".format(env.repository))
def set_up_server():
print(colored("\n =========== Install dependencies ===========", "yellow"))
run('sudo apt-get -y update')
run('sudo apt-get -y install git fish vim')
print(colored("\n =========== Install python ===========", "yellow"))
run('sudo add-apt-repository -y ppa:deadsnakes/ppa')
run('sudo apt-get -y update')
run('sudo apt-get -y install python3.6')
print(colored("\n =========== Download pip ===========", "yellow"))
run('mkdir -p ~/Downloads/')
with cd('~/Downloads'):
run('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py')
print(colored('\n ======= Install supervisord =======', 'yellow'))
run("sudo apt-get -y install supervisor")
print(colored('\n ======= Install pip and flask =======', 'yellow'))
with cd('~/Downloads'):
run('sudo python3.6 get-pip.py')
run('sudo pip install Flask requests Flask-RESTful termcolor')
print(
colored('\n ======= Change permissions prior to pull =======',
'yellow'))
with warn_only():
run('sudo chmod -R 777 /home/single-payer')
print(colored('\n ======= Cloning repo =======', 'yellow'))
with cd("/home"):
with warn_only():
run('rm -rf single-payer')
run('sudo git clone https://github.com/alexgoodell/single-payer.git')
print(colored('\n ======= Change permissions after pull =======',
'yellow'))
run('sudo chmod -R 777 /home/single-payer')
print(colored('\n ======= Configure supervisord =======', 'yellow'))
with warn_only():
run("sudo service supervisor stop")
run('sudo rm -rf /etc/supervisor/conf.d/*.conf')
run('sudo rm -rf /var/log/sp.err.log')
run('sudo rm -rf /var/log/sp.out.log')
run('sudo touch /var/log/sp.err.log')
run('sudo touch /var/log/sp.out.log')
put('supervisor/single-payer.conf',
'/etc/supervisor/conf.d',
use_sudo=True)
print(
colored('\n ======= Start test server / supervisord =======',
'yellow'))
with warn_only():
run("sudo service supervisor start")
run('sudo supervisorctl reread')
run('sudo supervisorctl update')
status = run('sudo supervisorctl status')
if "RUNNING" in status:
print(
colored('\n ----> Server launched at IP {}'.format(env.host),
'green'))
else:
print(colored('\n ----> !! Error !! Server failed to launch', 'red'))
print(
colored(
'\n ----> !! Warning !! Currently using DEVELOPMENT version of flask server',
'yellow'))
def post_deploy(config):
with cd(config["compose_path"]):
run(config["post_deploy"])
def deploy(build, project_root):
if build == 'prod':
local('npm run build')
elif build == 'test':
local('npm run build-test')
else:
print("Please deploy in specific mode like test or production.")
return
with cd(project_root):
run('mkdir deploy', quiet=True)
put('dist', 'deploy')
run('rm -rf dist_bak')
run('mv dist dist_bak', quiet=True)
run('rm -rf dist')
run('mv deploy/dist dist')
run('rm -rf deploy')
def refresh_server():
print(
colored('\n ======= Change permissions prior to pull =======',
'yellow'))
run('sudo chmod -R 777 /home/single-payer')
print(colored('\n ======= Cloning repo =======', 'yellow'))
with cd("/home"):
with warn_only():
run('rm -rf single-payer')
run('sudo git clone https://github.com/alexgoodell/single-payer.git')
print(colored('\n ======= Change permissions after pull =======',
'yellow'))
run('sudo chmod -R 777 /home/single-payer')
print(colored('\n ======= Configure supervisord =======', 'yellow'))
with warn_only():
run("sudo service supervisor stop")
run('sudo rm -rf /etc/supervisor/conf.d/*.conf')
run('sudo rm -rf /var/log/sp.err.log')
run('sudo rm -rf /var/log/sp.out.log')
run('sudo touch /var/log/sp.err.log')
run('sudo touch /var/log/sp.out.log')
put('supervisor/single-payer.conf',
'/etc/supervisor/conf.d',
use_sudo=True)
print(
colored('\n ======= Start test server / supervisord =======',
'yellow'))
with warn_only():
run("sudo service supervisor start")
run('sudo supervisorctl reread')
run('sudo supervisorctl update')
status = run('sudo supervisorctl status')
if "RUNNING" in status:
print(
colored('\n ----> Server launched at IP {}'.format(env.host),
'green'))
else:
print(colored('\n ----> !! Error !! Server failed to launch', 'red'))
print(
colored(
'\n ----> !! Warning !! Currently using DEVELOPMENT version of flask server',
'yellow'))
def host_type():
run('uname -a')
def reload_docker(config):
with cd(config["compose_path"]):
run('docker-compose up -d')
def get_app(version):
run(('wget ' + 'https://codeload.github.com/phodal/growth_studio/tar.gz/v' + '%s') % version)
run('tar xvf v%s' % version)
def do_deploy(archive_path):
"""function to deploy"""
if not path.exists(archive_path):
return False
try:
h, t = ntpath.split(archive_path)
if t:
archive = t
else:
archive = ntpath.basename(h)
h, t = ntpath.splitext(archive)
if h:
name = h
else:
name = ntpath.basename(h)
put(archive_path, "/tmp/{}".format(archive))
run("sudo mkdir -p /data/web_static/releases/{}/".format(name))
run("sudo tar -xzf /tmp/{} -C /data/web_static/releases/{}/".format(
archive, name))
run("sudo mv /data/web_static/releases/{}/web_static/*\
/data/web_static/releases/{}/".format(name, name))
run("sudo rm /tmp/{}".format(archive))
run("sudo rm -rf /data/web_static/current")
run("sudo rm -rf /data/web_static/releases/{}/web_static".format(name))
run("sudo ln -s /data/web_static/releases/{}/ /data/web_static/current"
.format(name))
print("New version deployed!")
except Exception:
return False
def config_app():
with cd('growth-studio'):
with prefix('source ' + virtual_env_path):
run('python manage.py collectstatic -v0 --noinput')
run('python manage.py migrate')
def prepare_ac():
with cd('growth-studio'):
with prefix('source ' + virtual_env_path):
run('echo "from django.contrib.auth.models import User; User.objects.create_superuser(%s, %s, %s)" | python manage.py shell' % ("'test'", "'*****@*****.**'", "'test'"))
def deploy():
with cd('/var/www/html'):
run('git pull --rebase origin master')
def setup_app(version):
with prefix('source ' + virtual_env_path):
run('pip3 install -r growth-studio-%s/requirements/prod.txt' % version)
run('rm -f growth-studio')
run('ln -s growth-studio-%s growth-studio' % version)
def dev(build_bundle=False,
branch_name='',
del_exist=False,
build_firefox_ext=False,
build_chrome_ext=False,
fetch_lang=False,
u=False):
"""
Clones a branch and deploys it to beta.developers.mega.co.nz.
It will then output a test link which can be pasted into a Redmine
ticket or run in the browser.
note: Unless specific username is given, this will tries to get local git email address to trackdown username to use on the beta server.
Beta server username and email used on gitlab should be match for this to work automaically.
Usage:
1) Enter your code directory and run: Fab dev
2) Alternatively to clone any branch run:
Fab dev:xxx-branch-name
3) Delete existing branch on beta and clone new one:
Fab dev:del_exist=True
"""
# If username is given use it, otherwise try to use same username from local git.
if u:
username = u
else:
local_email = local('git config user.email', capture=True)
username = local_email.replace('@mega.co.nz', '')
BETA_DEV_HOST = username + '@beta.developers.mega.co.nz:28999'
# If none other given, use beta.developers server
if not env.host_string:
beta_dev()
env.host_string = BETA_DEV_HOST
# Get the current branch if not passed in
if branch_name == '':
branch_name = local('git rev-parse --abbrev-ref HEAD', capture=True)
# If branch name is still empty, something is wrong.
if branch_name == '':
print('Something went wrong with get current branch name.\n')
exit()
# Get the remote path e.g. /var/www/xxx-branch-name
remote_branch_path = os.path.join(env.target_dir, branch_name)
# Delete existing branch.
if del_exist:
print('deleting existing {} branch.\n').format(branch_name)
run('rm -rf {}'.format(remote_branch_path))
# Clone the repo into /var/www/xxx-branch-name
# but not the full git history to save on storage space
with cd(env.target_dir):
result = run('git clone --depth 1'
' [email protected]:web/webclient.git'
' {} -b {}'.format(branch_name, branch_name),
warn_only=True)
# If successful
if result.return_code == 0:
# Show last commit from the branch
with cd(remote_branch_path):
run('git log -1')
# Output beta server test link
print('\nCloned branch {} to {}'.format(branch_name,
remote_branch_path))
else:
print('Branch already exists on beta, updating instead.\n')
with cd(remote_branch_path):
# Force revert the bundle.js IF it was re-built in earlier deployments
run('git checkout js/chat/bundle.js')
run('git pull --update-shallow')
run('git log -1')
print('Pulling latest language file from babel.\n')
with cd(remote_branch_path):
run('./scripts/lang.sh')
# Update version info.
version = None
with cd(remote_branch_path):
run("touch current_ver.txt")
run("cat current_ver.txt >> last_ver.txt")
run("git rev-parse HEAD > current_ver.txt")
version = run("cat current_ver.txt")
# Installs dependencies and builds bundles.
if build_bundle:
_build_chat_bundle(remote_branch_path)
# Keep just the hostname e.g. [email protected]:28999 -> beta.developers.mega.co.nz
host_name = env.host_string.split('@')[-1].split(':')[0]
boot_html = ('sandbox3'
if env.host_string == SANDBOX3_HOST else 'devboot-beta')
# Build the Firefox extension if requested
if build_firefox_ext:
with cd('~/deployment/webclient-updatebuild/'):
run('BETA_BUILD=1 BETA_WEBCLIENT_PATH=/var/www/' +
branch_name + ' php 2b-update-firefox-web-ext.php')
# Build the Chrome extension if requested
if build_chrome_ext:
with cd('~/deployment/webclient-updatebuild/'):
run('BETA_BUILD=1 BETA_WEBCLIENT_PATH=/var/www/' +
branch_name + ' php 2a-update-chrome.php')
# Provide test link
print(
'Test link: https://{branch_name}.{host}/dont-deploy/sandbox3.html?apipath=prod&jj=1'
.format(host=host_name.replace("beta.", ""),
branch_name=branch_name,
boot_html=boot_html))
# Provide test link to Firefox ext
if build_firefox_ext:
print('Firefox ext link: https://{branch_name}.{host}/meganz.xpi'.
format(host=host_name.replace("beta.", ""),
branch_name=branch_name))
# Provide test link to Chrome ext
if build_chrome_ext:
print(
'Chrome ext link: https://{branch_name}.{host}/chrome-extension.zip'
.format(host=host_name.replace("beta.", ""),
branch_name=branch_name))
# Provide commit info
print("Latest commit deployed: {}".format(version))
def backup(self, filename):
run('PGPASSWORD={} pg_dump -d {} -U {} -h {} | bzip2 -9 -c > {}'.format(
self.password, self.database, self.user, self.hostname, filename))
def can_connect_postgresql():
result = run(f'psql -U {DB_USER} {DB_NAME} -c ""',
warn_only=True, quiet=True)
return not result.return_code
def deploy():
"""
Deploys to the web root of a server (only use for beta.mega.nz).
"""
with cd(env.target_dir):
run("touch current_ver.txt")
run("cat current_ver.txt >> ~/deployer/last_ver.txt")
run("git pull -u origin develop")
run("git rev-parse HEAD > current_ver.txt")
with cd("logger"):
run("git pull -u")
version = run("cat current_ver.txt")
print("Latest version deployed: {}".format(version))
def set_env():
env.project_path = Path(PROJECT_PATH)
env.home = Path(run('echo $HOME', quiet=True))
env.virtual_env = env.home / RELATIVE_WORKON_HOME / VIRTUALENV_NAME
def can_connect_redis():
result = run(f'redis-cli -s "{REDIS_SOCKET}" ECHO ""',
warn_only=True, quiet=True)
return not result.return_code
def update_index():
with workon_dezede():
run('./manage.py update_index')
def install_less_css():
result = run('lessc', warn_only=True, quiet=True)
if result.return_code:
sudo('npm install')
sudo('ln -s /usr/bin/nodejs /usr/bin/node', warn_only=True)
def set_permissions():
with workon_dezede():
if not exists('media'):
run('mkdir media')
run('chmod -R o-rwx *', warn_only=True)
run('chmod -R o+rx static media')
def save_remote_db():
run(f'pg_dump -U {DB_USER} -Fc -b -v -f "{REMOTE_BACKUP}" {DB_NAME}')
local(f'rsync --info=progress2 '
f'"{env.hosts[0]}":"{REMOTE_BACKUP}" "{LOCAL_BACKUP}"')
def collectstatic():
with workon_dezede():
run('./manage.py collectstatic --noinput')
def migrate_db():
with workon_dezede():
run('./manage.py migrate')
