def upload_static_src(c, local_path, serv_path, includes):
archive_name = 'src.'\
+datetime.utcnow().strftime('%y%m%d_%H%M%S')\
+'.tar.gz'
#includes = ['*.jpg', '*.mp4', '*.svg', '*.png', '*.doc']
tar_cmd = 'cd '+Config[local_path]+' && tar -czvf '+archive_name\
+' --exclude=\'.*\' '\
+' '.join(includes)
c.local(tar_cmd, echo=True)
#here we temperarily set the owner to ec2 user for ftp access
c.sudo('chown -R '+Config['ec2_usrname']+' '
+Config['git_repo_dist_release_path'].format(''), echo=True)
remote_src_path = Config['git_repo_dist_prod_link']+'/'+Config[serv_path]
archive_file_path = Config[local_path]+'/'+archive_name
info('starting putting:'+archive_file_path+' to:'+remote_src_path)
c.put(archive_file_path, remote_src_path)
info('put the archived src to '+remote_src_path+'/..')
#c.sudo('mv '+remote_src_path+ ' '+remote_src_path+'_bak', echo=True)
#c.sudo('rm -rf '+remote_src_path+'/*', echo=True)
c.run('cd '+remote_src_path
+' && tar -xzvf '+archive_name, echo=True)
c.sudo('rm '+remote_src_path+'/'+archive_name)
#now we set the owner back to www_client
c.sudo('chown -R '+Config['website_client_role']
+':'+Config['website_client_group']
+' '+Config['git_repo_dist_release_path'].format(''), echo=True)
c.local('cd '+Config[local_path]+' && rm '+archive_name, echo=True)
def create_virtual_env(c):
remove_virtual_env(c)
info('Start creating conda env')
c.sudo(Config['conda_install_path'] + '/bin/conda env create'
#+' -n '+Config['env_name']
+ ' -f=' + Config['env_yaml_path'] + ' -p ' +
Config['env_dir_path'])
def bower_pkg_install(c):
info('Start installing bower package')
if c.sudo('test -d ' + Config['bower_path'] + '/bower_components',
warn=True).ok:
c.sudo('rm -rf ' + Config['bower_path'] + '/bower_components')
info('remove existing bower_components')
c.run('cd ' + Config['bower_path'] + ' && bower install', echo=True)
def create_deploy_env(c):
remove_deploy_env(c)
info('Start creating conda env:deploy_py2')
c.sudo(Config['conda_install_path'] + '/bin/conda env create'
#+' -n '+Config['env_name']
+ ' -f=' + Config['deploy_env_yaml_path'] + ' -p ' +
Config['deploy_env_path'])
def insert_testing_data(c):
info('Start inserting QA testing data into db')
info('Make sure the db path is under correct Access previlege')
#here we temperarily set the owner to ec2 user for ftp access
c.sudo('chown -R '+Config['ec2_usrname']+' '
+Config['git_repo_dist_release_path'].format(''), echo=True)
info('step1> drop existing tables')
c.run('source /etc/anaconda/bin/activate '+Config['env_dir_path']
+' && cd '+Config['git_repo_dist_prod_link']
+' && python manage.py dropdb', echo=True)
info('step2> create tables')
c.run('source /etc/anaconda/bin/activate '+Config['env_dir_path']
+' && cd '+Config['git_repo_dist_prod_link']
+' && python manage.py insert_data', echo=True)
info('step3> create tables')
c.run('source /etc/anaconda/bin/activate '+Config['env_dir_path']
+' && cd '+Config['git_repo_dist_prod_link']
+' && python manage.py db_insertSensorData', echo=True)
#now we set the owner back to www_client
c.sudo('chown -R '+Config['website_client_role']
+':'+Config['website_client_group']
+' '+Config['git_repo_dist_release_path'].format(''), echo=True)
def uninstall_bower(c):
info('Start uninstalling bower and npm')
c.sudo('npm uninstall bower')
c.sudo('yum remove nodejs npm',
pty=True,
watchers=[Respond['npm_(un)install_confirm']])
info('uninstallation done')
def update_current_release(c):
info('Start updating current prod release')
#here we temperarily set the owner to ec2 user for git access
c.sudo('chown -R ' + Config['ec2_usrname'] + ' ' +
Config['git_repo_dist_release_path'].format(''),
echo=True)
#later on the auth tool will set the owner back to www_client
c.run('cd ' + Config['git_repo_dist_prod_link'] + ' && git pull',
echo=True)
def restart(c):
info('Restart supervisor fcgi program')
c.run(_source_env_str + ' && supervisorctl' + _use_config_str +
' stop all' + ' && supervisorctl' + _use_config_str + ' start all',
echo=True)
result = c.run(_source_env_str + ' && supervisorctl' + _use_config_str +
' status',
hide='out')
info('fcgi program restarted:\n' + result.stdout.strip())
def git_clone(c):
git_remove_repo(c)
if c.run('test -d ' + Config['git_repo_dist_path'], warn=True).failed:
c.sudo('mkdir -p ' + Config['git_repo_dist_path'])
c.sudo('chown -R ' + Config['ec2_usrname'] + ' ' +
Config['git_repo_dist_path'])
info('Create git repo dir:' + Config['git_repo_dist_path'])
info('clone git repo to:' + Config['git_repo_dist_path'])
c.run('git clone {0} {1}'.format(Config['github_repo_url'],
Config['git_repo_dist_path']))
def chown_of_dist_repo(c):
info('Start chown of related folders')
c.sudo('chown -R ' + Config['website_client_role'] + ':' +
Config['website_client_group'] + ' ' +
Config['git_repo_dist_release_path'].format(''),
echo=True)
#when content is very large, nginx will use buffer for uploading and
#it requires the access role
c.sudo('chown -R ' + Config['website_client_role'] + ':' +
Config['website_client_group'] + ' ' +
Config['nginx_tmp_buffer'].format(''),
echo=True)
def install_bower(c):
uninstall_bower(c)
info('Start installing bower')
info('Step1>install node & npm')
c.sudo('yum install nodejs npm --enablerepo=epel',
pty=True,
watchers=[Respond['npm_(un)install_confirm']])
info('Step2>install bower')
#turn off ssl check or you will get Error: UNABLE_TO_GET_ISSUER_CERT_LOCALLY
c.sudo('npm config set strict-ssl false')
c.sudo('npm install -g bower')
info('bower installation done')
def install_anaconda(c):
uninstall_anaconda(c)
info('step1> Use wget to download Anaconda')
c.sudo('wget https://repo.continuum.io/archive/'\
+Config['conda_version']+' -O '+Config['conda_script_path'])
info('step2> Run script to install anaconda')
c.sudo('bash '+Config['conda_script_path']+' -b -p '\
+Config['conda_install_path'])
#c.run('export PATH="/etc/anaconda/bin:$PATH"')
#c.run('source /etc/anaconda/bin/activate')
info('step3> Anaconda installation finished. Clean up the temp script.')
c.run('rm ' + Config['conda_script_path'])
def start(c):
info('Start launching the Supervisord')
result = c.run(_source_env_str + ' && supervisorctl' + _use_config_str +
' pid',
hide='out')
if result.stdout.strip().isdigit():
info('Supervisord is already running')
restart(c)
else:
c.run(_source_env_str + ' && supervisord' + _use_config_str, echo=True)
result = c.run(_source_env_str + ' && supervisorctl' +
_use_config_str + ' status',
hide='out')
info('Supervisord started:\n' + result.stdout.strip())
def create_role_and_group(c):
info('Start adding role and group')
result = c.run('less /etc/group | grep -n ' +
Config['website_client_group'],
hide='out',
warn=True)
if result.stdout.strip():
info('group already exists:' + result.stdout.strip())
else:
c.sudo('groupadd ' + Config['website_client_group'], echo=True)
result = c.run('less /etc/passwd | grep -n ' +
Config['website_client_role'],
hide='out',
warn=True)
if result.stdout.strip():
info('username already exists:' + result.stdout.strip())
else:
c.sudo('useradd -s /sbin/nologin ' + Config['website_client_role'] +
' -g ' + Config['website_client_group'],
echo=True)
def d_start(c):
'''
supervisord needs to be restarted if there is a change of
the set of programs controlled by it
'''
info('Start launching the Supervisord')
ctl_path = c.run('which supervisorctl').stdout.strip()
d_path = c.run('which supervisord').stdout.strip()
result = c.run(_source_env_str+' && sudo '+ctl_path+' '
+_use_config_str+' pid', hide='out', warn=True)
if result.stdout.strip().isdigit():
info('Supervisord is already running, restarting')
c.sudo('kill -SIGTERM '+result.stdout.strip(), echo=True)
c.run(_source_env_str+' && cd '+Config['git_repo_dist_prod_link']
+' && sudo '+d_path+' '+_use_config_str, echo=True, pty=True)
result = c.run(_source_env_str+' && sudo '+ctl_path+' '
+_use_config_str+' status', hide='out')
info('Supervisord started:\n'+result.stdout.strip())
def update_bower_pkg(c):
info('Start updating bower package')
c.run('cd ' + Config['bower_path'] + ' && bower install', echo=True)
def remove_deploy_env(c):
info('Start removing conda env:deploy_py2')
if c.sudo('test -d ' + Config['deploy_env_path'], warn=True).ok:
c.sudo('rm -rf ' + Config['deploy_env_path'])
info(Config['deploy_env_path'] + ' has been removed')
def upgrade(c):
info('Start upgrade SQLAlchemy DB')
c.run('source /etc/anaconda/bin/activate ' + Config['env_dir_path'] +
' && cd ' + Config['git_repo_dist_path'] +
' && python manage.py db upgrade',
echo=True)
def remove_role_and_group(c):
info('Start removing role and group')
c.sudo('userdel -r ' + Config['website_client_role'] + ' && groupdel ' +
Config['website_client_group'],
echo=True)
def start(c):
info('Copy from ' + _conf_repo_path + ' to ' + _conf_srv_path)
c.sudo('cp ' + _conf_repo_path + ' ' + _conf_srv_path, echo=True)
info('Start nginx service')
c.sudo('nginx', echo=True)
def reload(c):
info('Copy from ' + _conf_repo_path + ' to ' + _conf_srv_path)
c.sudo('cp ' + _conf_repo_path + ' ' + _conf_srv_path, echo=True)
info('reload nginx service')
c.sudo('nginx -s reload', echo=True)
def upgrade(c):
info('Start upgrade SQLAlchemy DB')
info('Make sure the db path is under correct Access previlege')
c.run('source /etc/anaconda/bin/activate '+Config['env_dir_path']
+' && cd '+Config['git_repo_dist_prod_link']
+' && python manage.py db upgrade', echo=True)
def update_deploy_env(c):
info('Start updating conda env:deploy_py2')
c.sudo(Config['conda_install_path'] + '/bin/conda env update'
#+' -n '+Config['env_name'] #the -n will override the -p
+ ' -f=' + Config['deploy_env_yaml_path'] + ' -p ' +
Config['deploy_env_path'])
def uninstall_git(c):
info('Start uninstalling git')
result = c.sudo('yum remove git',
pty=True,
watchers=[Respond['git_(un)install_confirm']])
info('Uninstallation finished, exit code:' + str(result.exited))
def update_virtual_env(c):
info('Start updating conda env')
c.sudo(Config['conda_install_path'] + '/bin/conda env update'
#+' -n '+Config['env_name'] #the -n will override the -p
+ ' -f=' + Config['env_yaml_path'] + ' -p ' +
Config['env_dir_path'])
def update_git_repo(c):
info('Start updating git repo')
c.run('cd ' + Config['git_repo_dist_path'] + ' && git pull', echo=True)
def remove_virtual_env(c):
info('Start removing conda env')
if c.sudo('test -d ' + Config['env_dir_path'], warn=True).ok:
c.sudo('rm -rf ' + Config['env_dir_path'])
info(Config['env_dir_path'] + ' has been removed')
def uninstall_anaconda(c):
info('Start uninstalling the anaconda if already installed')
if c.sudo('test -f ' + Config['conda_script_path'], warn=True).ok:
c.sudo('rm ' + Config['conda_script_path'])
if c.sudo('test -d ' + Config['conda_install_path'], warn=True).ok:
c.sudo('rm -rf ' + Config['conda_install_path'])
def install_git(c):
uninstall_git(c)
info('Start installing git')
#the pty flag enforce the terminal to print
#the prompt line instead of buffering it.
#So that we can do the watcher pattern matching
c.sudo('yum install git',
pty=True,
watchers=[Respond['git_(un)install_confirm']])
result = c.run('git --version', hide='out')
info('git installation Done. Version:' + result.stdout.strip())
info('Start setting up git config')
c.run('git config --global user.name \"' + Config['git_username'] + '\"')
c.run('git config --global user.email \"' + Config['git_useremail'] + '\"')
if c.run('test -f '+Config['git_ssh_key_path'], warn=True).ok\
and c.run('test -f '+Config['git_ssh_key_path']+'.pub', warn=True).ok:
info('Rsa key already exists, skip key generation')
else:
info('Start generating ssh key')
c.run('ssh-keygen -t rsa -C \"' + Config['git_useremail'] + '\"',
pty=True,
watchers=[
Respond['git_ssh_path_confirm'], Respond['git_passphrase']
])
ssh_result = c.run('ssh -T [email protected]', hide=True, warn=True)
info('Key generation done. Try ssh\n{0}, exit code:{1}'.format(
'stdout:' + ssh_result.stdout.strip() + '\nstderr:' +
ssh_result.stderr.strip(), ssh_result.exited))
def git_remove_repo(c):
info('Start removing git repo if exists')
if c.run('test -d ' + Config['git_repo_dist_path'], warn=True).ok:
c.sudo('rm -rf ' + Config['git_repo_dist_path'], echo=True)
