def make_default_values():
if not hasattr(env, 'apitaxi_dir'):
env.apitaxi_dir = lambda now: env.deployment_dir(now) + '/APITaxi-master'
if not hasattr(env, 'uwsgi_config_path'):
env.uwsgi_config_path = lambda now: env.apitaxi_dir(now) + '/uwsgi.ini'
if not hasattr(env, 'uwsgi_file'):
env.uwsgi_file = lambda now: env.apitaxi_dir(now) + '/api_taxi.uwsgi'
if not hasattr(env, 'apitaxi_venv_path'):
env.apitaxi_venv_path = lambda now: env.deployment_dir(now) + '/venvAPITaxi'
if not hasattr(env, 'apitaxi_config_path'):
env.apitaxi_config_path = lambda now: env.apitaxi_dir(now) + '/APITaxi/prod_settings.py'
def deploy_api(commit='master'):
now = int(time.time())
require.files.directory(env.deployment_dir(now))
with cd(env.deployment_dir(now)):
run(u'wget {}'.format(env.apitaxi_archive.format(commit)))
run('unzip {}.zip'.format(commit))
if commit != 'master':
run('mv APITaxi-{} APITaxi-master'.format(commit))
with cd(env.apitaxi_dir(now)):
require.python.virtualenv(env.apitaxi_venv_path(now))
with python.virtualenv(env.apitaxi_venv_path(now)):
python.install_pip(use_sudo=False)
require.python.package('uwsgi')
python.install_requirements('requirements.txt')
put(environ['APITAXI_CONFIG_FILE'], env.apitaxi_config_path(now))
with shell_env(APITAXI_CONFIG_FILE=env.apitaxi_config_path(now)):
for i in range(1, 30):
if service.is_running('supervisor'):
break
time.sleep(1)
run('python manage.py db upgrade')
install_admin_user()
deploy_front(now)
deploy_nginx_api_site(now)
if not service.is_running('nginx'):
service.start('nginx')
clean_directories(now)
stop_old_processes(now)
restart_stats_workers(now)
def stop_old_processes(now):
def stop_process(name, visitor):
l = run('for i in /etc/supervisor/conf.d/{}_*; do echo $i; done'.format(name)).split("\n")
for f in [f.replace('\r', '') for f in l]:
print 'To remove: {}'.format(f)
if str(now) in f:
continue
file_ = f.split('/')[-1]
process = file_[:-len('.conf')]
visitor(process)
files.remove(f, use_sudo=True)
stop_process('uwsgi', lambda p:supervisor.stop_process(p))
def stop_queues(process):
#Request' status is failure after 15 secs in received
#So even if queue is not empty we can shutdown the process
for i in range(1, 17):
res = run('python manage.py active_tasks {}'.format(process))
if res == '':
break
time.sleep(1)
supervisor.stop_process(process)
with cd(env.apitaxi_dir(now)):
with python.virtualenv(env.apitaxi_venv_path(now)),\
shell_env(APITAXI_CONFIG_FILE=env.apitaxi_config_path(now)):
stop_process('send_hail', stop_queues)
def deploy_nginx_api_site(now):
files.upload_template('templates/uwsgi.ini', env.uwsgi_config_path(now),
context={
'config_path': env.apitaxi_config_path(now),
'api_path': env.apitaxi_dir(now),
'venv_path': env.apitaxi_venv_path(now),
'uwsgi_file': env.uwsgi_file(now),
'uwsgi_pid_file': env.uwsgi_pid_file(now),
'uwsgi_log_dir': env.uwsgi_logdir,
'uwsgi_launcher_logdir': env.uwsgi_launcher_logdir,
'socket': env.uwsgi_socket(now),
'processes': env.wsgi_processes,
'threads': env.wsgi_threads
}
)
uwsgi = path.join(env.apitaxi_venv_path(now), 'bin', 'uwsgi')
require.supervisor.process('uwsgi_{}'.format(now),
command='{} --ini {}'.format(uwsgi, env.uwsgi_config_path(now)),
directory=env.apitaxi_venv_path(now),
stdout_logfile = '/var/log/nginx/apitaxi.log',
user='******'
)
test_uwsgi_is_started(now)
require.nginx.site('apitaxi',
template_source='templates/nginx_site.conf',
domain_name=getattr(env.conf_api, 'HOST', 'localhost'),
port=getattr(env.conf_api, 'PORT', 80),
socket=env.uwsgi_socket(now)
)
def make_default_values():
if not hasattr(env, 'apitaxi_dir'):
env.apitaxi_dir = lambda now: env.deployment_dir(now) + '/APITaxi-master'
if not hasattr(env, 'fronttaxi_dir'):
env.fronttaxi_dir = lambda now: env.deployment_dir(now) + '/APITaxi_front-master'
if not hasattr(env, 'uwsgi_api_config_path'):
env.uwsgi_api_config_path = lambda now: env.apitaxi_dir(now) + '/uwsgi_api.ini'
if not hasattr(env, 'uwsgi_front_config_path'):
env.uwsgi_front_config_path = lambda now: env.fronttaxi_dir(now) + '/uwsgi_front.ini'
if not hasattr(env, 'uwsgi_api_file'):
env.uwsgi_api_file = lambda now: env.apitaxi_dir(now) + '/api_taxi.uwsgi'
if not hasattr(env, 'uwsgi_front_file'):
env.uwsgi_front_file = lambda now: env.fronttaxi_dir(now) + '/front_taxi.uwsgi'
if not hasattr(env, 'apitaxi_venv_path'):
env.apitaxi_venv_path = lambda now: env.deployment_dir(now) + '/venvAPITaxi'
if not hasattr(env, 'apitaxi_config_path'):
env.apitaxi_config_path = lambda now: env.apitaxi_dir(now) + '/APITaxi/prod_settings.py'
if not hasattr(env, 'fronttaxi_config_path'):
env.fronttaxi_config_path = lambda now: env.fronttaxi_dir(now) + '/APITaxi_front/prod_settings.py'
if not hasattr(env, 'geotaxi_authentication'):
env.geotaxi_authentication = False
def deploy_api():
now = int(time.time())
require.files.directory(env.deployment_dir(now))
with cd(env.deployment_dir(now)):
run('wget https://github.com/openmaraude/APITaxi/archive/master.zip')
run('unzip master.zip')
with cd(env.apitaxi_dir(now)):
require.python.virtualenv(env.apitaxi_venv_path(now))
with python.virtualenv(env.apitaxi_venv_path(now)):
python.install_pip()
require.python.package('uwsgi')
python.install_requirements('requirements.txt')
put(environ['APITAXI_CONFIG_FILE'], env.apitaxi_config_path(now))
run('python manage.py db upgrade')
deploy_nginx_api_site(now)
if not service.is_running('nginx'):
service.start('nginx')
clean_directories(now)
stop_old_processes(now)
def restart_stats_workers(now=None):
l = run('for i in {}/deployment_*; do echo $i; done'.format(env.deploy_dir)).split("\n")
if not now:
now = int(sorted(l)[-1].split('_')[-1])
celery = path.join(env.apitaxi_venv_path(now), 'bin', 'celery')
commandline = '{} {{command}} --app=celery_worker.celery --workdir={} --pidfile={}'
commandline = commandline.format(celery, env.apitaxi_dir(now),
path.join(env.uwsgi_pid_dir, 'stat_{command}.pid'))
celery_directory = path.join(env.deploy_dir, 'celery')
require.files.directory(celery_directory, owner='www-data', group='adm',
use_sudo=True)
require.supervisor.process('stat_beat',
command=commandline.format(command='beat'),
directory=env.apitaxi_venv_path(now),
environment='APITAXI_CONFIG_FILE="{}"'.format(
env.apitaxi_config_path(now)),
user='******')
require.supervisor.process('stat_worker',
command=commandline.format(command='worker'),
directory=env.apitaxi_venv_path(now),
environment='APITAXI_CONFIG_FILE="{}"'.format(
env.apitaxi_config_path(now)),
user='******')
def deploy_nginx_api_site(now):
files.upload_template('templates/uwsgi.ini', env.uwsgi_api_config_path(now),
context={
'config_path': env.apitaxi_config_path(now),
'api_path': env.apitaxi_dir(now),
'venv_path': env.apitaxi_venv_path(now),
'uwsgi_file': env.uwsgi_api_file(now),
'uwsgi_pid_file': env.uwsgi_api_pid_file(now),
'uwsgi_log_file1': env.uwsgi_logdir + '/api_launcher.log',
'uwsgi_log_file2': env.uwsgi_logdir + '/api_uwsgi.log',
'uwsgi_launcher_logdir': env.uwsgi_launcher_logdir,
'socket': env.uwsgi_socket_api(now),
'processes': env.wsgi_processes,
'threads': env.wsgi_threads,
'now': now
}
)
files.upload_template('templates/uwsgi.ini', env.uwsgi_front_config_path(now),
context={
'config_path': env.fronttaxi_config_path(now),
'api_path': env.fronttaxi_dir(now),
'venv_path': env.apitaxi_venv_path(now),
'uwsgi_file': env.uwsgi_front_file(now),
'uwsgi_pid_file': env.uwsgi_front_pid_file(now),
'uwsgi_log_file1': env.uwsgi_logdir + '/front_launcher.log',
'uwsgi_log_file2': env.uwsgi_logdir + '/front_uwsgi.log',
'socket': env.uwsgi_socket_front(now),
'processes': env.wsgi_processes,
'threads': env.wsgi_threads,
'now': now
}
)
uwsgi = path.join(env.apitaxi_venv_path(now), 'bin', 'uwsgi')
require.supervisor.process('uwsgi_api_{}'.format(now),
command='{} --ini {}'.format(uwsgi, env.uwsgi_api_config_path(now)),
directory=env.apitaxi_venv_path(now),
stdout_logfile = '/var/log/nginx/apitaxi.log',
user='******'
)
require.supervisor.process('uwsgi_front_{}'.format(now),
command='{} --ini {}'.format(uwsgi, env.uwsgi_front_config_path(now)),
directory=env.apitaxi_venv_path(now),
stdout_logfile = '/var/log/nginx/fronttaxi.log',
user='******'
)
test_uwsgi_is_started(now)
celery = path.join(env.apitaxi_venv_path(now), 'bin', 'celery')
worker_name = 'send_hail_{}'.format(now)
command = '{} worker --app=celery_worker.celery -Q {} -n {} --workdir={}'
require.supervisor.process(worker_name,
command=command.format(celery, worker_name, worker_name, env.apitaxi_dir(now)),
directory=env.apitaxi_dir(now),
stdout_logfile='/var/log/celery/send_hail.log',
user='******',
environment='APITAXI_CONFIG_FILE=prod_settings.py'
)
swagger_dir = install_swagger_ui()
zupc_dir = install_zupc_cache()
require.nginx.site('apitaxi',
template_source='templates/nginx_site.conf',
domain_name=getattr(env.conf_api, 'HOST', 'localhost'),
env='NOW={}'.format(now),
port=getattr(env.conf_api, 'PORT', 80),
socket_api=env.uwsgi_socket_api(now),
socket_front=env.uwsgi_socket_front(now),
doc_dir=swagger_dir,
zupc_cache_dir=zupc_dir
)
path_redis = '{}/redis.sh'.format(env.deployment_dir(now))
require.files.template_file(path=path_redis,
template_source='templates/redis.sh',
context={'deployment_dir':env.deployment_dir(now)},
mode='770')
require.supervisor.process('redis',
command=path_redis,
stdout_logfile='/var/log/redis/error.log'
)
