def purge_data_nav(force=False):
"""
purge temp/data.nav.lz4 files
the whole process will be skipped as soon as a single condition is encountered:
- temp data file is more recent than actual data file
- temp data file exists but actual data file is missing
"""
if not force:
print("Checking lz4 temp files purge conditions before proceeding...")
reason = {}
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target))
if exists(plain_target):
if exists(temp_target) and files.getmtime(temp_target) > files.getmtime(plain_target):
reason[instance.name] = "{} is more recent than {}".format(temp_target, plain_target)
elif exists(temp_target):
reason[instance.name] = "{} does not exists".format(plain_target)
if reason:
print(yellow("Error: Can't purge lz4 temp files, reasons:"))
for k, v in reason.iteritems():
print(" {}: {}".format(k, v))
exit(1)
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target))
if exists(temp_target):
files.remove(temp_target)
def app_deploy():
# Pull repository
run('eval `ssh-agent -s` && ssh-add')
with cd("/home/developer/scheduler"):
run("git fetch origin")
run("git reset origin/master --hard")
# Create / update config
if files.is_file('/etc/nginx/sites-enabled/default'):
files.remove('/etc/nginx/sites-enabled/default', use_sudo=True)
update_file('/etc/systemd/system/gunicorn.service',
source='conf/gunicorn.service')
update_file('/etc/nginx/sites-available/www.azacili.com',
source='conf/www.azacili.com')
# Create / update Python environment
with python.virtualenv("/home/developer/.pyenv/versions/azacili/"):
with cd('/home/developer/scheduler'):
run("pip install -r requirements.txt --upgrade --upgrade-strategy eager"
)
run(r"find . -name '*.pyc' -exec rm -rf {} \;")
run("python manage.py collectstatic -c --noinput --settings=core.settings.prod"
)
sudo("systemctl daemon-reload")
sudo("service gunicorn restart")
sudo("service nginx restart")
return True
def upload_sqldump(app):
if app.dbdump:
if re.match('http(s)?://', app.dbdump):
fname = 'dump.sql.gz'
url = app.dbdump
auth = ''
else:
latest = cdstar.get_latest_bitstream(app.dbdump)
fname, url = latest.name, latest.url
auth = '-u"{0}:{1}" '.format(os.environ['CDSTAR_USER_BACKUP'], os.environ['CDSTAR_PWD_BACKUP'])
target = pathlib.PurePosixPath('/tmp') / fname
run('curl -s -o {0} {1} {2}'.format(target, auth, url))
else:
db_name = prompt('Replace with dump of local database:', default=app.name)
sqldump = pathlib.Path(tempfile.mktemp(suffix='.sql.gz', prefix='%s-' % db_name))
target = pathlib.PurePosixPath('/tmp') / sqldump.name
db_user = '******' if PLATFORM == 'windows' else ''
local('pg_dump %s--no-owner --no-acl -Z 9 -f %s %s' % (db_user, sqldump, db_name))
require.file(str(target), source=str(sqldump))
sqldump.unlink()
if app.stack == 'soundcomparisons':
sudo('echo "drop database {0};" | mysql'.format(app.name))
require.mysql.database(app.name, owner=app.name)
sudo('gunzip -c {0} | mysql -u {1} --password={1} -D {1}'.format(target, app.name), user=app.name)
else:
# TODO: assert supervisor.process_status(app.name) != 'RUNNING'
if postgres.database_exists(app.name):
require_postgres(app, drop=True)
sudo('gunzip -c %s | psql -d %s' % (target, app.name), user=app.name)
sudo('vacuumdb -zf %s' % app.name, user='******')
files.remove(str(target))
def uncache(app):
with settings(warn_only=True):
files.remove(str(app.varnish_site), use_sudo=True)
_update_varnish_sites(app.varnish_site.parent)
_update_nginx(app, with_varnish=False)
def purge_data_nav(force=False):
"""
purge temp/data.nav.lz4 files
the whole process will be skipped as soon as a single condition is encountered:
- temp data file is more recent than actual data file
- temp data file exists but actual data file is missing
"""
if not force:
print("Checking lz4 temp files purge conditions before proceeding...")
reason = {}
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp',
os.path.basename(plain_target))
if exists(plain_target):
if exists(temp_target) and files.getmtime(
temp_target) > files.getmtime(plain_target):
reason[instance.name] = "{} is more recent than {}".format(
temp_target, plain_target)
elif exists(temp_target):
reason[instance.name] = "{} does not exists".format(
plain_target)
if reason:
print(yellow("Error: Can't purge lz4 temp files, reasons:"))
for k, v in reason.iteritems():
print(" {}: {}".format(k, v))
exit(1)
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp',
os.path.basename(plain_target))
if exists(temp_target):
files.remove(temp_target)
def test_uwsgi_is_started(now):
testing_file = '/tmp/test_uwsgi.py'
if files.is_file(testing_file):
files.remove(testing_file)
put('files/test_uwsgi.py', testing_file)
output = run('python {} {} {}'.format(testing_file, env.uwsgi_socket(now), env.server_name))
assert '"message"' in output
def stop_process(name, visitor):
l = run('for i in /etc/supervisor/conf.d/{}_*; do echo $i; done'.format(name)).split("\n")
for f in [f.replace('\r', '') for f in l]:
print 'To remove: {}'.format(f)
if str(now) in f:
continue
file_ = f.split('/')[-1]
process = file_[:-len('.conf')]
visitor(process)
files.remove(f, use_sudo=True)
def stop_old_processes(now):
l = run('for i in /etc/supervisor/conf.d/uwsgi_*; do echo $i; done').split("\n")
for f in l:
if f == env.deployment_dir:
continue
m = re.search(r'([^/]*).conf$', f)
if not m:
continue
process = m.group(1)
supervisor.stop_process(process)
files.remove(f, use_sudo=True)
def remove_old_install():
"""
remove old packages from navitia
"""
if deb.is_installed('navitia-stat-persistor'):
execute(stop_stat_persistor)
deb.uninstall('navitia-stat-persistor', purge=True)
files.remove('/srv/stat_persistor/alembic.ini')
files.remove('/srv/stat_persistor/stat_persistor.json')
else:
print yellow('No old installation detected')
def clean_directories(now):
l = run('for i in {}/deployment_*; do echo $i; done'.format(env.deploy_dir)).split("\n")
for d in [d.replace('\r', '') for d in l]:
if not files.is_dir(d):
continue
if d == env.deployment_dir(now):
continue
files.remove(d, recursive=True)
l = run('for i in {}/apitaxi_*; do echo $i; done'.format(env.uwsgi_socket_dir)).split("\n")
for f in [f.replace('\r', '') for f in l]:
if f == env.uwsgi_socket_api(now):
continue
files.remove(f, use_sudo=True)
def clean_directories(now):
l = run('for i in {}/deployment_*; do echo $i; done'.format(env.uwsgi_dir)).split("\n")
for d in l:
if not files.is_dir(d):
continue
if d == env.deployment_dir(now):
continue
files.remove(d, recursive=True)
l = run('for i in {}/apitaxi_*; do echo $i; done'.format(env.uwsgi_socket_dir)).split("\n")
for f in l:
if not files.is_file(f):
continue
if f == env.uwsgi_socket(now):
continue
files.remove(f)
def test_uwsgi_is_started(now):
for i in range(1, 30):
status = supervisor.process_status('uwsgi_{}'.format(now))
if status == 'RUNNING':
break
time.sleep(1)
testing_file = '/tmp/test_uwsgi.py'
if files.is_file(testing_file):
files.remove(testing_file)
put('files/test_uwsgi.py', '/tmp/')
require.python.package('six', use_sudo=True)
output = run('python {} {} {} aa'.format(testing_file, env.uwsgi_socket_api(now),
'{}/ads/'.format(env.conf_api.SERVER_NAME)))
assert '"message"' in output
from test_api import test_api
test_api(testing_file, env.uwsgi_socket_api(now), env.conf_api.SERVER_NAME)
def uninstall(app): # pragma: no cover
"""uninstall the app"""
for path in (app.nginx_location, app.nginx_site, app.venv_dir):
if exists(str(path)):
if path == app.nginx_site:
nginx.disable(app.nginx_site.name)
files.remove(str(path), recursive=True, use_sudo=True)
stop.execute_inner(app)
if user.exists(app.name):
if app.stack != 'soundcomparisons':
sudo('dropdb --if-exists %s' % app.name, user='******')
else:
sudo('echo "drop database {0};" | mysql'.format(app.name))
sudo('userdel -rf %s' % app.name)
if exists(str(app.supervisor)):
files.remove(str(app.supervisor), recursive=True, use_sudo=True)
supervisor.update_config()
service.reload('nginx')
systemd.uninstall(app, pathlib.Path(os.getcwd()) / 'systemd')
def test_remove_recursive(mock_run):
from fabtools.files import remove
remove('/tmp/src', recursive=True)
mock_run.assert_called_with('/bin/rm -r /tmp/src')
def teardown():
with settings(warn_only=True):
files.remove(CIDADEILUMINADA_WORK_PATH, use_sudo=True, recursive=True)
def test_remove(mock_run):
from fabtools.files import remove
remove('/tmp/src')
mock_run.assert_called_with('/bin/rm /tmp/src')
def test_remove_recursive(mock_run):
from fabtools.files import remove
remove('/tmp/src', recursive=True)
mock_run.assert_called_with('/bin/rm -r /tmp/src')
def test_remove(mock_run):
from fabtools.files import remove
remove('/tmp/src')
mock_run.assert_called_with('/bin/rm /tmp/src')
def teardown():
with settings(warn_only=True):
files.remove(CIDADEILUMINADA_WORK_PATH, use_sudo=True, recursive=True)
def test_remove_force(mock_run):
from fabtools.files import remove
remove('/tmp/src.txt', force=True)
mock_run.assert_called_with('/bin/rm -f /tmp/src.txt')
def teardown():
with settings(warn_only=True):
files.remove(HOTSITE_WORK_PATH, use_sudo=True, recursive=True)