def install_python_package_remotely(name, version, url, **kwargs):
tries = 0
max_tries = 3
server = get_current_veil_server()
while True:
tries += 1
try:
if url:
shell_execute(
'pip install -i {} --trusted-host {} --timeout 30 {}'.
format(server.pypi_index_url, server.pypi_index_host, url),
capture=True,
debug=True,
**kwargs)
else:
if name == 'tornado':
shell_execute('pip install --timeout 30 {}=={}'.format(
name, version),
capture=True,
debug=True,
**kwargs)
else:
shell_execute(
'pip install -i {} --trusted-host {} --timeout 30 {}=={}'
.format(server.pypi_index_url, server.pypi_index_host,
name, version),
capture=True,
debug=True,
**kwargs)
except Exception:
if tries >= max_tries:
raise
else:
break
def download_python_package(name, version=None, url=None, **kwargs):
tries = 0
max_tries = 3
server = get_current_veil_server()
name_term = '{}{}'.format(name, '=={}'.format(version) if version else '')
while True:
tries += 1
try:
if url:
shell_execute(
'pip download -i {} --trusted-host {} --timeout 30 -d {} {}'.format(server.pypi_index_url, server.pypi_index_host, PYPI_ARCHIVE_DIR, url),
capture=True, debug=True, **kwargs)
else:
if name == 'tornado':
shell_execute('pip download --timeout 30 -d {} {name_term}'.format(PYPI_ARCHIVE_DIR, name_term=name_term), capture=True, debug=True,
**kwargs)
else:
shell_execute('pip download -i {} --trusted-host {} --timeout 30 -d {} {name_term}'.format(server.pypi_index_url, server.pypi_index_host,
PYPI_ARCHIVE_DIR, name_term=name_term),
capture=True, debug=True, **kwargs)
except Exception:
if tries >= max_tries:
raise
else:
break
downloaded_version = get_downloaded_python_package_version(name, version)
assert not version or version == downloaded_version, \
'the downloaded version of python package {} is {}, different from the specific version {}'.format(name, downloaded_version, version)
return downloaded_version
def sync_to_backup_mirror(source_path, remote_path, base_path=None):
server = get_current_veil_server()
if not server.is_guard:
raise AssertionError('only guard but not {} should be able to sync to backup mirror'.format(server.fullname))
backup_mirror = server.backup_mirror
backup_mirror_path = VEIL_BACKUP_MIRROR_ROOT / VEIL_ENV.name / server.host_base_name / remote_path
ssh_option = 'ssh -i {} -p {} -T -x -o Compression=no -o StrictHostKeyChecking=no'.format(SSH_KEY_PATH, backup_mirror.ssh_port)
shell_execute('rsync -avzhPH {} -e "{}" --rsync-path="mkdir -p {} && rsync" --numeric-ids --delete --bwlimit={} {} {}@{}:{}/'.format(
'-R' if base_path else '', ssh_option, backup_mirror_path, backup_mirror.bandwidth_limit, source_path, backup_mirror.ssh_user, backup_mirror.host_ip,
backup_mirror_path), debug=True, cwd=base_path)
def get_installed_package_remote_latest_version(name):
global outdated_package_name2latest_version
if outdated_package_name2latest_version is None:
outdated_package_name2latest_version = {}
server = get_current_veil_server()
if name == 'tornado':
lines = shell_execute('pip list -l -o | grep Latest:', capture=True, debug=True).splitlines()
else:
lines = shell_execute('pip list -i {} --trusted-host {} -l -o | grep Latest:'.format(server.pypi_index_url, server.pypi_index_host), capture=True,
debug=True).splitlines()
for line in lines:
match = RE_OUTDATED_PACKAGE.match(line)
outdated_package_name2latest_version[match.group(1)] = match.group(2)
return outdated_package_name2latest_version.get(name)
def patch():
shell_execute("veil migrate")
threads = []
for program_name, program in get_current_veil_server().programs.items():
if program.get("patchable"):
program_name = "{}:{}".format(program["group"], program_name) if program.get("group") else program_name
thread = threading.Thread(
target=functools.partial(
shell_execute, "veil server supervisor restart-program {}".format(program_name)
)
)
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def bring_up_barman_recover(crontab_expression, purpose):
@run_every(crontab_expression)
def work():
ssh_command = 'ssh -p {} -i /etc/ssh/id_ed25519-barman {}@{}'.format(backup_mirror.ssh_port, backup_mirror.ssh_user, backup_mirror.host_ip)
path = VEIL_BACKUP_MIRROR_ROOT / VEIL_ENV.name / 'latest-database-recover' / purpose
assert path.startswith('~/')
try:
shell_execute('barman recover --remote-ssh-command "{}" {} latest {}'.format(ssh_command, purpose, path[2:]), capture=True)
except:
pass
server = get_current_veil_server()
if not server.is_barman:
raise AssertionError('only barman but not {} should be able to run barman recover'.format(server.fullname))
backup_mirror = server.backup_mirror
work()
def patch():
shell_execute('veil migrate')
threads = []
current_veil_server = get_current_veil_server()
server_specified_resources = current_veil_server.get('resources', [])
for resource in server_specified_resources:
if 'frontend_static_resource' in resource[0]:
install_resource(resource)
for program_name, program in current_veil_server.programs.items():
if program.get('patchable'):
program_name = '{}:{}'.format(program['group'], program_name) if program.get('group') else program_name
thread = threading.Thread(target=functools.partial(shell_execute, 'sudo -E veil server supervisor restart-program {}'.format(program_name)))
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def download_python_package(name, version=None, url=None, **kwargs):
tries = 0
max_tries = 3
server = get_current_veil_server()
name_term = '{}{}'.format(name, '=={}'.format(version) if version else '')
while True:
tries += 1
try:
if url:
shell_execute(
'pip download -i {} --trusted-host {} --timeout 30 -d {} {}'
.format(server.pypi_index_url, server.pypi_index_host,
PYPI_ARCHIVE_DIR, url),
capture=True,
debug=True,
**kwargs)
else:
if name == 'tornado':
shell_execute(
'pip download --timeout 30 -d {} {name_term}'.format(
PYPI_ARCHIVE_DIR, name_term=name_term),
capture=True,
debug=True,
**kwargs)
else:
shell_execute(
'pip download -i {} --trusted-host {} --timeout 30 -d {} {name_term}'
.format(server.pypi_index_url,
server.pypi_index_host,
PYPI_ARCHIVE_DIR,
name_term=name_term),
capture=True,
debug=True,
**kwargs)
except Exception:
if tries >= max_tries:
raise
else:
break
downloaded_version = get_downloaded_python_package_version(name, version)
assert not version or version == downloaded_version, \
'the downloaded version of python package {} is {}, different from the specific version {}'.format(name, downloaded_version, version)
return downloaded_version
def get_installed_package_remote_latest_version(name):
global outdated_package_name2latest_version
if outdated_package_name2latest_version is None:
outdated_package_name2latest_version = {}
server = get_current_veil_server()
if name == 'tornado':
lines = shell_execute('pip list -l -o | grep Latest:',
capture=True,
debug=True).splitlines()
else:
lines = shell_execute(
'pip list -i {} --trusted-host {} -l -o | grep Latest:'.format(
server.pypi_index_url, server.pypi_index_host),
capture=True,
debug=True).splitlines()
for line in lines:
match = RE_OUTDATED_PACKAGE.match(line)
outdated_package_name2latest_version[match.group(1)] = match.group(
2)
return outdated_package_name2latest_version.get(name)
def install_python_package_remotely(name, version, url, **kwargs):
tries = 0
max_tries = 3
server = get_current_veil_server()
while True:
tries += 1
try:
if url:
shell_execute('pip install -i {} --trusted-host {} --timeout 30 {}'.format(server.pypi_index_url, server.pypi_index_host, url), capture=True,
debug=True, **kwargs)
else:
if name == 'tornado':
shell_execute('pip install --timeout 30 {}=={}'.format(name, version), capture=True, debug=True, **kwargs)
else:
shell_execute(
'pip install -i {} --trusted-host {} --timeout 30 {}=={}'.format(server.pypi_index_url, server.pypi_index_host, name, version),
capture=True, debug=True, **kwargs)
except Exception:
if tries >= max_tries:
raise
else:
break
def install_python_package_remotely(name, version, url, **kwargs):
tries = 0
max_tries = 3
server = get_current_veil_server()
pip_index_args = '-i {}'.format(server.pypi_index_url) if server.pypi_index_url else ''
while True:
tries += 1
try:
if url:
shell_execute('pip install {} --timeout 30 {}'.format(pip_index_args, url), capture=True, debug=True,
**kwargs)
else:
if name == 'ibm-db':
shell_execute('pip install --timeout 180 {} {}=={}'.format(pip_index_args, name, version),
capture=True, debug=True, **kwargs)
else:
shell_execute('pip install {} --timeout 30 {}=={}'.format(pip_index_args, name, version),
capture=True, debug=True, **kwargs)
except Exception:
if tries >= max_tries:
raise
else:
break
def on_supervisor_to_be_down():
LOGGER.info('try to rewrite redis aof before supervisor shutdown')
requested = _rewrite_redis_aof(get_current_veil_server())
if requested:
time.sleep(10) # sleep enough time for redis to complete aof rewrite