Exemplo n.º 1
0
def install_python_package_remotely(name, version, url, **kwargs):
    tries = 0
    max_tries = 3
    server = get_current_veil_server()
    while True:
        tries += 1
        try:
            if url:
                shell_execute(
                    'pip install -i {} --trusted-host {} --timeout 30 {}'.
                    format(server.pypi_index_url, server.pypi_index_host, url),
                    capture=True,
                    debug=True,
                    **kwargs)
            else:
                if name == 'tornado':
                    shell_execute('pip install --timeout 30 {}=={}'.format(
                        name, version),
                                  capture=True,
                                  debug=True,
                                  **kwargs)
                else:
                    shell_execute(
                        'pip install -i {} --trusted-host {} --timeout 30 {}=={}'
                        .format(server.pypi_index_url, server.pypi_index_host,
                                name, version),
                        capture=True,
                        debug=True,
                        **kwargs)
        except Exception:
            if tries >= max_tries:
                raise
        else:
            break
Exemplo n.º 2
0
def download_python_package(name, version=None, url=None, **kwargs):
    tries = 0
    max_tries = 3
    server = get_current_veil_server()
    name_term = '{}{}'.format(name, '=={}'.format(version) if version else '')
    while True:
        tries += 1
        try:
            if url:
                shell_execute(
                    'pip download -i {} --trusted-host {} --timeout 30 -d {} {}'.format(server.pypi_index_url, server.pypi_index_host, PYPI_ARCHIVE_DIR, url),
                    capture=True, debug=True, **kwargs)
            else:
                if name == 'tornado':
                    shell_execute('pip download --timeout 30 -d {} {name_term}'.format(PYPI_ARCHIVE_DIR, name_term=name_term), capture=True, debug=True,
                                  **kwargs)
                else:
                    shell_execute('pip download -i {} --trusted-host {} --timeout 30 -d {} {name_term}'.format(server.pypi_index_url, server.pypi_index_host,
                                                                                                               PYPI_ARCHIVE_DIR, name_term=name_term),
                                  capture=True, debug=True, **kwargs)
        except Exception:
            if tries >= max_tries:
                raise
        else:
            break
    downloaded_version = get_downloaded_python_package_version(name, version)
    assert not version or version == downloaded_version, \
        'the downloaded version of python package {} is {}, different from the specific version {}'.format(name, downloaded_version, version)
    return downloaded_version
Exemplo n.º 3
0
def sync_to_backup_mirror(source_path, remote_path, base_path=None):
    server = get_current_veil_server()
    if not server.is_guard:
        raise AssertionError('only guard but not {} should be able to sync to backup mirror'.format(server.fullname))

    backup_mirror = server.backup_mirror
    backup_mirror_path = VEIL_BACKUP_MIRROR_ROOT / VEIL_ENV.name / server.host_base_name / remote_path
    ssh_option = 'ssh -i {} -p {} -T -x -o Compression=no -o StrictHostKeyChecking=no'.format(SSH_KEY_PATH, backup_mirror.ssh_port)
    shell_execute('rsync -avzhPH {} -e "{}" --rsync-path="mkdir -p {} && rsync" --numeric-ids --delete --bwlimit={} {} {}@{}:{}/'.format(
        '-R' if base_path else '', ssh_option, backup_mirror_path, backup_mirror.bandwidth_limit, source_path, backup_mirror.ssh_user, backup_mirror.host_ip,
        backup_mirror_path), debug=True, cwd=base_path)
Exemplo n.º 4
0
def get_installed_package_remote_latest_version(name):
    global outdated_package_name2latest_version
    if outdated_package_name2latest_version is None:
        outdated_package_name2latest_version = {}
        server = get_current_veil_server()
        if name == 'tornado':
            lines = shell_execute('pip list -l -o | grep Latest:', capture=True, debug=True).splitlines()
        else:
            lines = shell_execute('pip list -i {} --trusted-host {} -l -o | grep Latest:'.format(server.pypi_index_url, server.pypi_index_host), capture=True,
                                  debug=True).splitlines()
        for line in lines:
            match = RE_OUTDATED_PACKAGE.match(line)
            outdated_package_name2latest_version[match.group(1)] = match.group(2)
    return outdated_package_name2latest_version.get(name)
Exemplo n.º 5
0
def patch():
    shell_execute("veil migrate")
    threads = []
    for program_name, program in get_current_veil_server().programs.items():
        if program.get("patchable"):
            program_name = "{}:{}".format(program["group"], program_name) if program.get("group") else program_name
            thread = threading.Thread(
                target=functools.partial(
                    shell_execute, "veil server supervisor restart-program {}".format(program_name)
                )
            )
            threads.append(thread)
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()
Exemplo n.º 6
0
def bring_up_barman_recover(crontab_expression, purpose):
    @run_every(crontab_expression)
    def work():
        ssh_command = 'ssh -p {} -i /etc/ssh/id_ed25519-barman {}@{}'.format(backup_mirror.ssh_port, backup_mirror.ssh_user, backup_mirror.host_ip)
        path = VEIL_BACKUP_MIRROR_ROOT / VEIL_ENV.name / 'latest-database-recover' / purpose
        assert path.startswith('~/')
        try:
            shell_execute('barman recover --remote-ssh-command "{}" {} latest {}'.format(ssh_command, purpose, path[2:]), capture=True)
        except:
            pass

    server = get_current_veil_server()
    if not server.is_barman:
        raise AssertionError('only barman but not {} should be able to run barman recover'.format(server.fullname))
    backup_mirror = server.backup_mirror

    work()
Exemplo n.º 7
0
def patch():
    shell_execute('veil migrate')
    threads = []
    current_veil_server = get_current_veil_server()
    server_specified_resources = current_veil_server.get('resources', [])
    for resource in server_specified_resources:
        if 'frontend_static_resource' in resource[0]:
            install_resource(resource)
    for program_name, program in current_veil_server.programs.items():
        if program.get('patchable'):
            program_name = '{}:{}'.format(program['group'], program_name) if program.get('group') else program_name
            thread = threading.Thread(target=functools.partial(shell_execute, 'sudo -E veil server supervisor restart-program {}'.format(program_name)))
            threads.append(thread)
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()
Exemplo n.º 8
0
def download_python_package(name, version=None, url=None, **kwargs):
    tries = 0
    max_tries = 3
    server = get_current_veil_server()
    name_term = '{}{}'.format(name, '=={}'.format(version) if version else '')
    while True:
        tries += 1
        try:
            if url:
                shell_execute(
                    'pip download -i {} --trusted-host {} --timeout 30 -d {} {}'
                    .format(server.pypi_index_url, server.pypi_index_host,
                            PYPI_ARCHIVE_DIR, url),
                    capture=True,
                    debug=True,
                    **kwargs)
            else:
                if name == 'tornado':
                    shell_execute(
                        'pip download --timeout 30 -d {} {name_term}'.format(
                            PYPI_ARCHIVE_DIR, name_term=name_term),
                        capture=True,
                        debug=True,
                        **kwargs)
                else:
                    shell_execute(
                        'pip download -i {} --trusted-host {} --timeout 30 -d {} {name_term}'
                        .format(server.pypi_index_url,
                                server.pypi_index_host,
                                PYPI_ARCHIVE_DIR,
                                name_term=name_term),
                        capture=True,
                        debug=True,
                        **kwargs)
        except Exception:
            if tries >= max_tries:
                raise
        else:
            break
    downloaded_version = get_downloaded_python_package_version(name, version)
    assert not version or version == downloaded_version, \
        'the downloaded version of python package {} is {}, different from the specific version {}'.format(name, downloaded_version, version)
    return downloaded_version
Exemplo n.º 9
0
def get_installed_package_remote_latest_version(name):
    global outdated_package_name2latest_version
    if outdated_package_name2latest_version is None:
        outdated_package_name2latest_version = {}
        server = get_current_veil_server()
        if name == 'tornado':
            lines = shell_execute('pip list -l -o | grep Latest:',
                                  capture=True,
                                  debug=True).splitlines()
        else:
            lines = shell_execute(
                'pip list -i {} --trusted-host {} -l -o | grep Latest:'.format(
                    server.pypi_index_url, server.pypi_index_host),
                capture=True,
                debug=True).splitlines()
        for line in lines:
            match = RE_OUTDATED_PACKAGE.match(line)
            outdated_package_name2latest_version[match.group(1)] = match.group(
                2)
    return outdated_package_name2latest_version.get(name)
Exemplo n.º 10
0
def install_python_package_remotely(name, version, url, **kwargs):
    tries = 0
    max_tries = 3
    server = get_current_veil_server()
    while True:
        tries += 1
        try:
            if url:
                shell_execute('pip install -i {} --trusted-host {} --timeout 30 {}'.format(server.pypi_index_url, server.pypi_index_host, url), capture=True,
                              debug=True, **kwargs)
            else:
                if name == 'tornado':
                    shell_execute('pip install --timeout 30 {}=={}'.format(name, version), capture=True, debug=True, **kwargs)
                else:
                    shell_execute(
                        'pip install -i {} --trusted-host {} --timeout 30 {}=={}'.format(server.pypi_index_url, server.pypi_index_host, name, version),
                        capture=True, debug=True, **kwargs)
        except Exception:
            if tries >= max_tries:
                raise
        else:
            break
Exemplo n.º 11
0
def install_python_package_remotely(name, version, url, **kwargs):
    tries = 0
    max_tries = 3
    server = get_current_veil_server()
    pip_index_args = '-i {}'.format(server.pypi_index_url) if server.pypi_index_url else ''
    while True:
        tries += 1
        try:
            if url:
                shell_execute('pip install {} --timeout 30 {}'.format(pip_index_args, url), capture=True, debug=True,
                              **kwargs)
            else:
                if name == 'ibm-db':
                    shell_execute('pip install --timeout 180 {} {}=={}'.format(pip_index_args, name, version),
                                  capture=True, debug=True, **kwargs)
                else:
                    shell_execute('pip install {} --timeout 30 {}=={}'.format(pip_index_args, name, version),
                                  capture=True, debug=True, **kwargs)
        except Exception:
            if tries >= max_tries:
                raise
        else:
            break
Exemplo n.º 12
0
def on_supervisor_to_be_down():
    LOGGER.info('try to rewrite redis aof before supervisor shutdown')
    requested = _rewrite_redis_aof(get_current_veil_server())
    if requested:
        time.sleep(10)  # sleep enough time for redis to complete aof rewrite