Esempio n. 1
0
def check_call(conn, logger, args, *a, **kw):
    """
    Wraps ``subprocess.check_call`` for a remote call via ``pushy``
    doing all the capturing and logging nicely upon failure/success

    The mangling of the traceback when an exception ocurrs, is because the
    caller gets eating up by not being executed in the actual function of
    a given module (e.g. ``centos/install.py``) but rather here, where the
    stack trace is no longer relevant.

    :param args: The args to be passed onto ``check_call``
    """
    command = ' '.join(args)
    patch = kw.pop('patch', True)  # Always patch unless explicitly told to
    mangle = kw.pop('mangle_exc', False)  # Default to not mangle exceptions
    stop_on_error = kw.pop('stop_on_error', True)  # Halt on remote exceptions
    logger.info('Running command: %s' % command)
    kw.setdefault(
        'env',
        {
            'PATH':
            '/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin'
        }
    )

    def remote_call(args, *a, **kw):
        import subprocess
        return subprocess.check_call(
            args,
            *a,
            **kw
        )

    with context.remote(conn, logger, remote_call, mangle_exc=mangle, patch=patch) as call:
        try:
            return call(args, *a, **kw)
        except Exception as err:
            import inspect
            stack = inspect.getframeinfo(inspect.currentframe().f_back)
            if hasattr(err, 'remote_traceback'):
                logger.error('Traceback (most recent call last):')
                logger.error('  File "%s", line %s, in %s' % (
                    stack[0],
                    stack[1],
                    stack[2])
                )
                err.remote_traceback.pop(0)
                for line in err.remote_traceback:
                    if line:
                        logger.error(line)
                if stop_on_error:
                    raise RuntimeError(
                        'Failed to execute command: %s' % ' '.join(args)
                    )
            else:
                if stop_on_error:
                    raise err
Esempio n. 2
0
def check_call(conn, logger, args, *a, **kw):
    """
    Wraps ``subprocess.check_call`` for a remote call via ``pushy``
    doing all the capturing and logging nicely upon failure/success

    The mangling of the traceback when an exception ocurrs, is because the
    caller gets eating up by not being executed in the actual function of
    a given module (e.g. ``centos/install.py``) but rather here, where the
    stack trace is no longer relevant.

    :param args: The args to be passed onto ``check_call``
    """
    command = ' '.join(args)
    patch = kw.pop('patch', True)  # Always patch unless explicitly told to
    mangle = kw.pop('mangle_exc', False)  # Default to not mangle exceptions
    stop_on_error = kw.pop('stop_on_error', True)  # Halt on remote exceptions
    logger.info('Running command: %s' % command)
    kw.setdefault('env', {
        'PATH':
        '/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin'
    })

    def remote_call(args, *a, **kw):
        import subprocess
        return subprocess.check_call(args, *a, **kw)

    with context.remote(conn,
                        logger,
                        remote_call,
                        mangle_exc=mangle,
                        patch=patch) as call:
        try:
            return call(args, *a, **kw)
        except Exception as err:
            import inspect
            stack = inspect.getframeinfo(inspect.currentframe().f_back)
            if hasattr(err, 'remote_traceback'):
                logger.error('Traceback (most recent call last):')
                logger.error('  File "%s", line %s, in %s' %
                             (stack[0], stack[1], stack[2]))
                err.remote_traceback.pop(0)
                for line in err.remote_traceback:
                    if line:
                        logger.error(line)
                if stop_on_error:
                    raise RuntimeError('Failed to execute command: %s' %
                                       ' '.join(args))
            else:
                if stop_on_error:
                    raise err
Esempio n. 3
0
def Popen(conn, logger, args, *a, **kw):
    """
    Wraps ``subprocess.Popen`` for a remote call via ``pushy``
    doing all the capturing and logging nicely upon failure/success

    The mangling of the traceback when an exception ocurrs, is because the
    caller gets eating up by not being executed in the actual function of
    a given module (e.g. ``centos/install.py``) but rather here, where the
    stack trace is no longer relevant.

    :param args: The args to be passed onto ``Popen``
    """
    command = ' '.join(args)
    patch = kw.pop('patch', True)  # Always patch unless explicitly told to
    logger.info('Running command: %s' % command)

    def remote_call(args, *a, **kw):
        import subprocess
        process = subprocess.Popen(args,
                                   *a,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   **kw)
        stdout, stderr = process.communicate()
        return stdout, stderr, process.wait()

    with context.remote(conn,
                        logger,
                        remote_call,
                        mangle_exc=False,
                        patch=patch) as call:
        try:
            return call(args, *a, **kw)
        except Exception as err:
            import inspect
            stack = inspect.getframeinfo(inspect.currentframe().f_back)
            if hasattr(err, 'remote_traceback'):
                logger.error('Traceback (most recent call last):')
                logger.error('  File "%s", line %s, in %s' %
                             (stack[0], stack[1], stack[2]))
                err.remote_traceback.pop(0)
                for line in err.remote_traceback:
                    if line:
                        logger.error(line)
                raise RuntimeError('Failed to execute command: %s' %
                                   ' '.join(args))
            else:
                raise err
Esempio n. 4
0
def Popen(conn, logger, args, *a, **kw):
    """
    Wraps ``subprocess.Popen`` for a remote call via ``pushy``
    doing all the capturing and logging nicely upon failure/success

    The mangling of the traceback when an exception ocurrs, is because the
    caller gets eating up by not being executed in the actual function of
    a given module (e.g. ``centos/install.py``) but rather here, where the
    stack trace is no longer relevant.

    :param args: The args to be passed onto ``Popen``
    """
    command = ' '.join(args)
    patch = kw.pop('patch', True)  # Always patch unless explicitly told to
    logger.info('Running command: %s' % command)

    def remote_call(args, *a, **kw):
        import subprocess
        process = subprocess.Popen(
            args,
            *a,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            **kw
        )
        stdout, stderr = process.communicate()
        return stdout, stderr, process.wait()

    with context.remote(conn, logger, remote_call, mangle_exc=False, patch=patch) as call:
        try:
            return call(args, *a, **kw)
        except Exception as err:
            import inspect
            stack = inspect.getframeinfo(inspect.currentframe().f_back)
            if hasattr(err, 'remote_traceback'):
                logger.error('Traceback (most recent call last):')
                logger.error('  File "%s", line %s, in %s' % (
                    stack[0],
                    stack[1],
                    stack[2])
                )
                err.remote_traceback.pop(0)
                for line in err.remote_traceback:
                    if line:
                        logger.error(line)
                raise RuntimeError('Failed to execute command: %s' % ' '.join(args))
            else:
                raise err
Esempio n. 5
0
def install(distro, logger, version_kind, version):
    codename = distro.codename
    machine = distro.sudo_conn.modules.platform.machine()

    if version_kind in ['stable', 'testing']:
        key = 'release'
    else:
        key = 'autobuild'

    # Make sure ca-certificates is installed
    check_call(distro.sudo_conn, logger, [
        'env',
        'DEBIAN_FRONTEND=noninteractive',
        'apt-get',
        '-q',
        'install',
        '--assume-yes',
        'ca-certificates',
    ])

    check_call(
        distro.sudo_conn,
        logger,
        [
            'wget -q -O- \'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc\' | apt-key add -'
            .format(key=key)
        ],
        shell=True,
    )

    if version_kind == 'stable':
        url = 'http://ceph.com/debian-{version}/'.format(version=version, )
    elif version_kind == 'testing':
        url = 'http://ceph.com/debian-testing/'
    elif version_kind == 'dev':
        url = 'http://gitbuilder.ceph.com/ceph-deb-{codename}-{machine}-basic/ref/{version}'.format(
            codename=codename,
            machine=machine,
            version=version,
        )
    else:
        raise RuntimeError('Unknown version kind: %r' % version_kind)

    def write_sources_list(url, codename):
        """add ceph deb repo to sources.list"""
        with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
            f.write('deb {url} {codename} main\n'.format(
                url=url,
                codename=codename,
            ))

    with remote(distro.sudo_conn, logger, write_sources_list) as remote_func:
        remote_func(url, codename)

    check_call(
        distro.sudo_conn,
        logger,
        ['apt-get', '-q', 'update'],
    )

    # TODO this does not downgrade -- should it?
    check_call(
        distro.sudo_conn,
        logger,
        [
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'DEBIAN_PRIORITY=critical',
            'apt-get',
            '-q',
            '-o',
            'Dpkg::Options::=--force-confnew',
            'install',
            '--no-install-recommends',
            '--assume-yes',
            '--',
            'ceph',
            'ceph-mds',
            'ceph-common',
            'ceph-fs-common',
            # ceph only recommends gdisk, make sure we actually have
            # it; only really needed for osds, but minimal collateral
            'gdisk',
        ],
    )

    # Check the ceph version
    common.ceph_version(distro.sudo_conn, logger)
Esempio n. 6
0
def mon_create(distro, logger, args, monitor_keyring, hostname):
    logger.debug('remote hostname: %s' % hostname)
    path = paths.mon.path(args.cluster, hostname)
    done_path = paths.mon.done(args.cluster, hostname)
    init_path = paths.mon.init(args.cluster, hostname, distro.init)

    configuration = conf.load(args)
    conf_data = StringIO()
    configuration.write(conf_data)

    with remote(distro.sudo_conn, logger, conf.write_conf) as remote_func:
        remote_func(args.cluster, conf_data.getvalue(), overwrite=args.overwrite_conf)

    if not distro.sudo_conn.modules.os.path.exists(path):
        logger.info('creating path: %s' % path)
        distro.sudo_conn.modules.os.makedirs(path)

    logger.debug('checking for done path: %s' % done_path)
    if not distro.sudo_conn.modules.os.path.exists(done_path):
        logger.debug('done path does not exist: %s' % done_path)
        if not distro.sudo_conn.modules.os.path.exists(paths.mon.constants.tmp_path):
            logger.info('creating tmp path: %s' % paths.mon.constants.tmp_path)
            distro.sudo_conn.modules.os.makedirs(paths.mon.constants.tmp_path)
        keyring = paths.mon.keyring(args.cluster, hostname)

        def write_monitor_keyring(keyring, monitor_keyring):
            """create the monitor keyring file"""
            with file(keyring, 'w') as f:
                f.write(monitor_keyring)

        logger.info('creating keyring file: %s' % keyring)
        with remote(distro.sudo_conn, logger, write_monitor_keyring) as remote_func:
            remote_func(keyring, monitor_keyring)

        check_call(
            distro.sudo_conn,
            logger,
            [
                'ceph-mon',
                '--cluster', args.cluster,
                '--mkfs',
                '-i', hostname,
                '--keyring', keyring,
            ],
        )

        logger.info('unlinking keyring file %s' % keyring)
        distro.sudo_conn.modules.os.unlink(keyring)

    def create_done_path(done_path):
        """create a done file to avoid re-doing the mon deployment"""
        with file(done_path, 'w'):
            pass

    with remote(distro.sudo_conn, logger, create_done_path) as remote_func:
        remote_func(done_path)

    def create_init_path(init_path):
        """create the init path if it does not exist"""
        import os
        if not os.path.exists(init_path):
            with file(init_path, 'w'):
                pass

    with remote(distro.sudo_conn, logger, create_init_path) as remote_func:
        remote_func(init_path)
Esempio n. 7
0
def install(distro, logger, version_kind, version, adjust_repos):
    codename = distro.codename
    machine = distro.sudo_conn.modules.platform.machine()

    if version_kind in ['stable', 'testing']:
        key = 'release'
    else:
        key = 'autobuild'

    # Make sure ca-certificates is installed
    check_call(
        distro.sudo_conn,
        logger,
        [
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'apt-get',
            '-q',
            'install',
            '--assume-yes',
            'ca-certificates',
        ]
    )

    if adjust_repos:
        check_call(
            distro.sudo_conn,
            logger,
            ['wget -q -O- \'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc\' | apt-key add -'.format(key=key)],
            shell=True,
        )

        if version_kind == 'stable':
            url = 'http://ceph.com/debian-{version}/'.format(
                version=version,
                )
        elif version_kind == 'testing':
            url = 'http://ceph.com/debian-testing/'
        elif version_kind == 'dev':
            url = 'http://gitbuilder.ceph.com/ceph-deb-{codename}-{machine}-basic/ref/{version}'.format(
                codename=codename,
                machine=machine,
                version=version,
                )
        else:
            raise RuntimeError('Unknown version kind: %r' % version_kind)

        def write_sources_list(url, codename):
            """add ceph deb repo to sources.list"""
            with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
                f.write('deb {url} {codename} main\n'.format(
                        url=url,
                        codename=codename,
                        ))

        with remote(distro.sudo_conn, logger, write_sources_list) as remote_func:
            remote_func(url, codename)

    check_call(
        distro.sudo_conn,
        logger,
        ['apt-get', '-q', 'update'],
        )

    # TODO this does not downgrade -- should it?
    check_call(
        distro.sudo_conn,
        logger,
        [
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'DEBIAN_PRIORITY=critical',
            'apt-get',
            '-q',
            '-o', 'Dpkg::Options::=--force-confnew',
            '--no-install-recommends',
            '--assume-yes',
            'install',
            '--',
            'ceph',
            'ceph-mds',
            'ceph-common',
            'ceph-fs-common',
            # ceph only recommends gdisk, make sure we actually have
            # it; only really needed for osds, but minimal collateral
            'gdisk',
            ],
        )

    # Check the ceph version
    common.ceph_version(distro.sudo_conn, logger)
Esempio n. 8
0
def mon_create(distro, logger, args, monitor_keyring, hostname):
    logger.debug('remote hostname: %s' % hostname)
    path = paths.mon.path(args.cluster, hostname)
    done_path = paths.mon.done(args.cluster, hostname)
    init_path = paths.mon.init(args.cluster, hostname, distro.init)

    configuration = conf.load(args)
    conf_data = StringIO()
    configuration.write(conf_data)

    with remote(distro.sudo_conn, logger, conf.write_conf) as remote_func:
        remote_func(args.cluster,
                    conf_data.getvalue(),
                    overwrite=args.overwrite_conf)

    if not distro.sudo_conn.modules.os.path.exists(path):
        logger.info('creating path: %s' % path)
        distro.sudo_conn.modules.os.makedirs(path)

    logger.debug('checking for done path: %s' % done_path)
    if not distro.sudo_conn.modules.os.path.exists(done_path):
        logger.debug('done path does not exist: %s' % done_path)
        if not distro.sudo_conn.modules.os.path.exists(
                paths.mon.constants.tmp_path):
            logger.info('creating tmp path: %s' % paths.mon.constants.tmp_path)
            distro.sudo_conn.modules.os.makedirs(paths.mon.constants.tmp_path)
        keyring = paths.mon.keyring(args.cluster, hostname)

        def write_monitor_keyring(keyring, monitor_keyring):
            """create the monitor keyring file"""
            with file(keyring, 'w') as f:
                f.write(monitor_keyring)

        logger.info('creating keyring file: %s' % keyring)
        with remote(distro.sudo_conn, logger,
                    write_monitor_keyring) as remote_func:
            remote_func(keyring, monitor_keyring)

        check_call(
            distro.sudo_conn,
            logger,
            [
                'ceph-mon',
                '--cluster',
                args.cluster,
                '--mkfs',
                '-i',
                hostname,
                '--keyring',
                keyring,
            ],
        )

        logger.info('unlinking keyring file %s' % keyring)
        distro.sudo_conn.modules.os.unlink(keyring)

    def create_done_path(done_path):
        """create a done file to avoid re-doing the mon deployment"""
        with file(done_path, 'w'):
            pass

    with remote(distro.sudo_conn, logger, create_done_path) as remote_func:
        remote_func(done_path)

    def create_init_path(init_path):
        """create the init path if it does not exist"""
        import os
        if not os.path.exists(init_path):
            with file(init_path, 'w'):
                pass

    with remote(distro.sudo_conn, logger, create_init_path) as remote_func:
        remote_func(init_path)