Exemplo n.º 1
0
def wait_for_radosgw(url):
    """ poll the given url until it starts accepting connections

    add_daemon() doesn't wait until radosgw finishes startup, so this is used
    to avoid racing with later tasks that expect radosgw to be up and listening
    """
    # use a connection pool with retry/backoff to poll until it starts listening
    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
    http.request('GET', url)
Exemplo n.º 2
0
def wait_for_radosgw(url):
    """ poll the given url until it starts accepting connections

    add_daemon() doesn't wait until radosgw finishes startup, so this is used
    to avoid racing with later tasks that expect radosgw to be up and listening
    """
    # use a connection pool with retry/backoff to poll until it starts listening
    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
    http.request('GET', url)
Exemplo n.º 3
0
def _get_from_remote(sanitized_url):
	"""
	Get contents of a sanitized remote URL.

	"""
	try:
		http = PoolManager()
		return http.request("GET", "http://{0}".format(sanitized_url))
	except MaxRetryError:
		raise TextMiningException("URL {0} is unavailable.".format(sanitized_url))
Exemplo n.º 4
0
    def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
        """
        Initializes a urllib3 PoolManager.

        This method should not be called from user code, and is only
        exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param connections: The number of urllib3 connection pools to cache.
        :param maxsize: The maximum number of connections to save in the pool.
        :param block: Block when no free connections are available.
        :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
        """
        self._pool_connections = connections
        self._pool_maxsize = maxsize
        self._pool_block = block
        self.poolmanager = PoolManager(
            num_pools=connections,
            maxsize=maxsize,
            block=block,
            strict=True,
            ssl_version=self.SSL_VERSION,
            **pool_kwargs
        )
Exemplo n.º 5
0
def start_rgw(ctx, config, on_client=None, except_client=None):
    """
    Start rgw on remote sites.
    """
    log.info('Starting rgw...')
    log.debug('client %r', on_client)
    clients_to_run = [on_client]
    if on_client is None:
        clients_to_run = config.keys()
    testdir = teuthology.get_testdir(ctx)
    for client in clients_to_run:
        if client == except_client:
            continue
        (remote, ) = ctx.cluster.only(client).remotes.iterkeys()
        cluster_name, daemon_type, client_id = teuthology.split_role(client)
        client_with_id = daemon_type + '.' + client_id
        client_with_cluster = cluster_name + '.' + client_with_id
        zone = rgw_utils.zone_for_client(ctx, client)
        log.debug('zone %s', zone)
        client_config = config.get(client)
        if client_config is None:
            client_config = {}
        log.info("rgw %s config is %s", client, client_config)
        id_ = client.split('.', 1)[1]
        log.info('client {client} is id {id}'.format(client=client, id=id_))
        cmd_prefix = [
            'sudo',
            'adjust-ulimits',
            'ceph-coverage',
            '{tdir}/archive/coverage'.format(tdir=testdir),
            'daemon-helper',
            'term',
        ]

        rgw_cmd = ['radosgw']

        host, port = ctx.multisite.role_endpoints[client]
        rgw_cmd.extend([
            '--rgw-frontends',
            'civetweb port={port}'.format(port=port),
        ])

        if zone is not None:
            rgw_cmd.extend(['--rgw-zone', zone])

        rgw_cmd.extend([
            '-n',
            client_with_id,
            '--cluster',
            cluster_name,
            '-k',
            '/etc/ceph/{client_with_cluster}.keyring'.format(
                client_with_cluster=client_with_cluster),
            '--log-file',
            '/var/log/ceph/rgw.{client_with_cluster}.log'.format(
                client_with_cluster=client_with_cluster),
            '--rgw_ops_log_socket_path',
            '{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(
                tdir=testdir, client_with_cluster=client_with_cluster),
            '--foreground',
            run.Raw('|'),
            'sudo',
            'tee',
            '/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(
                tdir=testdir, client_with_cluster=client_with_cluster),
            run.Raw('2>&1'),
        ])

        run_cmd = list(cmd_prefix)
        run_cmd.extend(rgw_cmd)

        ctx.daemons.add_daemon(
            remote,
            'rgw',
            client,
            args=run_cmd,
            logger=log.getChild(client),
            stdin=run.PIPE,
            wait=False,
        )

    # XXX: add_daemon() doesn't let us wait until radosgw finishes startup
    # use a connection pool with retry/backoff to poll each gateway until it starts listening
    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
    for client in clients_to_run:
        if client == except_client:
            continue
        host, port = ctx.multisite.role_endpoints[client]
        endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
        log.info(
            'Polling {client} until it starts accepting connections on {endpoint}'
            .format(client=client, endpoint=endpoint))
        http.request('GET', endpoint)

    try:
        yield
    finally:
        teuthology.stop_daemons_of_type(ctx, 'rgw')
        for client in config.iterkeys():
            ctx.cluster.only(client).run(args=[
                'rm',
                '-f',
                '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
                                                         client=client),
            ], )
Exemplo n.º 6
0
 def init_poolmanager(self, connections, maxsize, block=False):
     self.poolmanager = PoolManager(num_pools=connections,
                                    maxsize=maxsize,
                                    ssl_version=ssl.PROTOCOL_SSLv3)
Exemplo n.º 7
0
def start_rgw(ctx, config, on_client = None, except_client = None):
    """
    Start rgw on remote sites.
    """
    log.info('Starting rgw...')
    log.debug('client %r', on_client)
    clients_to_run = [on_client]
    if on_client is None:
        clients_to_run = config.keys()
    testdir = teuthology.get_testdir(ctx)
    for client in clients_to_run:
        if client == except_client:
            continue
        (remote,) = ctx.cluster.only(client).remotes.iterkeys()
        zone = rgw_utils.zone_for_client(ctx, client)
        log.debug('zone %s', zone)
        client_config = config.get(client)
        if client_config is None:
            client_config = {}
        log.info("rgw %s config is %s", client, client_config)
        id_ = client.split('.', 1)[1]
        log.info('client {client} is id {id}'.format(client=client, id=id_))
        cmd_prefix = [
            'sudo',
            'adjust-ulimits',
            'ceph-coverage',
            '{tdir}/archive/coverage'.format(tdir=testdir),
            'daemon-helper',
            'term',
            ]

        rgw_cmd = ['radosgw']

        if ctx.rgw.frontend == 'apache':
            if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote):
                rgw_cmd.extend([
                    '--rgw-socket-path',
                    '{tdir}/apache/tmp.{client}/fastcgi_sock/rgw_sock'.format(
                        tdir=testdir,
                        client=client,
                    ),
                    '--rgw-frontends',
                    'fastcgi',
                ])
            else:
                # for mod_proxy_fcgi, using tcp
                rgw_cmd.extend([
                    '--rgw-socket-path', '',
                    '--rgw-print-continue', 'false',
                    '--rgw-frontends',
                    'fastcgi socket_port=9000 socket_host=0.0.0.0',
                ])

        elif ctx.rgw.frontend == 'civetweb':
            host, port = ctx.rgw.role_endpoints[client]
            rgw_cmd.extend([
                '--rgw-frontends',
                'civetweb port={port}'.format(port=port),
            ])

        if zone is not None:
            rgw_cmd.extend(['--rgw-zone', zone])

        rgw_cmd.extend([
            '-n', client,
            '-k', '/etc/ceph/ceph.{client}.keyring'.format(client=client),
            '--log-file',
            '/var/log/ceph/rgw.{client}.log'.format(client=client),
            '--rgw_ops_log_socket_path',
            '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
                                                     client=client),
            '--foreground',
            run.Raw('|'),
            'sudo',
            'tee',
            '/var/log/ceph/rgw.{client}.stdout'.format(tdir=testdir,
                                                       client=client),
            run.Raw('2>&1'),
            ])

        if client_config.get('valgrind'):
            cmd_prefix = teuthology.get_valgrind_args(
                testdir,
                client,
                cmd_prefix,
                client_config.get('valgrind')
                )

        run_cmd = list(cmd_prefix)
        run_cmd.extend(rgw_cmd)

        ctx.daemons.add_daemon(
            remote, 'rgw', client,
            args=run_cmd,
            logger=log.getChild(client),
            stdin=run.PIPE,
            wait=False,
            )

    # XXX: add_daemon() doesn't let us wait until radosgw finishes startup
    # use a connection pool with retry/backoff to poll each gateway until it starts listening
    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
    for client in clients_to_run:
        if client == except_client:
            continue
        host, port = ctx.rgw.role_endpoints[client]
        endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
        log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint))
        http.request('GET', endpoint)

    try:
        yield
    finally:
        teuthology.stop_daemons_of_type(ctx, 'rgw')
        for client in config.iterkeys():
            ctx.cluster.only(client).run(
                args=[
                    'rm',
                    '-f',
                    '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
                                                             client=client),
                    ],
                )
Exemplo n.º 8
0
def wait_for_radosgw(ctx, client):

    url_endpoint = extract_endpoints(ctx, client)
    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
    http.request('GET', url_endpoint)
Exemplo n.º 9
0
 def init_poolmanager(self, connections, maxsize, block=False):
     self.poolmanager = PoolManager(num_pools=connections,
                                    maxsize=maxsize,
                                    block=block,
                                    assert_fingerprint=self.fingerprint)