Пример #1
0
    def _init_endpoints(self, providers, min_conns_per_pool,
                        max_conns_per_pool):
        LOG.debug("Initializing API endpoints")

        def _create_conn(p):
            def _conn():
                # called when a pool needs to create a new connection
                try:
                    return self._http_provider.new_connection(self, p)
                except Exception as e:
                    if self._http_provider.is_conn_open_exception(e):
                        LOG.warning(
                            "Timeout while trying to open a "
                            "connection with %s", p)

            return _conn

        self._endpoints = {}
        for provider in providers:
            pool = pools.Pool(min_size=min_conns_per_pool,
                              max_size=max_conns_per_pool,
                              order_as_stack=True,
                              create=_create_conn(provider))

            endpoint = Endpoint(provider, pool)
            self._endpoints[provider.id] = endpoint

        # service requests using round robin
        self._endpoint_schedule = itertools.cycle(self._endpoints.values())

        # duck type to proxy http invocations
        for method in ClusteredAPI._HTTP_VERBS:
            setattr(self, method, self._proxy_stub(method))

        conns = greenpool.GreenPool()
        for endpoint in self._endpoints.values():
            conns.spawn(self._validate, endpoint)
        eventlet.sleep(0)
        while conns.running():
            if (self.health == ClusterHealth.GREEN
                    or self.health == ClusterHealth.ORANGE):
                # only wait for 1 or more endpoints to reduce init time
                break
            eventlet.sleep(0.5)

        for endpoint in self._endpoints.values():
            # dynamic loop for each endpoint to ensure connectivity
            loop = loopingcall.DynamicLoopingCall(self._endpoint_keepalive,
                                                  endpoint)
            loop.start(initial_delay=self._keepalive_interval,
                       periodic_interval_max=self._keepalive_interval,
                       stop_on_exception=False)

        LOG.debug(
            "Done initializing API endpoint(s). "
            "API cluster health: %s", self.health)
Пример #2
0
    def get_http(self, address, timeout=0):
        try:
            configured_timeout = self.bitHopper.config.getfloat(
                'main', 'work_request_timeout')
        except:
            configured_timeout = 2.5
            pass
        if timeout == 0:
            timeout = configured_timeout

        if address not in self.connect_pool:
            self.connect_pool[address] = pools.Pool(
                min_size=0,
                create=lambda: httplib2.Http(
                    disable_ssl_certificate_validation=True, timeout=timeout))
        return self.connect_pool[address].item()
Пример #3
0
    def __init__(self,
                 app_key=conf.API_KEY,
                 server='api.eventful.com',
                 make_dumps=False):
        self.app_key = app_key
        self.server = server
        self.httpool = pools.Pool()
        self.httpool.create = httplib2.Http
        self.make_dumps = make_dumps
        self.dump_dir = os.path.join(settings.IMPORT_ROOT_DIR, conf.IMPORT_DIR,
                                     'dumps')
        if make_dumps and not os.path.exists(self.dump_dir):
            os.makedirs(self.dump_dir)

        # prepare image download directory
        self.img_dir = conf.IMPORT_IMAGE_DIR
        if not os.path.exists(self.img_dir):
            os.makedirs(self.img_dir)
Пример #4
0
    def test_create_contention(self):
        creates = [0]
        def sleep_create():
            creates[0] += 1
            eventlet.sleep()
            return "slept"
            
        p = pools.Pool(max_size=4, create=sleep_create)

        def do_get():
            x = p.get()
            self.assertEquals(x, "slept")
            p.put(x)

        gp = eventlet.GreenPool()
        for i in xrange(100):
            gp.spawn_n(do_get)
        gp.waitall()
        self.assertEquals(creates[0], 4)
Пример #5
0
 def test_abstract(self):
     ## Going for 100% coverage here
     ## A Pool cannot be used without overriding create()
     pool = pools.Pool()
     self.assertRaises(NotImplementedError, pool.get)
Пример #6
0
 def regenerate_pool(self):
     self.pool = pools.Pool(min_size=self.pool.min_size,
                            max_size=self.pool.max_size,
                            order_as_stack=True,
                            create=self.pool.create)