def _init_endpoints(self, providers, min_conns_per_pool,
max_conns_per_pool):
LOG.debug("Initializing API endpoints")
def _create_conn(p):
def _conn():
# called when a pool needs to create a new connection
try:
return self._http_provider.new_connection(self, p)
except Exception as e:
if self._http_provider.is_conn_open_exception(e):
LOG.warning(
"Timeout while trying to open a "
"connection with %s", p)
return _conn
self._endpoints = {}
for provider in providers:
pool = pools.Pool(min_size=min_conns_per_pool,
max_size=max_conns_per_pool,
order_as_stack=True,
create=_create_conn(provider))
endpoint = Endpoint(provider, pool)
self._endpoints[provider.id] = endpoint
# service requests using round robin
self._endpoint_schedule = itertools.cycle(self._endpoints.values())
# duck type to proxy http invocations
for method in ClusteredAPI._HTTP_VERBS:
setattr(self, method, self._proxy_stub(method))
conns = greenpool.GreenPool()
for endpoint in self._endpoints.values():
conns.spawn(self._validate, endpoint)
eventlet.sleep(0)
while conns.running():
if (self.health == ClusterHealth.GREEN
or self.health == ClusterHealth.ORANGE):
# only wait for 1 or more endpoints to reduce init time
break
eventlet.sleep(0.5)
for endpoint in self._endpoints.values():
# dynamic loop for each endpoint to ensure connectivity
loop = loopingcall.DynamicLoopingCall(self._endpoint_keepalive,
endpoint)
loop.start(initial_delay=self._keepalive_interval,
periodic_interval_max=self._keepalive_interval,
stop_on_exception=False)
LOG.debug(
"Done initializing API endpoint(s). "
"API cluster health: %s", self.health)
def get_http(self, address, timeout=0):
try:
configured_timeout = self.bitHopper.config.getfloat(
'main', 'work_request_timeout')
except:
configured_timeout = 2.5
pass
if timeout == 0:
timeout = configured_timeout
if address not in self.connect_pool:
self.connect_pool[address] = pools.Pool(
min_size=0,
create=lambda: httplib2.Http(
disable_ssl_certificate_validation=True, timeout=timeout))
return self.connect_pool[address].item()
def __init__(self,
app_key=conf.API_KEY,
server='api.eventful.com',
make_dumps=False):
self.app_key = app_key
self.server = server
self.httpool = pools.Pool()
self.httpool.create = httplib2.Http
self.make_dumps = make_dumps
self.dump_dir = os.path.join(settings.IMPORT_ROOT_DIR, conf.IMPORT_DIR,
'dumps')
if make_dumps and not os.path.exists(self.dump_dir):
os.makedirs(self.dump_dir)
# prepare image download directory
self.img_dir = conf.IMPORT_IMAGE_DIR
if not os.path.exists(self.img_dir):
os.makedirs(self.img_dir)
def test_create_contention(self):
creates = [0]
def sleep_create():
creates[0] += 1
eventlet.sleep()
return "slept"
p = pools.Pool(max_size=4, create=sleep_create)
def do_get():
x = p.get()
self.assertEquals(x, "slept")
p.put(x)
gp = eventlet.GreenPool()
for i in xrange(100):
gp.spawn_n(do_get)
gp.waitall()
self.assertEquals(creates[0], 4)
def test_abstract(self):
## Going for 100% coverage here
## A Pool cannot be used without overriding create()
pool = pools.Pool()
self.assertRaises(NotImplementedError, pool.get)
def regenerate_pool(self):
self.pool = pools.Pool(min_size=self.pool.min_size,
max_size=self.pool.max_size,
order_as_stack=True,
create=self.pool.create)