def _next_server(self, cause=None):
""" Selects the next server in the list, refreshes the server list. """
_log.debug("Selection next machine in cache. Available machines: %s",
self._machines_cache)
try:
mach = self._machines_cache.pop()
except IndexError as e:
_log.error("Machines cache is empty, no machines to try.")
raise etcd.EtcdConnectionFailed('No more machines in the cluster', cause=cause) from e
else:
_log.info("Selected new etcd server %s", mach)
return mach
async def wrapper(self, path, method, params=None, **kw):
some_request_failed = False
response = False
if not path.startswith('/'):
raise ValueError('Path does not start with /')
if not self._machines_available:
await self._update_machines()
while not response:
some_request_failed = False
try:
response = await payload(self, path, method, params=params, **kw)
# Check the cluster ID hasn't changed under us. We use
# preload_content=False above so we can read the headers
# before we wait for the content of a watch.
self._check_cluster_id(response)
# Now force the data to be preloaded in order to trigger any
# IO-related errors in this method rather than when we try to
# access it later.
_ = await response.read()
# urllib3 doesn't wrap all httplib exceptions and earlier versions
# don't wrap socket errors either.
except (ClientResponseError, DisconnectedError, HTTPException,
socket.error, asyncio.TimeoutError) as e:
if (isinstance(params, dict) and
params.get("wait") == "true" and
isinstance(e, ReadTimeoutError)):
_log.debug("Watch timed out.")
raise etcd.EtcdWatchTimedOut(
"Watch timed out: %r" % e,
cause=e
)
_log.error("Request to server %s failed: %r",
self._base_uri, e)
if self._allow_reconnect:
_log.info("Reconnection allowed, looking for another "
"server.")
# _next_server() raises EtcdException if there are no
# machines left to try, breaking out of the loop.
self._base_uri = self._next_server(cause=e)
some_request_failed = True
# if exception is raised on _ = response.data
# the condition for while loop will be False
# but we should retry
response = False
else:
_log.debug("Reconnection disabled, giving up.")
raise etcd.EtcdConnectionFailed(
"Connection to etcd failed due to %r" % e,
cause=e
)
except etcd.EtcdClusterIdChanged as e:
_log.warning(e)
raise
except asyncio.CancelledError:
# don't complain
raise
except Exception:
_log.exception("Unexpected request failure, re-raising.")
raise
else:
try:
response = await self._handle_server_response(response)
except etcd.EtcdException as e:
if "during rolling upgrades" in e.payload['message']:
response = False
some_request_failed = True
else:
raise
if some_request_failed:
if not self._use_proxies:
# The cluster may have changed since last invocation
self._machines_cache = await self.machines()
if self._base_uri in self._machines_cache:
self._machines_cache.remove(self._base_uri)
return response
async def api_execute(self, path, method, params=None):
""" Executes the query. """
if not path.startswith('/'):
raise ValueError('Path does not start with /')
url = self._base_uri + path
async with aiohttp.ClientSession(loop=self._loop) as session:
some_request_failed = False
response = False
if not path.startswith('/'):
raise ValueError('Path does not start with /')
if not self._machines_available:
await self._update_machines()
while not response:
try:
response = await session.request(
method,
url,
params=params,
auth=self._get_auth(),
allow_redirects=self.allow_redirect)
# Check the cluster ID hasn't changed under us. We use
# preload_content=False above so we can read the headers
# before we wait for the content of a watch.
self._check_cluster_id(response)
# Now force the data to be preloaded in order to trigger any
# IO-related errors in this method rather than when we try to
# access it later.
_ = await response.read() # noqa
# urllib3 doesn't wrap all httplib exceptions and earlier versions
# don't wrap socket errors either.
except (ClientConnectionError, ClientResponseError,
HTTPException, socket.error) as e:
logger.error("Request to server %s failed: %r",
self._base_uri, e)
if self._allow_reconnect:
logger.info(
"Reconnection allowed, looking for another "
"server.")
# _next_server() raises EtcdException if there are no
# machines left to try, breaking out of the loop.
self._base_uri = self._next_server(cause=e)
some_request_failed = True
# if exception is raised on _ = response.data
# the condition for while loop will be False
# but we should retry
response = False
else:
logger.debug("Reconnection disabled, giving up.")
raise aio_etcd.EtcdConnectionFailed(
"Connection to etcd failed due to %r" % e, cause=e)
except aio_etcd.EtcdClusterIdChanged as e:
logger.warning(e)
raise
except asyncio.CancelledError:
# don't complain
raise
except Exception:
logger.exception("Unexpected request failure, re-raising.")
raise
else:
try:
response = await self._handle_server_response(response)
except aio_etcd.EtcdException as e:
if "during rolling upgrades" in e.payload['message']:
response = False
some_request_failed = True
else:
raise
if some_request_failed:
if not self._use_proxies:
# The cluster may have changed since last invocation
self._machines_cache = await self.machines()
if self._base_uri in self._machines_cache:
self._machines_cache.remove(self._base_uri)
return response