def _Connect(sock, address, timeout, allow_non_master):
sock.settimeout(timeout)
try:
sock.connect(address)
except socket.timeout as err:
raise errors.TimeoutError("Connect timed out: %s" % str(err))
except socket.error as err:
error_code = err.args[0]
if error_code in (errno.ENOENT, errno.ECONNREFUSED):
if not allow_non_master:
# Verify if we're actually on the master node before trying
# again.
ss = ssconf.SimpleStore()
try:
master, myself = ssconf.GetMasterAndMyself(ss=ss)
except ganeti.errors.ConfigurationError:
raise errors.NoMasterError(address)
if master != myself:
raise errors.NoMasterError(address)
raise utils.RetryAgain()
elif error_code in (errno.EPERM, errno.EACCES):
raise errors.PermissionError(address)
elif error_code == errno.EAGAIN:
# Server's socket backlog is full at the moment
raise utils.RetryAgain()
raise
def _CheckNetworkConfig():
info = self._GetShowInfo(minor)
if not "local_addr" in info or not "remote_addr" in info:
raise utils.RetryAgain()
if (info["local_addr"] != (lhost, lport) or info["remote_addr"] !=
(rhost, rport)):
raise utils.RetryAgain()
def _CheckSshDaemon():
if netutils.TcpPing(hostip, port, timeout=1.0, live_port_needed=True):
logging.debug(
"SSH daemon on %s:%s (IP address %s) has become"
" responsive", hostname, port, hostip)
else:
raise utils.RetryAgain()
def _RetryAndSucceed(self, retries):
self.time += self.time_for_retry_and_succeed
if self.retries < retries:
self.retries += 1
raise utils.RetryAgain()
else:
return True
def _CheckMasterDaemon():
try:
cl = luxi.Client()
(cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
except Exception:
raise utils.RetryAgain()
logging.debug("Received cluster name %s from master", cluster_name)
def _CheckInstance():
new_info = self.GetInstanceInfo(instance.name, hvparams=instance.hvparams)
# check if the domain ID has changed or the run time has decreased
if (new_info is not None and
(new_info[1] != ini_info[1] or new_info[5] < ini_info[5])):
return
raise utils.RetryAgain()
def _WaitForDisconnect():
if self.GetProcStatus().is_standalone:
return
# retry the disconnect, it seems possible that due to a well-time
# disconnect on the peer, my disconnect command might be ignored and
# forgotten
dstatus.ever_disconnected = \
base.IgnoreError(self._ShutdownNet, self.minor) or \
dstatus.ever_disconnected
raise utils.RetryAgain()
def _CheckResponse():
if salt not in self._requests:
# expired?
if self._logger:
self._logger.debug("Discarding unknown/expired request: %s" % salt)
return MISSING
rq = self._requests[salt]
if len(rq.rcvd) >= expected:
# already got all replies
return (False, len(rq.sent), len(rq.rcvd))
# else wait, using default timeout
self.ReceiveReply()
raise utils.RetryAgain()
def _RunInstanceList(fn, instance_list_errors):
"""Helper function for L{_GetAllInstanceList} to retrieve the list
of instances from xen.
@type fn: callable
@param fn: Function to query xen for the list of instances
@type instance_list_errors: list
@param instance_list_errors: Error list
@rtype: list
"""
result = fn()
if result.failed:
logging.error("Retrieving the instance list from xen failed (%s): %s",
result.fail_reason, result.output)
instance_list_errors.append(result)
raise utils.RetryAgain()
# skip over the heading
return result.stdout.splitlines()
def _CheckNodeDaemon():
# Pylint bug <http://www.logilab.org/ticket/35642>
# pylint: disable=E1101
result = rpc.BootstrapRunner().call_version([node_name])[node_name]
if result.fail_msg:
raise utils.RetryAgain()
def _check_ip(expected):
if netutils.TcpPing(master_ip,
constants.DEFAULT_NODED_PORT) != expected:
raise utils.RetryAgain()
def _WaitForMinorSyncParams():
"""Call _SetMinorSyncParams and raise RetryAgain on errors.
"""
if self._SetMinorSyncParams(minor, self.params):
raise utils.RetryAgain()
def _CheckFile():
if not (os.path.isfile(path) and
utils.ReadFile(path).strip() == expected):
raise utils.RetryAgain()
class Transport:
"""Low-level transport class.
This is used on the client side.
This could be replaced by any other class that provides the same
semantics to the Client. This means:
- can send messages and receive messages
- safe for multithreading
"""
def __init__(self, address, timeouts=None):
"""Constructor for the Client class.
Arguments:
- address: a valid address the the used transport class
- timeout: a list of timeouts, to be used on connect and read/write
There are two timeouts used since we might want to wait for a long
time for a response, but the connect timeout should be lower.
If not passed, we use a default of 10 and respectively 60 seconds.
Note that on reading data, since the timeout applies to an
invidual receive, it might be that the total duration is longer
than timeout value passed (we make a hard limit at twice the read
timeout).
"""
self.address = address
if timeouts is None:
self._ctimeout, self._rwtimeout = DEF_CTMO, DEF_RWTO
else:
self._ctimeout, self._rwtimeout = timeouts
self.socket = None
self._buffer = ""
self._msgs = collections.deque()
try:
self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Try to connect
try:
utils.Retry(self._Connect, 1.0, self._ctimeout,
args=(self.socket, address, self._ctimeout))
except utils.RetryTimeout:
raise errors.TimeoutError("Connect timed out")
self.socket.settimeout(self._rwtimeout)
except (socket.error, errors.NoMasterError):
if self.socket is not None:
self.socket.close()
self.socket = None
raise
@staticmethod
def _Connect(sock, address, timeout):
sock.settimeout(timeout)
try:
sock.connect(address)
except socket.timeout, err:
raise errors.TimeoutError("Connect timed out: %s" % str(err))
except socket.error, err:
error_code = err.args[0]
if error_code in (errno.ENOENT, errno.ECONNREFUSED):
raise utils.RetryAgain()
elif error_code in (errno.EPERM, errno.EACCES):
raise errors.PermissionError(address)
elif error_code == errno.EAGAIN:
# Server's socket backlog is full at the moment
raise utils.RetryAgain()
raise
class Transport:
"""Low-level transport class.
This is used on the client side.
This could be replaced by any other class that provides the same
semantics to the Client. This means:
- can send messages and receive messages
- safe for multithreading
"""
def __init__(self, address, timeouts=None, allow_non_master=None):
"""Constructor for the Client class.
There are two timeouts used since we might want to wait for a long
time for a response, but the connect timeout should be lower.
If not passed, we use the default luxi timeouts from the global
constants file.
Note that on reading data, since the timeout applies to an
invidual receive, it might be that the total duration is longer
than timeout value passed (we make a hard limit at twice the read
timeout).
@type address: socket address
@param address: address the transport connects to
@type timeouts: list of ints
@param timeouts: timeouts to be used on connect and read/write
@type allow_non_master: bool
@param allow_non_master: skip checks for the master node on errors
"""
self.address = address
if timeouts is None:
self._ctimeout, self._rwtimeout = DEF_CTMO, DEF_RWTO
else:
self._ctimeout, self._rwtimeout = timeouts
self.socket = None
self._buffer = ""
self._msgs = collections.deque()
try:
self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Try to connect
try:
utils.Retry(self._Connect,
1.0,
self._ctimeout,
args=(self.socket, address, self._ctimeout,
allow_non_master))
except utils.RetryTimeout:
raise errors.TimeoutError("Connect timed out")
self.socket.settimeout(self._rwtimeout)
except (socket.error, errors.NoMasterError):
if self.socket is not None:
self.socket.close()
self.socket = None
raise
@staticmethod
def _Connect(sock, address, timeout, allow_non_master):
sock.settimeout(timeout)
try:
sock.connect(address)
except socket.timeout, err:
raise errors.TimeoutError("Connect timed out: %s" % str(err))
except socket.error, err:
error_code = err.args[0]
if error_code in (errno.ENOENT, errno.ECONNREFUSED):
if not allow_non_master:
# Verify if we're actually on the master node before trying
# again.
ss = ssconf.SimpleStore()
try:
master, myself = ssconf.GetMasterAndMyself(ss=ss)
except ganeti.errors.ConfigurationError:
raise errors.NoMasterError(address)
if master != myself:
raise errors.NoMasterError(address)
raise utils.RetryAgain()
elif error_code in (errno.EPERM, errno.EACCES):
raise errors.PermissionError(address)
elif error_code == errno.EAGAIN:
# Server's socket backlog is full at the moment
raise utils.RetryAgain()
raise
def _RaiseRetryAgainWithArg(args):
raise utils.RetryAgain(*args)
def _RaiseRetryAgain():
raise utils.RetryAgain()
