def waitForSSH(self,
timeout,
level=xenrt.RC_FAIL,
desc="Operation",
username="******",
cmd="true"):
now = xenrt.util.timenow()
deadline = now + timeout
while 1:
if not self.password:
self.findPassword()
if xenrt.ssh.SSH(self.getIP(trafficType="SSH"),
cmd,
port=self.getPort(trafficType="SSH"),
password=self.password,
level=xenrt.RC_OK,
timeout=20,
username=username,
nowarn=True) == xenrt.RC_OK:
xenrt.TEC().logverbose(" ... OK reply from %s:%s" %
(self.getIP(trafficType="SSH"),
self.getPort(trafficType="SSH")))
return xenrt.RC_OK
now = xenrt.util.timenow()
if now > deadline:
# if level == xenrt.RC_FAIL:
# self.checkHealth(unreachable=True)
return xenrt.XRT("%s timed out" % (desc), level)
xenrt.sleep(15, log=False)
def __init__(self,
ip,
username="******",
timeout=1250,
level=xenrt.RC_ERROR,
password=None,
nowarn=False,
port=22):
xenrt.TEC().logverbose("SFTP session to %s@%s" % (username, ip))
self.ip = ip
self.port = port
self.username = username
self.timeout = timeout
self.level = level
self.password = password
self.nowarn = nowarn
SSHSession.__init__(self,
ip,
username=username,
timeout=timeout,
level=level,
password=password,
nowarn=nowarn,
port=port)
try:
# We do this rather than the simple trans.open_sftp_client() because
# if we don't then we don't get a timeout set so we can hang forever
c = self.trans.open_channel("session")
c.settimeout(timeout)
c.invoke_subsystem("sftp")
self.client = paramiko.SFTPClient(c)
except:
self.reply = xenrt.XRT("SFTP connection failed", self.level)
self.toreply = 1
self.close()
def poll(self, state, timeout=600, level=xenrt.RC_FAIL, pollperiod=5):
"""Poll our VM for reaching the specified state"""
deadline = xenrt.timenow() + timeout
while True:
status = self.getState()
if state == status:
return
if xenrt.timenow() > deadline:
xenrt.XRT("Timed out waiting for VM %s to be %s" %
(self.name, state), level)
time.sleep(pollperiod)
def __init__(self,
ip,
command,
username="******",
timeout=1200,
level=xenrt.RC_ERROR,
password=None,
nowarn=False,
newlineok=False,
nolog=False,
useThread=False,
usePty=False,
port=22):
SSHSession.__init__(self,
ip,
username=username,
timeout=timeout,
level=level,
password=password,
nowarn=nowarn,
useThread=useThread,
port=port)
self.command = command
self.nolog = nolog
if string.find(command, "\n") > -1 and not newlineok:
xenrt.TEC().warning("Command with newline: '%s'" % (command))
try:
self.client = self.open_session()
if self.debug:
xenrt.TEC().logverbose("settimeout")
self.client.settimeout(timeout)
if self.debug:
xenrt.TEC().logverbose("set_combine_stderr")
self.client.set_combine_stderr(True)
if usePty:
if self.debug:
xenrt.TEC().logverbose("get_pty")
self.client.get_pty()
if self.debug:
xenrt.TEC().logverbose("exec_command")
self.client.exec_command(command)
if self.debug:
xenrt.TEC().logverbose("shutdown(1)")
self.client.shutdown(1)
if self.debug:
xenrt.TEC().logverbose("makefile")
self.fh = self.client.makefile()
except Exception, e:
if self.debug:
traceback.print_exc(file=sys.stderr)
self.reply = xenrt.XRT("SSH connection failed", self.level)
self.toreply = 1
self.close()
def _osParent_pollPowerState(self,
state,
timeout=600,
level=xenrt.RC_FAIL,
pollperiod=15):
"""Poll for reaching the specified state"""
deadline = xenrt.timenow() + timeout
while 1:
status = self.getPowerState()
if state == status:
return
if xenrt.timenow() > deadline:
xenrt.XRT(
"Timed out waiting for VM %s to be %s" %
(self.name, state), level)
xenrt.sleep(15, log=False)
def arpwatch(self, iface, mac, timeout=600, level=xenrt.RC_FAIL):
"""Monitor an interface (or bridge) for an ARP reply"""
deadline = xenrt.util.timenow() + timeout
while True:
ip = self.checkLeases(mac)
if ip:
break
xenrt.sleep(20)
if xenrt.util.timenow() > deadline:
xenrt.XRT("Timed out monitoring for guest DHCP lease",
level,
data=mac)
return ip
def getInstanceIP(self, instance, timeout=600, level=xenrt.RC_ERROR):
if self.getInstancePowerState(instance) != xenrt.PowerState.up:
raise xenrt.XRTError("Instance not running")
if instance.mainip is not None:
return instance.mainip
# If we haven't got an IP, use arpwatch to try and find one
# TODO: Find the mac in a 'cleaner' way
mac = instance.vifs[0][2]
ip = self.getHypervisor(instance).arpwatch("xenbr0", mac, timeout)
if not ip:
raise xenrt.XRT("Timed out monitoring for guest ARP/DHCP",
level,
data=mac)
instance.mainip = ip
return ip
def read(self, retval="code", fh=None):
"""Process the output and result of the command.
@param retval: Whether to return the result code (default) or
stdout as a string.
string : Return a stdout as a string.
code : Return the result code. (Default).
If "string" is used then a failure results in an exception.
"""
if self.toreply:
if retval == "string":
raise self.reply
return self.reply
reply = ""
while True:
try:
if fh:
output = self.fh.read(4096)
else:
if self.debug:
xenrt.TEC().logverbose("readline")
output = self.fh.readline()
except socket.timeout:
if self.debug:
xenrt.TEC().logverbose("close")
self.close()
return xenrt.XRT("SSH timed out", self.level)
if len(output) == 0:
break
if fh:
fh.write(output)
elif retval == "string":
reply = reply + output
if not self.nolog and not fh:
xenrt.TEC().log(output)
if self.debug:
xenrt.TEC().logverbose("recv_exit_status")
self.exit_status = self.client.recv_exit_status()
# Local clean up.
if self.debug:
xenrt.TEC().logverbose("close (2)")
self.close()
if retval == "code":
return self.exit_status
if self.exit_status == -1:
return xenrt.XRT("SSH channel closed unexpectedly",
self.level,
data=reply)
elif not self.exit_status == 0:
return xenrt.XRT("SSH command exited with error (%s)" %
(self.command), self.level, data=reply)
if self.debug:
xenrt.TEC().logverbose("done (3)")
return reply
def __init__(self,
ip,
username="******",
timeout=300,
level=xenrt.RC_ERROR,
password=None,
nowarn=False,
useThread=False,
port=22):
self.level = level
self.toreply = 0
self.debug = False
self.trans = None
for tries in range(3):
self.trans = None
try:
if useThread:
t = xenrt.util.ThreadWithException(target=self.connect,
args=(ip, port, username,
password, timeout))
# Make the thread daemonic (so python will exit if it ends
# up hung and still running)
t.setDaemon(True)
t.start()
t.join(timeout)
if t.isAlive():
raise xenrt.XRTFailure("Connection appears to have hung")
if t.exception:
raise t.exception
else:
self.connect(ip, port, username, password, timeout)
except Exception, e:
traceback.print_exc(file=sys.stderr)
desc = str(e)
xenrt.TEC().logverbose("SSH exception %s" % (desc))
if string.find(desc, "Signature verification") > -1 or \
string.find(desc,
"Error reading SSH protocol banner") > -1:
# Have another go
if not nowarn:
xenrt.TEC().warning(\
"Retrying SSH connection after '%s'" % (desc))
try:
self.close()
except:
pass
xenrt.sleep(1)
continue
elif string.find(desc, "Authentication failed") > -1:
self.reply = xenrt.XRT("SSH authentication failed",
self.level)
self.toreply = 1
self.close()
break
else:
# Probably a legitimate exception
pass
self.reply = xenrt.XRT("SSH connection failed", self.level)
self.toreply = 1
self.close()
break
if self.debug:
xenrt.TEC().logverbose("done")
# If we get here we have successfully opened a connection
return