def test2(self,output=None):
if not output or 'rtt' in output:
self.test_state = "being-tested"
args = ['-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-o', 'ConnectTimeout=30',
'-o', 'Batchmode=yes',
'-i', self.private_key,
"root@%s" % (self.pub_ip),
"echo TEST_SUCCESS" ]
self.logger.info("Testing instance: ssh %s %s" % (self.id,self.pub_ip))
output = myutils.getProcessOutput('ssh', args, errortoo=True)
output.addCallback(self.tested)
elif self._test_retries <= self.TEST_MAX_RETRIES:
self.logger.warning("Rescheduling test ping (%s/%s)" %
(self._test_retries, self.TEST_MAX_RETRIES))
self.test_state = "rescheduled"
if output:
self.logger.debug("Test output: ping %s\n%s" % (self.id,output))
reactor.callLater(5, self.test)
else:
if output:
self.logger.error("Test output: ping %s\n%s" % (self.id,output))
self.logger.info("Test failed")
self.test_state = "failed"
output = self.getProcessOutput(euca_get_console_output,
args = [self.id], timeout=300)
output.addCallback(self.consoleOutput)
def test(self):
if self.state == "pending":
self._test_retries += 1
if self._test_retries <= self.TEST_MAX_RETRIES:
self.logger.error("Not running - aborting scheduled test.")
return
else:
self.logger.error(
"Instance remained in pending for MAX_RETRIES, terminating instance"
)
self.test_state = 'boot-failed'
output = self.getProcessOutput(euca_terminate_instances,
args=[self.id])
output.addCallback(self.terminate)
return
if self.state != "running":
self.logger.error("Not running - aborting scheduled test.")
return
self.logger.info("Testing instance: ping %s" % (self.id))
self.test_state = "being-tested"
self._test_retries += 1
output = myutils.getProcessOutput('bash', [
'-c',
"ping -U -n -q -i .2 -c 5 -w 60 -W 1 %s | awk '/^PING/||/^rtt/' | xargs echo -n"
% (self.pub_ip)
],
errortoo=True)
output.addCallback(self.test2)
def getProcessOutput(self, executable, args=(), timeout=60):
myargs = [ '-c', "%s --config=%s/eucarc %s" % (
executable,
self.config["cred_dir"],
string.join(args))]
self.logger.debug("Running bash command: %s " % (myargs))
return myutils.getProcessOutput('bash', myargs, timeout=timeout,
env=os.environ)
def test(self):
if self.state != "running":
self.logger.error("Not running - aborting scheduled test.")
return
self.logger.info("Testing instance: ping %s" % (self.id))
self.test_state = "being-tested"
self._test_retries += 1
output = myutils.getProcessOutput('bash', ['-c', "ping -U -n -q -i .2 -c 5 -w 60 -W 1 %s | awk '/^PING/||/^rtt/' | xargs echo -n" % (self.pub_ip) ], errortoo=True)
output.addCallback(self.test2)