def isMozpoolReady(device):
""" Checks if the mozpool server is available and the device is in 'ready' state
Returns MOZPOOL_STATE_READY if Mozpool is indeed listing device as ready
Returns MOZPOOL_STATE_UNKNOWN if Mozpool claims the device is in a non-ready state
Returns MOZPOOL_STATE_ERROR if Mozpool reports some other error.
Returns MOZPOOL_STATE_MISSING if there is no Mozpool server found in DNS.
"""
import socket
default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(5) # Don't let networking delay us too long
try:
socket.gethostbyname(MOZPOOL_CNAME)
except:
log.info("No mozpool server in this VLAN")
return MOZPOOL_STATE_MISSING
finally:
# Set socket timeout back
socket.setdefaulttimeout(default_timeout)
mpc = MozpoolHandler("http://%s" % MOZPOOL_CNAME, log)
try:
result = mpc.query_device_state(device)
except MozpoolException as e:
log.error("Unable to get mozpool state, mozpool returned error: %s" %
sys.exc_info()[1])
return MOZPOOL_STATE_ERROR
if result['state'] == "ready":
log.debug("Mozpool state is 'ready'")
return MOZPOOL_STATE_READY
else:
log.error("Mozpool state is '%s'" % result['state'])
return MOZPOOL_STATE_UNKNOWN
def isMozpoolReady(device):
""" Checks if the mozpool server is available and the device is in 'ready' state
Returns MOZPOOL_STATE_READY if Mozpool is indeed listing device as ready
Returns MOZPOOL_STATE_UNKNOWN if Mozpool claims the device is in a non-ready state
Returns MOZPOOL_STATE_ERROR if Mozpool reports some other error.
Returns MOZPOOL_STATE_MISSING if there is no Mozpool server found in DNS.
"""
import socket
default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(5) # Don't let networking delay us too long
try:
socket.gethostbyname(MOZPOOL_CNAME)
except:
log.info("No mozpool server in this VLAN")
return MOZPOOL_STATE_MISSING
finally:
# Set socket timeout back
socket.setdefaulttimeout(default_timeout)
mpc = MozpoolHandler("http://%s" % MOZPOOL_CNAME, log)
try:
result = mpc.query_device_state(device)
except MozpoolException as e:
log.error("Unable to get mozpool state, mozpool returned error: %s" % sys.exc_info()[1])
return MOZPOOL_STATE_ERROR
if result['state'] == "ready":
log.debug("Mozpool state is 'ready'")
return MOZPOOL_STATE_READY
else:
log.error("Mozpool state is '%s'" % result['state'])
return MOZPOOL_STATE_UNKNOWN
def query_mozpool_handler(self, device=None, mozpool_api_url=None):
if self.mozpool_handler != None:
return self.mozpool_handler
else:
self.mozpool_api_url = self.determine_mozpool_host(
device) if device else mozpool_api_url
assert self.mozpool_api_url != None, \
"query_mozpool_handler() requires either a device or mozpool_api_url!"
site_packages_path = self.query_python_site_packages_path()
mph_path = os.path.join(site_packages_path, 'mozpoolclient')
sys.path.append(mph_path)
sys.path.append(site_packages_path)
try:
from mozpoolclient import MozpoolHandler, MozpoolException, MozpoolConflictException
self.MozpoolException = MozpoolException
self.MozpoolConflictException = MozpoolConflictException
self.mozpool_handler = MozpoolHandler(self.mozpool_api_url,
log_obj=self)
except ImportError, e:
self.fatal(
"Can't instantiate MozpoolHandler until mozpoolclient python "
"package is installed! (VirtualenvMixin?): \n%s" % str(e))
return self.mozpool_handler
def reboot(name, update_bug=True):
"""Attempts to reboot the named slave a series of ways, escalating from
peacefully to mercilessly. Details of what was attempted and the result
are reported into the slave's problem tracking bug at the end. Reboots
are attempted through the following means (from most peaceful to least
merciful):
* SSH: Logs into the machine via SSH and reboots it with an \
appropriate command.
* Mozpool: Connects to Mozpool and issues a reboot command. \
If the slave has no mozpool interface, this is skipped.
* IPMI: Uses the slave's IPMI interface to initiate a hard \
reboot. If the slave has no IPMI interface, this is skipped.
* PDU: Powercycles the slave by turning off the power, and then \
turning it back on.
* Bugzilla: Requests that IT reboot the slave by updating or creating \
the appropriate bugs.
"""
status_text = ""
slave = Slave(name)
slave.load_inventory_info()
slave.load_devices_info()
slave.load_ipmi_info()
slave.load_bug_info(createIfMissing=False)
status_text += "Attempting SSH reboot..."
alive = False
# If the slave is pingable, try an SSH reboot...
try:
if ping(slave.fqdn):
console = get_console(slave, usebuildbotslave=False)
if console:
# Sometimes the SSH session goes away before the command can
# successfully complete. In order to avoid misinterpreting that
# as some sort of other failure, we need to assume that it suceeds.
# wait_for_reboot will confirm that the slave goes down before
# coming back up, so this is OK to do.
try:
console.reboot()
except:
logException(log.warning, "Eating exception during SSH reboot.")
pass
alive = wait_for_reboot(slave)
except:
logException(log.error, "Caught exception during SSH reboot.")
# If there is a mozpool server associated
if not alive and slave.mozpool_server:
status_text += "Failed.\n"
status_text += "Attempting reboot via Mozpool..."
try:
mozpoolhandler = MozpoolHandler(slave.mozpool_server)
mozpoolhandler.device_power_cycle(slave.name, None)
alive = wait_for_reboot(slave)
except:
logException(log.error, "Caught exception during mozpool reboot.")
# If that doesn't work, maybe an IPMI reboot will...
if not alive and slave.ipmi:
status_text += "Failed.\n"
status_text += "Attempting IPMI reboot..."
try:
try:
slave.ipmi.powercycle()
except:
logException(log.warning, "Eating exception during IPMI reboot.")
pass
alive = wait_for_reboot(slave)
except:
logException(log.error, "Caught exception during IPMI reboot.")
# Mayhaps a PDU reboot?
if not alive and slave.pdu:
status_text += "Failed.\n"
status_text += "Attempting PDU reboot..."
try:
try:
slave.pdu.powercycle()
except:
logException(log.warning, "Eating exception during PDU reboot.")
pass
alive = wait_for_reboot(slave)
except:
logException(log.error, "Caught exception during PDU reboot.")
if alive:
# To minimize bugspam, no comment is added to the bug if we were
# able to bring it back up.
status_text += "Success!"
return SUCCESS, status_text
else:
status_text += "Failed.\n"
if slave.reboot_bug:
status_text += "Slave already has reboot bug (%s), nothing to do." % slave.reboot_bug.id_
return FAILURE, status_text
else:
if not slave.bug:
slave.load_bug_info(createIfMissing=True)
if update_bug:
slave.reboot_bug = file_reboot_bug(slave)
status_text += "Filed IT bug for reboot (bug %s)" % slave.reboot_bug.id_
data = {}
if not slave.bug.data["is_open"]:
data["status"] = "REOPENED"
slave.bug.add_comment(status_text, data=data)
return FAILURE, status_text