def isMozpoolReady(device): """ Checks if the mozpool server is available and the device is in 'ready' state Returns MOZPOOL_STATE_READY if Mozpool is indeed listing device as ready Returns MOZPOOL_STATE_UNKNOWN if Mozpool claims the device is in a non-ready state Returns MOZPOOL_STATE_ERROR if Mozpool reports some other error. Returns MOZPOOL_STATE_MISSING if there is no Mozpool server found in DNS. """ import socket default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(5) # Don't let networking delay us too long try: socket.gethostbyname(MOZPOOL_CNAME) except: log.info("No mozpool server in this VLAN") return MOZPOOL_STATE_MISSING finally: # Set socket timeout back socket.setdefaulttimeout(default_timeout) mpc = MozpoolHandler("http://%s" % MOZPOOL_CNAME, log) try: result = mpc.query_device_state(device) except MozpoolException as e: log.error("Unable to get mozpool state, mozpool returned error: %s" % sys.exc_info()[1]) return MOZPOOL_STATE_ERROR if result['state'] == "ready": log.debug("Mozpool state is 'ready'") return MOZPOOL_STATE_READY else: log.error("Mozpool state is '%s'" % result['state']) return MOZPOOL_STATE_UNKNOWN
def query_mozpool_handler(self, device=None, mozpool_api_url=None): if self.mozpool_handler != None: return self.mozpool_handler else: self.mozpool_api_url = self.determine_mozpool_host( device) if device else mozpool_api_url assert self.mozpool_api_url != None, \ "query_mozpool_handler() requires either a device or mozpool_api_url!" site_packages_path = self.query_python_site_packages_path() mph_path = os.path.join(site_packages_path, 'mozpoolclient') sys.path.append(mph_path) sys.path.append(site_packages_path) try: from mozpoolclient import MozpoolHandler, MozpoolException, MozpoolConflictException self.MozpoolException = MozpoolException self.MozpoolConflictException = MozpoolConflictException self.mozpool_handler = MozpoolHandler(self.mozpool_api_url, log_obj=self) except ImportError, e: self.fatal( "Can't instantiate MozpoolHandler until mozpoolclient python " "package is installed! (VirtualenvMixin?): \n%s" % str(e)) return self.mozpool_handler
def reboot(name, update_bug=True): """Attempts to reboot the named slave a series of ways, escalating from peacefully to mercilessly. Details of what was attempted and the result are reported into the slave's problem tracking bug at the end. Reboots are attempted through the following means (from most peaceful to least merciful): * SSH: Logs into the machine via SSH and reboots it with an \ appropriate command. * Mozpool: Connects to Mozpool and issues a reboot command. \ If the slave has no mozpool interface, this is skipped. * IPMI: Uses the slave's IPMI interface to initiate a hard \ reboot. If the slave has no IPMI interface, this is skipped. * PDU: Powercycles the slave by turning off the power, and then \ turning it back on. * Bugzilla: Requests that IT reboot the slave by updating or creating \ the appropriate bugs. """ status_text = "" slave = Slave(name) slave.load_inventory_info() slave.load_devices_info() slave.load_ipmi_info() slave.load_bug_info(createIfMissing=False) status_text += "Attempting SSH reboot..." alive = False # If the slave is pingable, try an SSH reboot... try: if ping(slave.fqdn): console = get_console(slave, usebuildbotslave=False) if console: # Sometimes the SSH session goes away before the command can # successfully complete. In order to avoid misinterpreting that # as some sort of other failure, we need to assume that it suceeds. # wait_for_reboot will confirm that the slave goes down before # coming back up, so this is OK to do. try: console.reboot() except: logException(log.warning, "Eating exception during SSH reboot.") pass alive = wait_for_reboot(slave) except: logException(log.error, "Caught exception during SSH reboot.") # If there is a mozpool server associated if not alive and slave.mozpool_server: status_text += "Failed.\n" status_text += "Attempting reboot via Mozpool..." try: mozpoolhandler = MozpoolHandler(slave.mozpool_server) mozpoolhandler.device_power_cycle(slave.name, None) alive = wait_for_reboot(slave) except: logException(log.error, "Caught exception during mozpool reboot.") # If that doesn't work, maybe an IPMI reboot will... if not alive and slave.ipmi: status_text += "Failed.\n" status_text += "Attempting IPMI reboot..." try: try: slave.ipmi.powercycle() except: logException(log.warning, "Eating exception during IPMI reboot.") pass alive = wait_for_reboot(slave) except: logException(log.error, "Caught exception during IPMI reboot.") # Mayhaps a PDU reboot? if not alive and slave.pdu: status_text += "Failed.\n" status_text += "Attempting PDU reboot..." try: try: slave.pdu.powercycle() except: logException(log.warning, "Eating exception during PDU reboot.") pass alive = wait_for_reboot(slave) except: logException(log.error, "Caught exception during PDU reboot.") if alive: # To minimize bugspam, no comment is added to the bug if we were # able to bring it back up. status_text += "Success!" return SUCCESS, status_text else: status_text += "Failed.\n" if slave.reboot_bug: status_text += "Slave already has reboot bug (%s), nothing to do." % slave.reboot_bug.id_ return FAILURE, status_text else: if not slave.bug: slave.load_bug_info(createIfMissing=True) if update_bug: slave.reboot_bug = file_reboot_bug(slave) status_text += "Filed IT bug for reboot (bug %s)" % slave.reboot_bug.id_ data = {} if not slave.bug.data["is_open"]: data["status"] = "REOPENED" slave.bug.add_comment(status_text, data=data) return FAILURE, status_text