def reconnect(self, after=10, timeout=None, nolog=False): """ reconnect to the host:: after how long to wait for the first attempt timeout how long to wait for each attempt """ # default to this instance's current value if timeout is None: timeout = self.timeout self.close() start = now() while 1: sleep(after) try: if self.connect(timeout=timeout, nolog=nolog): break except (TIMEOUT, EOF) as e: self.close() if not nolog: logger.debug(str(e)) logger.debug("reconnected after %s" % timefmt(since(start))) return True
def wrapper(*args, **kwargs): result = ReturnCode(not case) if timeout is None: while bool(result) != bool(case): result = function(*args, **kwargs) if bool(result) != case: # no need to sleep if case is met sleep(wrapper.sleeptime) else: starttime = now() while now() - starttime < float(wrapper.timeout): result = function(*args, **kwargs) if bool(result) == case: break sleep(wrapper.sleeptime) if bool(result) != case: result = ReturnCode(False) result.message = "Timed out : {0} seconds".format(timeout) return result
def _runcmd(self, status, message): """ this private method is executed as a thread :type status: c_int representing a bool :type message: c_array of char """ r = self.initiator.connect() if not r: logger.critical("connect failed ... enabling paramiko logging") import paramiko paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG) r = self.initiator.connect() if not r: raise ConnectionError("Failed to connect %s" % r.message) monitortime = False if 'runtime' in self.config: config = self.config.split() monitortime = 0 for param in config: if 'runtime' in param or 'ramp_time' in param: monitortime += int(param.split('=')[1]) monitortime += 120 logger.critical("timeout set to %s" % monitortime) cmd = "%s fio --output-format=json %s | gzip" % (self.envvars, self.config) self.started = True start = now() try: if monitortime: result = self.initiator.run_and_check(cmd, monitortime) else: result = self.initiator.run_and_check(cmd) try: logger.critical(pformat("".join( gzip.GzipFile('', 'r', 0, cStringIO.StringIO(result.raw.stdout)).read()))) except IOError as e: logger.critical("couldn't write out result: %s %s", (e, result)) except InitiatorError, e: result = ReturnCode(False, message=str(e))
only when above case is met or timeout is exceeded. This will not work for a generator function. """ def waiter(function): @functools.wraps(function) def wrapper(*args, **kwargs): result = ReturnCode(not case) if timeout is None: while bool(result) != bool(case): result = function(*args, **kwargs) if bool(result) != case: # no need to sleep if case is met sleep(wrapper.sleeptime) else: starttime = now() while now() - starttime < float(wrapper.timeout): result = function(*args, **kwargs) if bool(result) == case: break sleep(wrapper.sleeptime) if bool(result) != case: result = ReturnCode(False) result.message = "Timed out : {0} seconds".format(timeout) return result wrapper.case = case wrapper.timeout = timeout wrapper.sleeptime = sleeptime return wrapper
def zpool_create(self, pname, targets, ptype='', num_devices=1, spares=None, timeout=600, expectation=True): """ This method will create a zpool of name pname; working on support for multiple vdevs :param pname: zpool name :param targets: a list of SRX targets in the form shelf.lun :param ptype: a type for the zpool to create, like raidz2, mirror, etc :param num_devices: number of devices to use per vdev :param spares: is a list of targets that will be used as spare for the zpool :param expectation: If the caller cares about failure and the command fails we raise a generic exception. Returns a ReturnCode object """ pools_list = self.zpool_list # get the list of current existing pools addlist = targets if pools_list.get(pname): # pool already exist and we need to add devices while addlist: r = self.zpool_add(pname, addlist[:num_devices], ptype, expectation) if r: addlist = addlist[num_devices:] elif not expectation: return ReturnCode(False, 'An error was detected while creating pool: %s' % r.message) else: # pool does not exist, we need to create it first. r = self._zpool_create(pname, addlist[:num_devices], ptype, spares) if expectation and not r: raise InitiatorError("%s" % r) addlist = addlist[num_devices:] while addlist: r = self.zpool_add(pname, addlist[:num_devices], ptype, expectation) if r: addlist = addlist[num_devices:] elif not expectation: return ReturnCode(False, 'An error was detected while creating pool:\n %s' % r.message) else: raise InitiatorError("%s" % r) start = now() deadline = start + timeout # This should probably become otto.lib.solaris.target_in_zpool waitlist = [self._target2device(target) for target in targets] while waitlist: for pool in self.zpool_status(): if pool['pool'] == pname: for target in waitlist: if target in pool['config'].keys(): waitlist.remove(target) if now() > deadline: raise InitiatorError("missed deadline: %s timed out %ssec waiting on:\n%s" % (deadline, now() - start, waitlist)) for pool in self.zpool_status(): if pool['pool'] == pname: return ReturnCode(True, message=pool) return ReturnCode(False, "Couldn't find pool after creation")
def zpool_create(self, pname, targets, ptype='', num_devices=1, spares=None, timeout=600, expectation=True): """ This method will create a zpool of name pname; working on support for multiple vdevs :param pname: zpool name :param targets: a list of SRX targets in the form shelf.lun :param ptype: a type for the zpool to create, like raidz2, mirror, etc :param num_devices: number of devices to use per vdev :param spares: is a list of targets that will be used as spare for the zpool :param expectation: If the caller cares about failure and the command fails we raise a generic exception. Returns a ReturnCode object """ pools_list = self.zpool_list # get the list of current existing pools addlist = targets if pools_list.get( pname): # pool already exist and we need to add devices while addlist: r = self.zpool_add(pname, addlist[:num_devices], ptype, expectation) if r: addlist = addlist[num_devices:] elif not expectation: return ReturnCode( False, 'An error was detected while creating pool: %s' % r.message) else: # pool does not exist, we need to create it first. r = self._zpool_create(pname, addlist[:num_devices], ptype, spares) if expectation and not r: raise InitiatorError("%s" % r) addlist = addlist[num_devices:] while addlist: r = self.zpool_add(pname, addlist[:num_devices], ptype, expectation) if r: addlist = addlist[num_devices:] elif not expectation: return ReturnCode( False, 'An error was detected while creating pool:\n %s' % r.message) else: raise InitiatorError("%s" % r) start = now() deadline = start + timeout # This should probably become otto.lib.solaris.target_in_zpool waitlist = [self._target2device(target) for target in targets] while waitlist: for pool in self.zpool_status(): if pool['pool'] == pname: for target in waitlist: if target in pool['config'].keys(): waitlist.remove(target) if now() > deadline: raise InitiatorError( "missed deadline: %s timed out %ssec waiting on:\n%s" % (deadline, now() - start, waitlist)) for pool in self.zpool_status(): if pool['pool'] == pname: return ReturnCode(True, message=pool) return ReturnCode(False, "Couldn't find pool after creation")