class QemuImgOperation(object): REGEXPR = re.compile(r'\s*\(([\d.]+)/100%\)\s*') def __init__(self, cmd, cwd=None): self._lock = threading.Lock() self._aborted = False self._progress = 0.0 self._stdout = bytearray() self._stderr = bytearray() self.cmd = wrap_command(cmd, with_nice=utils.NICENESS.HIGH, with_ioclass=utils.IOCLASS.IDLE) _log.debug(cmdutils.command_log_line(self.cmd, cwd=cwd)) self._command = CPopen(self.cmd, cwd=cwd, deathSignal=signal.SIGKILL) self._stream = utils.CommandStream(self._command, self._recvstdout, self._recvstderr) def _recvstderr(self, buffer): self._stderr += buffer def _recvstdout(self, buffer): self._stdout += buffer # Checking the presence of '\r' before splitting will prevent # generating the array when it's not needed. try: idx = self._stdout.rindex('\r') except ValueError: return # qemu-img updates progress by printing \r (0.00/100%) to standard out. # The output could end with a partial progress so we must discard # everything after the last \r and then try to parse a progress record. valid_progress = self._stdout[:idx] last_progress = valid_progress.rsplit('\r', 1)[-1] # No need to keep old progress information around del self._stdout[:idx + 1] m = self.REGEXPR.match(last_progress) if m is None: raise ValueError('Unable to parse: "%r"' % last_progress) self._progress = float(m.group(1)) @property def progress(self): """ Returns operation progress as float between 0 and 100. This method is threadsafe and may be called from any thread. """ return self._progress @property def error(self): return str(self._stderr) @property def finished(self): return self._command.poll() is not None def poll(self, timeout=None): self._stream.receive(timeout=timeout) if not self._stream.closed: return self._command.wait() if self._aborted: raise exception.ActionStopped() cmdutils.retcode_log_line(self._command.returncode, self.error) if self._command.returncode != 0: raise QImgError(self.cmd, self._command.returncode, "", self.error) def wait_for_completion(self): timeout = config.getint("irs", "progress_interval") while not self.finished: self.poll(timeout) _log.debug('qemu-img operation progress: %s%%', self.progress) def abort(self): """ Aborts running operation by sending a termination signal to the underlying qemu-img process. Note: this is asynchronous operation, returning before the process was terminated. You must use wait_for_completion to wait for the underlying qemu-img process. This method is threadsafe and may be called from any thread. """ with self._lock: if self._command is None: return if self._command.poll() is None: self._aborted = True self._command.terminate() def close(self): with self._lock: self._stream.close() self._command = None
class DirectioChecker(object): """ Check path availability using direct I/O. DirectioChecker is created with a complete callback. Each time a check cycle is completed, the complete callback will be invoked with a CheckResult instance. CheckResult provides a delay() method returning the read delay in seconds. If the check failed, the delay() method will raise the appropriate exception that can be reported to engine. Note that the complete callback must not block as it will block the entire event loop thread. The checker runs exactly every interval seconds. If a check did not complete before the next check is scheduled, the next check will be delayed to the next interval. Checker is not thread safe. Use EventLoop.call_soon_threadsafe() to start or stop a checker. The only thread safe method is wait(). Usage:: # Start the event loop thread loop = asyncevent.EventLoop() concurrent.thread(loop.run_forever).start() # The complete callback def complete(result): try: check_delay = result.delay() except Exception as e: check_error = e check_time = time.time() # Start a checker on the event loop thread checker = DirectioChecker(loop, path, complete) loop.call_soon_threadsafe(checker.start) ... # Stop a checker from another thread loop.call_soon_threadsafe(checker.stop) # If needed, wait until a checker actually stopped. checker.wait(30) """ log = logging.getLogger("storage.directiochecker") def __init__(self, loop, path, complete, interval=10.0): self._loop = loop self._path = path self._complete = complete self._interval = interval self._looper = asyncutils.LoopingCall(loop, self._check) self._check_time = None self._proc = None self._reader = None self._reaper = None self._err = None self._state = IDLE self._stopped = threading.Event() def start(self): """ Start the checker. Raises RuntimeError if the checker is running. """ if self._state is not IDLE: raise RuntimeError("Checker is %s", self._state) self._state = RUNNING _log.debug("Checker %r started", self._path) self._stopped.clear() self._looper.start(self._interval) def stop(self): """ Stop the checker. If the checker is waiting for the next check, the next check will be cancelled. If the checker is in the middle of a check, it will stop when the check completes. If the checker is not running, the call is ignored silently. """ if self._state is not RUNNING: return _log.debug("Checker %r stopping", self._path) self._state = STOPPING self._looper.stop() if self._proc is None: self._stop_completed() def wait(self, timeout=None): """ Wait until a checker has stopped. Returns True if checker has stopped, False if timeout expired. """ return self._stopped.wait(timeout) def is_running(self): return self._state is not IDLE def _stop_completed(self): self._state = IDLE _log.debug("Checker %r stopped", self._path) self._stopped.set() def _check(self): """ Called when starting the checker, and then every interval seconds until the checker is stopped. """ assert self._state is RUNNING if self._proc: _log.warning("Checker %r is blocked for %.2f seconds", self._path, self._loop.time() - self._check_time) return self._check_time = self._loop.time() _log.debug("START check %r (delay=%.2f)", self._path, self._check_time - self._looper.deadline) try: self._start_process() except Exception as e: self._err = "Error starting process: %s" % e self._check_completed(EXEC_ERROR) def _start_process(self): """ Starts a dd process performing direct I/O to path, reading the process stderr. When stderr has closed, _read_completed will be called. """ cmd = [constants.EXT_DD, "if=%s" % self._path, "of=/dev/null", "bs=4096", "count=1", "iflag=direct"] cmd = cmdutils.wrap_command(cmd) self._proc = CPopen(cmd, stdin=None, stdout=None, stderr=subprocess.PIPE) self._reader = self._loop.create_dispatcher( asyncevent.BufferedReader, self._proc.stderr, self._read_completed) def _read_completed(self, data): """ Called when dd process has closed stderr. At this point the process may be still running. """ assert self._state is not IDLE self._reader = None self._err = data rc = self._proc.poll() # About 95% of runs, the process has terminated at this point. If not, # start the reaper to wait for it. if rc is None: self._reaper = asyncevent.Reaper(self._loop, self._proc, self._check_completed) return self._check_completed(rc) def _check_completed(self, rc): """ Called when the dd process has exited with exit code rc. """ assert self._state is not IDLE now = self._loop.time() elapsed = now - self._check_time _log.debug("FINISH check %r (rc=%s, elapsed=%.02f)", self._path, rc, elapsed) self._reaper = None self._proc = None if self._state is STOPPING: self._stop_completed() return result = CheckResult(self._path, rc, self._err, self._check_time, elapsed) self._complete(result) def __repr__(self): info = [self.__class__.__name__, self._path, self._state] if self._state is RUNNING: info.append("next_check=%.2f" % self._looper.deadline) return "<%s at 0x%x>" % (" ".join(info), id(self))
class DirectioChecker(object): """ Check path availability using direct I/O. DirectioChecker is created with a complete callback. Each time a check cycle is completed, the complete callback will be invoked with a CheckResult instance. CheckResult provides a delay() method returning the read delay in seconds. If the check failed, the delay() method will raise the appropriate exception that can be reported to engine. Note that the complete callback must not block as it will block the entire event loop thread. The checker runs exactly every interval seconds. If a check did not complete before the next check is scheduled, the next check will be delayed to the next interval. Checker is not thread safe. Use EventLoop.call_soon_threadsafe() to start or stop a checker. The only thread safe method is wait(). Usage:: # Start the event loop thread loop = asyncevent.EventLoop() concurrent.thread(loop.run_forever).start() # The complete callback def complete(result): try: check_delay = result.delay() except Exception as e: check_error = e check_time = time.time() # Start a checker on the event loop thread checker = DirectioChecker(loop, path, complete) loop.call_soon_threadsafe(checker.start) ... # Stop a checker from another thread loop.call_soon_threadsafe(checker.stop) # If needed, wait until a checker actually stopped. checker.wait(30) """ log = logging.getLogger("storage.directiochecker") def __init__(self, loop, path, complete, interval=10.0): self._loop = loop self._path = path self._complete = complete self._interval = interval self._looper = asyncutils.LoopingCall(loop, self._check) self._check_time = None self._proc = None self._reader = None self._reaper = None self._err = None self._state = IDLE self._stopped = threading.Event() def start(self): """ Start the checker. Raises RuntimeError if the checker is running. """ if self._state is not IDLE: raise RuntimeError("Checker is %s", self._state) self._state = RUNNING _log.debug("Checker %r started", self._path) self._stopped.clear() self._looper.start(self._interval) def stop(self): """ Stop the checker. If the checker is waiting for the next check, the next check will be cancelled. If the checker is in the middle of a check, it will stop when the check completes. If the checker is not running, the call is ignored silently. """ if self._state is not RUNNING: return _log.debug("Checker %r stopping", self._path) self._state = STOPPING self._looper.stop() if self._proc is None: self._stop_completed() def wait(self, timeout=None): """ Wait until a checker has stopped. Returns True if checker has stopped, False if timeout expired. """ return self._stopped.wait(timeout) def is_running(self): return self._state is not IDLE def _stop_completed(self): self._state = IDLE _log.debug("Checker %r stopped", self._path) self._stopped.set() def _check(self): """ Called when starting the checker, and then every interval seconds until the checker is stopped. """ assert self._state is RUNNING if self._proc: _log.warning("Checker %r is blocked for %.2f seconds", self._path, self._loop.time() - self._check_time) return self._check_time = self._loop.time() _log.debug("START check %r (delay=%.2f)", self._path, self._check_time - self._looper.deadline) try: self._start_process() except Exception as e: self._err = "Error starting process: %s" % e self._check_completed(EXEC_ERROR) def _start_process(self): """ Starts a dd process performing direct I/O to path, reading the process stderr. When stderr has closed, _read_completed will be called. """ cmd = [ constants.EXT_DD, "if=%s" % self._path, "of=/dev/null", "bs=4096", "count=1", "iflag=direct" ] cmd = cmdutils.wrap_command(cmd) self._proc = CPopen(cmd, stdin=None, stdout=None, stderr=subprocess.PIPE) self._reader = self._loop.create_dispatcher(asyncevent.BufferedReader, self._proc.stderr, self._read_completed) def _read_completed(self, data): """ Called when dd process has closed stderr. At this point the process may be still running. """ assert self._state is not IDLE self._reader = None self._err = data rc = self._proc.poll() # About 95% of runs, the process has terminated at this point. If not, # start the reaper to wait for it. if rc is None: self._reaper = asyncevent.Reaper(self._loop, self._proc, self._check_completed) return self._check_completed(rc) def _check_completed(self, rc): """ Called when the dd process has exited with exit code rc. """ assert self._state is not IDLE now = self._loop.time() elapsed = now - self._check_time _log.debug("FINISH check %r (rc=%s, elapsed=%.02f)", self._path, rc, elapsed) self._reaper = None self._proc = None if self._state is STOPPING: self._stop_completed() return result = CheckResult(self._path, rc, self._err, self._check_time, elapsed) self._complete(result) def __repr__(self): info = [self.__class__.__name__, self._path, self._state] if self._state is RUNNING: info.append("next_check=%.2f" % self._looper.deadline) return "<%s at 0x%x>" % (" ".join(info), id(self))
class QemuImgOperation(object): REGEXPR = re.compile(r'\s*\(([\d.]+)/100%\)\s*') def __init__(self, cmd, cwd=None): self._lock = threading.Lock() self._aborted = False self._progress = 0.0 self._stdout = bytearray() self._stderr = bytearray() self.cmd = wrap_command( cmd, with_nice=utils.NICENESS.HIGH, with_ioclass=utils.IOCLASS.IDLE) _log.debug(cmdutils.command_log_line(self.cmd, cwd=cwd)) self._command = CPopen(self.cmd, cwd=cwd, deathSignal=signal.SIGKILL) self._stream = utils.CommandStream( self._command, self._recvstdout, self._recvstderr) def _recvstderr(self, buffer): self._stderr += buffer def _recvstdout(self, buffer): self._stdout += buffer # Checking the presence of '\r' before splitting will prevent # generating the array when it's not needed. try: idx = self._stdout.rindex('\r') except ValueError: return # qemu-img updates progress by printing \r (0.00/100%) to standard out. # The output could end with a partial progress so we must discard # everything after the last \r and then try to parse a progress record. valid_progress = self._stdout[:idx] last_progress = valid_progress.rsplit('\r', 1)[-1] # No need to keep old progress information around del self._stdout[:idx + 1] m = self.REGEXPR.match(last_progress) if m is None: raise ValueError('Unable to parse: "%r"' % last_progress) self._progress = float(m.group(1)) @property def progress(self): """ Returns operation progress as float between 0 and 100. This method is threadsafe and may be called from any thread. """ return self._progress @property def error(self): return str(self._stderr) @property def finished(self): return self._command.poll() is not None def poll(self, timeout=None): self._stream.receive(timeout=timeout) if not self._stream.closed: return self._command.wait() if self._aborted: raise exception.ActionStopped() cmdutils.retcode_log_line(self._command.returncode, self.error) if self._command.returncode != 0: raise QImgError(self.cmd, self._command.returncode, "", self.error) def wait_for_completion(self): timeout = config.getint("irs", "progress_interval") while not self.finished: self.poll(timeout) _log.debug('qemu-img operation progress: %s%%', self.progress) def abort(self): """ Aborts running operation by sending a termination signal to the underlying qemu-img process. Note: this is asynchronous operation, returning before the process was terminated. You must use wait_for_completion to wait for the underlying qemu-img process. This method is threadsafe and may be called from any thread. """ with self._lock: if self._command is None: return if self._command.poll() is None: self._aborted = True self._command.terminate() def close(self): with self._lock: self._stream.close() self._command = None