class WriterWrapper: """CSV writer wrapper class to escape the special strings.""" def __init__(self, stream, *args, **kwargs): self._stream = stream self._queue = StringIO() self._writer = csv.writer(self._queue, *args, **kwargs) def writerow(self, row): """Take a row and write it into the stream with escaping the terminator. """ self._writer.writerow(row) data = self._queue.getvalue() if data == '\\.\r\n': data = '"\\."\r\n' self._stream.write(data) self._queue.seek(0) self._queue.truncate(0) def writerows(self, rows): """Take a rows and write them into the stream with escaping the terminator. """ for row in rows: self.writerow(row)
class OutputConsole(TextEdit): def __init__(self, parent=None): super(OutputConsole, self).__init__(parent) self.stdin = StringIO() self.setReadOnly(True) font = QtGui.QFont('courier', 9) self.setFont(font) pal = QtGui.QPalette() bgc = QtGui.QColor(50, 50, 50) pal.setColor(QtGui.QPalette.Base, bgc) text_color = QtGui.QColor(175, 175, 175) pal.setColor(QtGui.QPalette.Text, text_color) self.setPalette(pal) def read_stdin(self): value = self.stdin.getvalue() value = value.replace("\0", "") self.document().setPlainText(value) self.verticalScrollBar().setValue(self.verticalScrollBar().maximum()) def clear(self): self.stdin.truncate(0) self.document().setPlainText("") def createStandardContextMenu(self): menu = super(TextEdit, self).createStandardContextMenu() menu.addSeparator() menu.addAction("Clear...", self.clear) return menu def contextMenuEvent(self, event): menu = self.createStandardContextMenu() menu.exec_(self.mapToGlobal(event.pos()))
def parse(text, encoding='utf8'): """Parse the querystring into a normalized form.""" # Initialize the query object. query = Query() # Decode the text if we got bytes. if isinstance(text, six.binary_type): text = text.decode(encoding) # Iterate through the characters in the query string; one-by-one # in order to perform one-pass parsing. stream = StringIO() for character in text: # We want to stop reading the query and pass it off to someone # when we reach a logical or grouping operator. if character in (constants.LOGICAL_AND, constants.LOGICAL_OR): if not stream.tell(): # There is no content in the stream; a logical operator # was found out of place. raise ValueError('Found `{}` out of place'.format( character)) # Parse the segment up till the combinator segment = parse_segment(stream.getvalue(), character) query.segments.append(segment) stream.truncate(0) stream.seek(0) else: # This isn't a special character, just roll with it. stream.write(character) # TODO: Throw some nonsense here if the query string ended with a # & or ;, because that makes no sense. if stream.tell(): # Append the remainder of the query string. query.segments.append(parse_segment(stream.getvalue())) # Return the constructed query object. return query
def _send(self): """ Send a message lazy formatted with args. External log attributes can be passed via named attribute `extra`, like in logging from the standart library. Note: * Attrs must be dict, otherwise the whole message would be skipped. * The key field in an attr is converted to string. * The value is sent as is if isinstance of (str, unicode, int, float, long, bool), otherwise we convert the value to string. """ buff = BytesIO() while True: msgs = list() try: msg = yield self.queue.get() # we need to connect first, as we issue verbosity request just after connection # and channels should strictly go in ascending order if not self._connected: yield self.connect() try: while True: msgs.append(msg) counter = next(self.counter) msgpack_pack([counter, EMIT, msg], buff) msg = self.queue.get_nowait() except queues.QueueEmpty: pass try: yield self.pipe.write(buff.getvalue()) except Exception: pass # clean the buffer or we will end up without memory buff.truncate(0) except Exception: for message in msgs: self._log_to_fallback(message)
def parse_segment(text, combinator=constants.LOGICAL_AND): # Initialize a query segment. segment = QuerySegment() # Construct an iterator over the segment text. iterator = iter(text) stream = StringIO() # Iterate through the characters in the segment; one-by-one # in order to perform one-pass parsing. for character in iterator: if (character == constants.NEGATION[1] and not stream.tell() and not segment.path): # We've been negated. segment.negated = not segment.negated continue if character in OPERATOR_BEGIN_CHARS: # Found an operator; pull out what we can. iterator = _parse_operator(segment, chain(character, iterator)) # We're done here; go to the value parser break if character == constants.SEP_PATH: # A path separator, push the current stack into the path segment.path.append(stream.getvalue()) stream.truncate(0) stream.seek(0) # Keep checking for more path segments. continue # Append the text to the stream stream.write(character) # Write any remaining information into the path. segment.path.append(stream.getvalue()) # Attempt to normalize the path. try: # The keyword 'not' can be the last item which # negates this query. if segment.path[-1] == constants.NEGATION[0]: segment.negated = not segment.negated segment.path.pop(-1) # The last keyword can explicitly state the operation; in which # case the operator symbol **must** be `=`. if segment.path[-1] in OPERATOR_KEYWORDS: if segment.operator != constants.OPERATOR_IEQUAL[0]: raise ValueError( 'Explicit operations must use the `=` symbol.') segment.operator = segment.path.pop(-1) # Make sure we still have a path left. if not segment.path: raise IndexError() except IndexError: # Ran out of path items after removing operations and negation. raise ValueError('No path specified in {}'.format(text)) # Values are not complicated (yet) so just slice and dice # until we get a list of possible values. segment.values = ''.join(iterator) if segment.values: segment.values = segment.values.split(constants.SEP_VALUE) # Set the combinator. segment.combinator = COMBINATORS[combinator] # Return the constructed query segment. return segment
class PMRequestListener(object): """ Daemon process that responds to requests """ def __init__(self, config, buildroot): self.config = config self.buildroot = buildroot self.rundir = buildroot.make_chroot_path(RUNDIR) self.socket_path = os.path.join(self.rundir, SOCKET_NAME) self.executed_commands = [] # util.do cannot return output when the command fails, we need to # capture it's logging self.log_buffer = StringIO() self.log = logging.getLogger("mockbuild.plugin.pm_request") self.log.level = logging.DEBUG self.log.addFilter(OutputFilter()) self.log.propagate = False self.log.addHandler(logging.StreamHandler(self.log_buffer)) def prepare_socket(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.connect(self.socket_path) except (socket.error, OSError): try: os.unlink(self.socket_path) except OSError: pass else: # there's another process listening sys.exit(0) util.mkdirIfAbsent(self.rundir) # Don't allow regular users to access the socket as they may not be in # the mock group os.chown(self.rundir, self.buildroot.chrootuid, self.buildroot.chrootgid) os.chmod(self.rundir, 0o770) sock.bind(self.socket_path) os.chown(self.socket_path, self.buildroot.chrootuid, self.buildroot.chrootgid) return sock def listen(self): sock = self.prepare_socket() sock.listen(MAX_CONNECTIONS) while True: try: connection, _ = sock.accept() try: line = connection.makefile().readline() command = shlex.split(line) # pylint:disable=E1101 if command == ["!LOG_EXECUTED"]: connection.sendall('\n'.join( self.executed_commands).encode()) elif command: success, out = self.execute_command(command) connection.sendall(b"ok\n" if success else b"nok\n") connection.sendall(out.encode()) if success: self.executed_commands.append(line.strip()) finally: connection.close() except socket.error: continue def execute_command(self, command): try: self.buildroot.pkg_manager.execute(*command, printOutput=False, logger=self.log, returnOutput=False, pty=False, raiseExc=True) success = True except Error: success = False out = self.log_buffer.getvalue() self.log_buffer.seek(0) self.log_buffer.truncate() return success, out
class Daemon: IFACE = "127.0.0.1" def __init__(self, ssl=None, **daemonargs): self.q = queue.Queue() self.logfp = StringIO() daemonargs["logfp"] = self.logfp self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs) self.thread.start() self.port = self.q.get(True, 5) self.urlbase = "%s://%s:%s" % ( "https" if ssl else "http", self.IFACE, self.port ) def __enter__(self): return self def __exit__(self, type, value, traceback): self.logfp.truncate(0) self.shutdown() return False def p(self, spec): """ Return a URL that will render the response in spec. """ return "%s/p/%s" % (self.urlbase, spec) def text_log(self): return self.logfp.getvalue() def wait_for_silence(self, timeout=5): self.thread.server.wait_for_silence(timeout=timeout) def expect_log(self, n, timeout=5): l = [] start = time.time() while True: l = self.log() if time.time() - start >= timeout: return None if len(l) >= n: break return l def last_log(self): """ Returns the last logged request, or None. """ l = self.expect_log(1) if not l: return None return l[-1] def log(self): """ Return the log buffer as a list of dictionaries. """ return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ return self.thread.server.clear_log() def shutdown(self): """ Shut the daemon down, return after the thread has exited. """ self.thread.server.shutdown() self.thread.join()
class Daemon: IFACE = "127.0.0.1" def __init__(self, ssl=None, **daemonargs): self.q = queue.Queue() self.logfp = StringIO() daemonargs["logfp"] = self.logfp self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs) self.thread.start() self.port = self.q.get(True, 5) self.urlbase = "%s://%s:%s" % ( "https" if ssl else "http", self.IFACE, self.port ) def __enter__(self): return self def __exit__(self, type, value, traceback): self.logfp.truncate(0) self.shutdown() return False def p(self, spec): """ Return a URL that will render the response in spec. """ return "%s/p/%s" % (self.urlbase, spec) def info(self): """ Return some basic info about the remote daemon. """ resp = requests.get("%s/api/info" % self.urlbase, verify=False) return resp.json() def text_log(self): return self.logfp.getvalue() def last_log(self): """ Returns the last logged request, or None. """ l = self.log() if not l: return None return l[0] def log(self): """ Return the log buffer as a list of dictionaries. """ resp = requests.get("%s/api/log" % self.urlbase, verify=False) return resp.json()["log"] def clear_log(self): """ Clear the log. """ self.logfp.truncate(0) resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) return resp.ok def shutdown(self): """ Shut the daemon down, return after the thread has exited. """ self.thread.server.shutdown() self.thread.join()
class Daemon: IFACE = "127.0.0.1" def __init__(self, ssl=None, **daemonargs): self.q = queue.Queue() self.logfp = StringIO() daemonargs["logfp"] = self.logfp self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs) self.thread.start() self.port = self.q.get(True, 5) self.urlbase = "%s://%s:%s" % ("https" if ssl else "http", self.IFACE, self.port) def __enter__(self): return self def __exit__(self, type, value, traceback): self.logfp.truncate(0) self.shutdown() return False def p(self, spec): """ Return a URL that will render the response in spec. """ return "%s/p/%s" % (self.urlbase, spec) def text_log(self): return self.logfp.getvalue() def wait_for_silence(self, timeout=5): start = time.time() while 1: if time.time() - start >= timeout: raise TimeoutError("%s service threads still alive" % self.thread.server.handler_counter.count) if self.thread.server.handler_counter.count == 0: return def expect_log(self, n, timeout=5): l = [] start = time.time() while True: l = self.log() if time.time() - start >= timeout: return None if len(l) >= n: break return l def last_log(self): """ Returns the last logged request, or None. """ l = self.expect_log(1) if not l: return None return l[-1] def log(self): """ Return the log buffer as a list of dictionaries. """ return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ return self.thread.server.clear_log() def shutdown(self): """ Shut the daemon down, return after the thread has exited. """ self.thread.server.shutdown() self.thread.join()
class Daemon: IFACE = "127.0.0.1" def __init__(self, ssl=None, **daemonargs): self.q = queue.Queue() self.logfp = StringIO() daemonargs["logfp"] = self.logfp self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs) self.thread.start() self.port = self.q.get(True, 5) self.urlbase = "%s://%s:%s" % ("https" if ssl else "http", self.IFACE, self.port) def __enter__(self): return self def __exit__(self, type, value, traceback): self.logfp.truncate(0) self.shutdown() return False def p(self, spec): """ Return a URL that will render the response in spec. """ return "%s/p/%s" % (self.urlbase, spec) def info(self): """ Return some basic info about the remote daemon. """ resp = requests.get("%s/api/info" % self.urlbase, verify=False) return resp.json() def text_log(self): return self.logfp.getvalue() def last_log(self): """ Returns the last logged request, or None. """ l = self.log() if not l: return None return l[0] def log(self): """ Return the log buffer as a list of dictionaries. """ resp = requests.get("%s/api/log" % self.urlbase, verify=False) return resp.json()["log"] def clear_log(self): """ Clear the log. """ self.logfp.truncate(0) resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) return resp.ok def shutdown(self): """ Shut the daemon down, return after the thread has exited. """ self.thread.server.shutdown() self.thread.join()
class PMRequestListener(object): """ Daemon process that responds to requests """ def __init__(self, config, buildroot): self.config = config self.buildroot = buildroot self.rundir = buildroot.make_chroot_path(RUNDIR) self.socket_path = os.path.join(self.rundir, SOCKET_NAME) self.executed_commands = [] # util.do cannot return output when the command fails, we need to # capture it's logging self.log_buffer = StringIO() self.log = logging.getLogger("mockbuild.plugin.pm_request") self.log.level = logging.DEBUG self.log.addFilter(OutputFilter()) self.log.propagate = False self.log.addHandler(logging.StreamHandler(self.log_buffer)) def prepare_socket(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.connect(self.socket_path) except (socket.error, OSError): try: os.unlink(self.socket_path) except OSError: pass else: # there's another process listening sys.exit(0) util.mkdirIfAbsent(self.rundir) # Don't allow regular users to access the socket as they may not be in # the mock group os.chown(self.rundir, self.buildroot.chrootuid, self.buildroot.chrootgid) os.chmod(self.rundir, 0o770) sock.bind(self.socket_path) os.chown(self.socket_path, self.buildroot.chrootuid, self.buildroot.chrootgid) return sock def listen(self): sock = self.prepare_socket() sock.listen(MAX_CONNECTIONS) while True: try: connection, _ = sock.accept() try: line = connection.makefile().readline() command = shlex.split(line) # pylint:disable=E1101 if command == ["!LOG_EXECUTED"]: connection.sendall('\n'.join(self.executed_commands).encode()) elif command: success, out = self.execute_command(command) connection.sendall(b"ok\n" if success else b"nok\n") connection.sendall(out.encode()) if success: self.executed_commands.append(line.strip()) finally: connection.close() except socket.error: continue def execute_command(self, command): try: self.buildroot.pkg_manager.execute( *command, printOutput=False, logger=self.log, returnOutput=False, pty=False, raiseExc=True) success = True except Error: success = False out = self.log_buffer.getvalue() self.log_buffer.seek(0) self.log_buffer.truncate() return success, out