class WriterWrapper:
"""CSV writer wrapper class to escape the special strings."""
def __init__(self, stream, *args, **kwargs):
self._stream = stream
self._queue = StringIO()
self._writer = csv.writer(self._queue, *args, **kwargs)
def writerow(self, row):
"""Take a row and write it into the stream
with escaping the terminator.
"""
self._writer.writerow(row)
data = self._queue.getvalue()
if data == '\\.\r\n':
data = '"\\."\r\n'
self._stream.write(data)
self._queue.seek(0)
self._queue.truncate(0)
def writerows(self, rows):
"""Take a rows and write them into the stream
with escaping the terminator.
"""
for row in rows:
self.writerow(row)
class OutputConsole(TextEdit):
def __init__(self, parent=None):
super(OutputConsole, self).__init__(parent)
self.stdin = StringIO()
self.setReadOnly(True)
font = QtGui.QFont('courier', 9)
self.setFont(font)
pal = QtGui.QPalette()
bgc = QtGui.QColor(50, 50, 50)
pal.setColor(QtGui.QPalette.Base, bgc)
text_color = QtGui.QColor(175, 175, 175)
pal.setColor(QtGui.QPalette.Text, text_color)
self.setPalette(pal)
def read_stdin(self):
value = self.stdin.getvalue()
value = value.replace("\0", "")
self.document().setPlainText(value)
self.verticalScrollBar().setValue(self.verticalScrollBar().maximum())
def clear(self):
self.stdin.truncate(0)
self.document().setPlainText("")
def createStandardContextMenu(self):
menu = super(TextEdit, self).createStandardContextMenu()
menu.addSeparator()
menu.addAction("Clear...", self.clear)
return menu
def contextMenuEvent(self, event):
menu = self.createStandardContextMenu()
menu.exec_(self.mapToGlobal(event.pos()))
def parse(text, encoding='utf8'):
"""Parse the querystring into a normalized form."""
# Initialize the query object.
query = Query()
# Decode the text if we got bytes.
if isinstance(text, six.binary_type):
text = text.decode(encoding)
# Iterate through the characters in the query string; one-by-one
# in order to perform one-pass parsing.
stream = StringIO()
for character in text:
# We want to stop reading the query and pass it off to someone
# when we reach a logical or grouping operator.
if character in (constants.LOGICAL_AND, constants.LOGICAL_OR):
if not stream.tell():
# There is no content in the stream; a logical operator
# was found out of place.
raise ValueError('Found `{}` out of place'.format(
character))
# Parse the segment up till the combinator
segment = parse_segment(stream.getvalue(), character)
query.segments.append(segment)
stream.truncate(0)
stream.seek(0)
else:
# This isn't a special character, just roll with it.
stream.write(character)
# TODO: Throw some nonsense here if the query string ended with a
# & or ;, because that makes no sense.
if stream.tell():
# Append the remainder of the query string.
query.segments.append(parse_segment(stream.getvalue()))
# Return the constructed query object.
return query
def _send(self):
""" Send a message lazy formatted with args.
External log attributes can be passed via named attribute `extra`,
like in logging from the standart library.
Note:
* Attrs must be dict, otherwise the whole message would be skipped.
* The key field in an attr is converted to string.
* The value is sent as is if isinstance of (str, unicode, int, float, long, bool),
otherwise we convert the value to string.
"""
buff = BytesIO()
while True:
msgs = list()
try:
msg = yield self.queue.get()
# we need to connect first, as we issue verbosity request just after connection
# and channels should strictly go in ascending order
if not self._connected:
yield self.connect()
try:
while True:
msgs.append(msg)
counter = next(self.counter)
msgpack_pack([counter, EMIT, msg], buff)
msg = self.queue.get_nowait()
except queues.QueueEmpty:
pass
try:
yield self.pipe.write(buff.getvalue())
except Exception:
pass
# clean the buffer or we will end up without memory
buff.truncate(0)
except Exception:
for message in msgs:
self._log_to_fallback(message)
def _send(self):
""" Send a message lazy formatted with args.
External log attributes can be passed via named attribute `extra`,
like in logging from the standart library.
Note:
* Attrs must be dict, otherwise the whole message would be skipped.
* The key field in an attr is converted to string.
* The value is sent as is if isinstance of (str, unicode, int, float, long, bool),
otherwise we convert the value to string.
"""
buff = BytesIO()
while True:
msgs = list()
try:
msg = yield self.queue.get()
# we need to connect first, as we issue verbosity request just after connection
# and channels should strictly go in ascending order
if not self._connected:
yield self.connect()
try:
while True:
msgs.append(msg)
counter = next(self.counter)
msgpack_pack([counter, EMIT, msg], buff)
msg = self.queue.get_nowait()
except queues.QueueEmpty:
pass
try:
yield self.pipe.write(buff.getvalue())
except Exception:
pass
# clean the buffer or we will end up without memory
buff.truncate(0)
except Exception:
for message in msgs:
self._log_to_fallback(message)
def parse_segment(text, combinator=constants.LOGICAL_AND):
# Initialize a query segment.
segment = QuerySegment()
# Construct an iterator over the segment text.
iterator = iter(text)
stream = StringIO()
# Iterate through the characters in the segment; one-by-one
# in order to perform one-pass parsing.
for character in iterator:
if (character == constants.NEGATION[1]
and not stream.tell() and not segment.path):
# We've been negated.
segment.negated = not segment.negated
continue
if character in OPERATOR_BEGIN_CHARS:
# Found an operator; pull out what we can.
iterator = _parse_operator(segment, chain(character, iterator))
# We're done here; go to the value parser
break
if character == constants.SEP_PATH:
# A path separator, push the current stack into the path
segment.path.append(stream.getvalue())
stream.truncate(0)
stream.seek(0)
# Keep checking for more path segments.
continue
# Append the text to the stream
stream.write(character)
# Write any remaining information into the path.
segment.path.append(stream.getvalue())
# Attempt to normalize the path.
try:
# The keyword 'not' can be the last item which
# negates this query.
if segment.path[-1] == constants.NEGATION[0]:
segment.negated = not segment.negated
segment.path.pop(-1)
# The last keyword can explicitly state the operation; in which
# case the operator symbol **must** be `=`.
if segment.path[-1] in OPERATOR_KEYWORDS:
if segment.operator != constants.OPERATOR_IEQUAL[0]:
raise ValueError(
'Explicit operations must use the `=` symbol.')
segment.operator = segment.path.pop(-1)
# Make sure we still have a path left.
if not segment.path:
raise IndexError()
except IndexError:
# Ran out of path items after removing operations and negation.
raise ValueError('No path specified in {}'.format(text))
# Values are not complicated (yet) so just slice and dice
# until we get a list of possible values.
segment.values = ''.join(iterator)
if segment.values:
segment.values = segment.values.split(constants.SEP_VALUE)
# Set the combinator.
segment.combinator = COMBINATORS[combinator]
# Return the constructed query segment.
return segment
class PMRequestListener(object):
""" Daemon process that responds to requests """
def __init__(self, config, buildroot):
self.config = config
self.buildroot = buildroot
self.rundir = buildroot.make_chroot_path(RUNDIR)
self.socket_path = os.path.join(self.rundir, SOCKET_NAME)
self.executed_commands = []
# util.do cannot return output when the command fails, we need to
# capture it's logging
self.log_buffer = StringIO()
self.log = logging.getLogger("mockbuild.plugin.pm_request")
self.log.level = logging.DEBUG
self.log.addFilter(OutputFilter())
self.log.propagate = False
self.log.addHandler(logging.StreamHandler(self.log_buffer))
def prepare_socket(self):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
sock.connect(self.socket_path)
except (socket.error, OSError):
try:
os.unlink(self.socket_path)
except OSError:
pass
else:
# there's another process listening
sys.exit(0)
util.mkdirIfAbsent(self.rundir)
# Don't allow regular users to access the socket as they may not be in
# the mock group
os.chown(self.rundir, self.buildroot.chrootuid,
self.buildroot.chrootgid)
os.chmod(self.rundir, 0o770)
sock.bind(self.socket_path)
os.chown(self.socket_path, self.buildroot.chrootuid,
self.buildroot.chrootgid)
return sock
def listen(self):
sock = self.prepare_socket()
sock.listen(MAX_CONNECTIONS)
while True:
try:
connection, _ = sock.accept()
try:
line = connection.makefile().readline()
command = shlex.split(line)
# pylint:disable=E1101
if command == ["!LOG_EXECUTED"]:
connection.sendall('\n'.join(
self.executed_commands).encode())
elif command:
success, out = self.execute_command(command)
connection.sendall(b"ok\n" if success else b"nok\n")
connection.sendall(out.encode())
if success:
self.executed_commands.append(line.strip())
finally:
connection.close()
except socket.error:
continue
def execute_command(self, command):
try:
self.buildroot.pkg_manager.execute(*command,
printOutput=False,
logger=self.log,
returnOutput=False,
pty=False,
raiseExc=True)
success = True
except Error:
success = False
out = self.log_buffer.getvalue()
self.log_buffer.seek(0)
self.log_buffer.truncate()
return success, out
class Daemon:
IFACE = "127.0.0.1"
def __init__(self, ssl=None, **daemonargs):
self.q = queue.Queue()
self.logfp = StringIO()
daemonargs["logfp"] = self.logfp
self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
self.thread.start()
self.port = self.q.get(True, 5)
self.urlbase = "%s://%s:%s" % (
"https" if ssl else "http",
self.IFACE,
self.port
)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.logfp.truncate(0)
self.shutdown()
return False
def p(self, spec):
"""
Return a URL that will render the response in spec.
"""
return "%s/p/%s" % (self.urlbase, spec)
def text_log(self):
return self.logfp.getvalue()
def wait_for_silence(self, timeout=5):
self.thread.server.wait_for_silence(timeout=timeout)
def expect_log(self, n, timeout=5):
l = []
start = time.time()
while True:
l = self.log()
if time.time() - start >= timeout:
return None
if len(l) >= n:
break
return l
def last_log(self):
"""
Returns the last logged request, or None.
"""
l = self.expect_log(1)
if not l:
return None
return l[-1]
def log(self):
"""
Return the log buffer as a list of dictionaries.
"""
return self.thread.server.get_log()
def clear_log(self):
"""
Clear the log.
"""
return self.thread.server.clear_log()
def shutdown(self):
"""
Shut the daemon down, return after the thread has exited.
"""
self.thread.server.shutdown()
self.thread.join()
class Daemon:
IFACE = "127.0.0.1"
def __init__(self, ssl=None, **daemonargs):
self.q = queue.Queue()
self.logfp = StringIO()
daemonargs["logfp"] = self.logfp
self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
self.thread.start()
self.port = self.q.get(True, 5)
self.urlbase = "%s://%s:%s" % (
"https" if ssl else "http",
self.IFACE,
self.port
)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.logfp.truncate(0)
self.shutdown()
return False
def p(self, spec):
"""
Return a URL that will render the response in spec.
"""
return "%s/p/%s" % (self.urlbase, spec)
def info(self):
"""
Return some basic info about the remote daemon.
"""
resp = requests.get("%s/api/info" % self.urlbase, verify=False)
return resp.json()
def text_log(self):
return self.logfp.getvalue()
def last_log(self):
"""
Returns the last logged request, or None.
"""
l = self.log()
if not l:
return None
return l[0]
def log(self):
"""
Return the log buffer as a list of dictionaries.
"""
resp = requests.get("%s/api/log" % self.urlbase, verify=False)
return resp.json()["log"]
def clear_log(self):
"""
Clear the log.
"""
self.logfp.truncate(0)
resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False)
return resp.ok
def shutdown(self):
"""
Shut the daemon down, return after the thread has exited.
"""
self.thread.server.shutdown()
self.thread.join()
class Daemon:
IFACE = "127.0.0.1"
def __init__(self, ssl=None, **daemonargs):
self.q = queue.Queue()
self.logfp = StringIO()
daemonargs["logfp"] = self.logfp
self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
self.thread.start()
self.port = self.q.get(True, 5)
self.urlbase = "%s://%s:%s" % ("https" if ssl else "http", self.IFACE,
self.port)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.logfp.truncate(0)
self.shutdown()
return False
def p(self, spec):
"""
Return a URL that will render the response in spec.
"""
return "%s/p/%s" % (self.urlbase, spec)
def text_log(self):
return self.logfp.getvalue()
def wait_for_silence(self, timeout=5):
start = time.time()
while 1:
if time.time() - start >= timeout:
raise TimeoutError("%s service threads still alive" %
self.thread.server.handler_counter.count)
if self.thread.server.handler_counter.count == 0:
return
def expect_log(self, n, timeout=5):
l = []
start = time.time()
while True:
l = self.log()
if time.time() - start >= timeout:
return None
if len(l) >= n:
break
return l
def last_log(self):
"""
Returns the last logged request, or None.
"""
l = self.expect_log(1)
if not l:
return None
return l[-1]
def log(self):
"""
Return the log buffer as a list of dictionaries.
"""
return self.thread.server.get_log()
def clear_log(self):
"""
Clear the log.
"""
return self.thread.server.clear_log()
def shutdown(self):
"""
Shut the daemon down, return after the thread has exited.
"""
self.thread.server.shutdown()
self.thread.join()
class Daemon:
IFACE = "127.0.0.1"
def __init__(self, ssl=None, **daemonargs):
self.q = queue.Queue()
self.logfp = StringIO()
daemonargs["logfp"] = self.logfp
self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
self.thread.start()
self.port = self.q.get(True, 5)
self.urlbase = "%s://%s:%s" % ("https" if ssl else "http", self.IFACE,
self.port)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.logfp.truncate(0)
self.shutdown()
return False
def p(self, spec):
"""
Return a URL that will render the response in spec.
"""
return "%s/p/%s" % (self.urlbase, spec)
def info(self):
"""
Return some basic info about the remote daemon.
"""
resp = requests.get("%s/api/info" % self.urlbase, verify=False)
return resp.json()
def text_log(self):
return self.logfp.getvalue()
def last_log(self):
"""
Returns the last logged request, or None.
"""
l = self.log()
if not l:
return None
return l[0]
def log(self):
"""
Return the log buffer as a list of dictionaries.
"""
resp = requests.get("%s/api/log" % self.urlbase, verify=False)
return resp.json()["log"]
def clear_log(self):
"""
Clear the log.
"""
self.logfp.truncate(0)
resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False)
return resp.ok
def shutdown(self):
"""
Shut the daemon down, return after the thread has exited.
"""
self.thread.server.shutdown()
self.thread.join()
class PMRequestListener(object):
""" Daemon process that responds to requests """
def __init__(self, config, buildroot):
self.config = config
self.buildroot = buildroot
self.rundir = buildroot.make_chroot_path(RUNDIR)
self.socket_path = os.path.join(self.rundir, SOCKET_NAME)
self.executed_commands = []
# util.do cannot return output when the command fails, we need to
# capture it's logging
self.log_buffer = StringIO()
self.log = logging.getLogger("mockbuild.plugin.pm_request")
self.log.level = logging.DEBUG
self.log.addFilter(OutputFilter())
self.log.propagate = False
self.log.addHandler(logging.StreamHandler(self.log_buffer))
def prepare_socket(self):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
sock.connect(self.socket_path)
except (socket.error, OSError):
try:
os.unlink(self.socket_path)
except OSError:
pass
else:
# there's another process listening
sys.exit(0)
util.mkdirIfAbsent(self.rundir)
# Don't allow regular users to access the socket as they may not be in
# the mock group
os.chown(self.rundir, self.buildroot.chrootuid, self.buildroot.chrootgid)
os.chmod(self.rundir, 0o770)
sock.bind(self.socket_path)
os.chown(self.socket_path, self.buildroot.chrootuid, self.buildroot.chrootgid)
return sock
def listen(self):
sock = self.prepare_socket()
sock.listen(MAX_CONNECTIONS)
while True:
try:
connection, _ = sock.accept()
try:
line = connection.makefile().readline()
command = shlex.split(line)
# pylint:disable=E1101
if command == ["!LOG_EXECUTED"]:
connection.sendall('\n'.join(self.executed_commands).encode())
elif command:
success, out = self.execute_command(command)
connection.sendall(b"ok\n" if success else b"nok\n")
connection.sendall(out.encode())
if success:
self.executed_commands.append(line.strip())
finally:
connection.close()
except socket.error:
continue
def execute_command(self, command):
try:
self.buildroot.pkg_manager.execute(
*command, printOutput=False, logger=self.log,
returnOutput=False, pty=False, raiseExc=True)
success = True
except Error:
success = False
out = self.log_buffer.getvalue()
self.log_buffer.seek(0)
self.log_buffer.truncate()
return success, out
