def _parse_operator(segment, iterator):
"""Parses the operator (eg. '==' or '<')."""
stream = StringIO()
for character in iterator:
if character == constants.NEGATION[1]:
if stream.tell():
# Negation can only occur at the start of an operator.
raise ValueError('Unexpected negation.')
# We've been negated.
segment.negated = not segment.negated
continue
if (stream.getvalue() + character not in OPERATOR_SYMBOL_MAP and
stream.getvalue() + character not in OPERATOR_BEGIN_CHARS):
# We're no longer an operator.
break
# Expand the operator
stream.write(character)
# Check for existance.
text = stream.getvalue()
if text not in OPERATOR_SYMBOL_MAP:
# Doesn't exist because of a mis-placed negation in the middle
# of the path.
raise ValueError('Unexpected negation.')
# Set the found operator.
segment.operator = OPERATOR_SYMBOL_MAP[text]
# Return the remaining characters.
return chain(character, iterator)
def parse(text, encoding='utf8'):
"""Parse the querystring into a normalized form."""
# Initialize the query object.
query = Query()
# Decode the text if we got bytes.
if isinstance(text, six.binary_type):
text = text.decode(encoding)
# Iterate through the characters in the query string; one-by-one
# in order to perform one-pass parsing.
stream = StringIO()
for character in text:
# We want to stop reading the query and pass it off to someone
# when we reach a logical or grouping operator.
if character in (constants.LOGICAL_AND, constants.LOGICAL_OR):
if not stream.tell():
# There is no content in the stream; a logical operator
# was found out of place.
raise ValueError('Found `{}` out of place'.format(
character))
# Parse the segment up till the combinator
segment = parse_segment(stream.getvalue(), character)
query.segments.append(segment)
stream.truncate(0)
stream.seek(0)
else:
# This isn't a special character, just roll with it.
stream.write(character)
# TODO: Throw some nonsense here if the query string ended with a
# & or ;, because that makes no sense.
if stream.tell():
# Append the remainder of the query string.
query.segments.append(parse_segment(stream.getvalue()))
# Return the constructed query object.
return query
def do_http(method, url, body=""):
if isinstance(body, str):
body = StringIO(body)
elif isinstance(body, unicode):
raise TypeError("do_http body must be a bytestring, not unicode")
else:
# We must give a Content-Length header to twisted.web, otherwise it
# seems to get a zero-length file. I suspect that "chunked-encoding"
# may fix this.
assert body.tell
assert body.seek
assert body.read
scheme, host, port, path = parse_url(url)
if scheme == "http":
c = httplib.HTTPConnection(host, port)
elif scheme == "https":
c = httplib.HTTPSConnection(host, port)
else:
raise ValueError("unknown scheme '%s', need http or https" % scheme)
c.putrequest(method, path)
c.putheader("Hostname", host)
c.putheader("User-Agent", allmydata.__full_version__ + " (tahoe-client)")
c.putheader("Accept", "text/plain, application/octet-stream")
c.putheader("Connection", "close")
old = body.tell()
body.seek(0, os.SEEK_END)
length = body.tell()
body.seek(old)
c.putheader("Content-Length", str(length))
try:
c.endheaders()
except socket_error as err:
return BadResponse(url, err)
while True:
data = body.read(8192)
if not data:
break
c.send(data)
return c.getresponse()
def do_http(method, url, body=""):
if isinstance(body, str):
body = StringIO(body)
elif isinstance(body, unicode):
raise TypeError("do_http body must be a bytestring, not unicode")
else:
# We must give a Content-Length header to twisted.web, otherwise it
# seems to get a zero-length file. I suspect that "chunked-encoding"
# may fix this.
assert body.tell
assert body.seek
assert body.read
scheme, host, port, path = parse_url(url)
if scheme == "http":
c = httplib.HTTPConnection(host, port)
elif scheme == "https":
c = httplib.HTTPSConnection(host, port)
else:
raise ValueError("unknown scheme '%s', need http or https" % scheme)
c.putrequest(method, path)
c.putheader("Hostname", host)
c.putheader("User-Agent", allmydata.__full_version__ + " (tahoe-client)")
c.putheader("Accept", "text/plain, application/octet-stream")
c.putheader("Connection", "close")
old = body.tell()
body.seek(0, os.SEEK_END)
length = body.tell()
body.seek(old)
c.putheader("Content-Length", str(length))
try:
c.endheaders()
except socket_error as err:
return BadResponse(url, err)
while True:
data = body.read(8192)
if not data:
break
c.send(data)
return c.getresponse()
def _read(sock):
preamble = sock.recv(10)
if preamble == '':
raise socket.error('transmission terminated')
mid, size, protocol = struct.unpack('!IIH', preamble)
log.debug('reading message id %d with %d bytes', mid, size)
sock.settimeout(10)
# Ensure we load the entire buffer. This is a blocking operation and
# probably should be optimized away. However, this is required for for
# reading the raw *.rcx binary data from the client. All other methods I
# have tested don't require this loop.
message = StringIO()
while True:
message.write(sock.recv(size))
if message.tell() == size:
break
return mid, protocol, message.getvalue()
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" %
displaypath)
f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
f.write("<hr>\n<ul>\n")
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f.write('<li><a href="%s">%s</a>\n' %
(urllib.quote(linkname), cgi.escape(displayname)))
f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
f.write("<hr>\n<ul>\n")
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f.write('<li><a href="%s">%s</a>\n'
% (urllib.quote(linkname), cgi.escape(displayname)))
f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def local_changes(self, path=None):
# -z is stable like --porcelain; see the git status documentation for details
cmd = ["status", "-z", "--ignore-submodules=all"]
if path is not None:
cmd.extend(["--", path])
rv = {}
data = self.git(*cmd)
if data == "":
return rv
assert data[-1] == "\0"
f = StringIO(data)
while f.tell() < len(data):
# First two bytes are the status in the stage (index) and working tree, respectively
staged = f.read(1)
worktree = f.read(1)
assert f.read(1) == " "
if staged == "R":
# When a file is renamed, there are two files, the source and the destination
files = 2
else:
files = 1
filenames = []
for i in range(files):
filenames.append("")
char = f.read(1)
while char != "\0":
filenames[-1] += char
char = f.read(1)
if not is_blacklisted(rel_path_to_url(filenames[0], self.url_base)):
rv.update(self.local_status(staged, worktree, filenames))
return rv
def parse_segment(text, combinator=constants.LOGICAL_AND):
# Initialize a query segment.
segment = QuerySegment()
# Construct an iterator over the segment text.
iterator = iter(text)
stream = StringIO()
# Iterate through the characters in the segment; one-by-one
# in order to perform one-pass parsing.
for character in iterator:
if (character == constants.NEGATION[1]
and not stream.tell() and not segment.path):
# We've been negated.
segment.negated = not segment.negated
continue
if character in OPERATOR_BEGIN_CHARS:
# Found an operator; pull out what we can.
iterator = _parse_operator(segment, chain(character, iterator))
# We're done here; go to the value parser
break
if character == constants.SEP_PATH:
# A path separator, push the current stack into the path
segment.path.append(stream.getvalue())
stream.truncate(0)
stream.seek(0)
# Keep checking for more path segments.
continue
# Append the text to the stream
stream.write(character)
# Write any remaining information into the path.
segment.path.append(stream.getvalue())
# Attempt to normalize the path.
try:
# The keyword 'not' can be the last item which
# negates this query.
if segment.path[-1] == constants.NEGATION[0]:
segment.negated = not segment.negated
segment.path.pop(-1)
# The last keyword can explicitly state the operation; in which
# case the operator symbol **must** be `=`.
if segment.path[-1] in OPERATOR_KEYWORDS:
if segment.operator != constants.OPERATOR_IEQUAL[0]:
raise ValueError(
'Explicit operations must use the `=` symbol.')
segment.operator = segment.path.pop(-1)
# Make sure we still have a path left.
if not segment.path:
raise IndexError()
except IndexError:
# Ran out of path items after removing operations and negation.
raise ValueError('No path specified in {}'.format(text))
# Values are not complicated (yet) so just slice and dice
# until we get a list of possible values.
segment.values = ''.join(iterator)
if segment.values:
segment.values = segment.values.split(constants.SEP_VALUE)
# Set the combinator.
segment.combinator = COMBINATORS[combinator]
# Return the constructed query segment.
return segment
class FileCacheObject(CacheObject):
_struct = struct.Struct('dII') # double and two ints
# timestamp, lifetime, position
@classmethod
def fromFile(cls, fd):
dat = cls._struct.unpack(fd.read(cls._struct.size))
obj = cls(None, None, dat[1], dat[0])
obj.position = dat[2]
return obj
def __init__(self, *args, **kwargs):
self._key = None
self._data = None
self._size = None
self._buff = StringIO()
super(FileCacheObject, self).__init__(*args, **kwargs)
@property
def size(self):
if self._size is None:
self._buff.seek(0, 2)
size = self._buff.tell()
if size == 0:
if (self._key is None) or (self._data is None):
raise RuntimeError
json.dump([self.key, self.data], self._buff)
self._size = self._buff.tell()
self._size = size
return self._size
@size.setter
def size(self, value):
self._size = value
@property
def key(self):
if self._key is None:
try:
self._key, self._data = json.loads(self._buff.getvalue())
except:
pass
return self._key
@key.setter
def key(self, value):
self._key = value
@property
def data(self):
if self._data is None:
self._key, self._data = json.loads(self._buff.getvalue())
return self._data
@data.setter
def data(self, value):
self._data = value
def load(self, fd):
fd.seek(self.position)
self._buff.seek(0)
self._buff.write(fd.read(self.size))
def dumpslot(self, fd):
pos = fd.tell()
fd.write(self._struct.pack(self.creation, self.lifetime,
self.position))
def dumpdata(self, fd):
self.size
fd.seek(self.position)
fd.write(self._buff.getvalue())
def split_segments(text, closing_paren=False):
"""Return objects representing segments."""
buf = StringIO()
# The segments we're building, and the combinators used to combine them.
# Note that after this is complete, this should be true:
# len(segments) == len(combinators) + 1
# Thus we can understand the relationship between segments and combinators
# like so:
# s1 (c1) s2 (c2) s3 (c3) where sN are segments and cN are combination
# functions.
# TODO: Figure out exactly where the querystring died and post cool
# error messages about it.
segments = []
combinators = []
# A flag dictating if the last character we processed was a group.
# This is used to determine if the next character (being a combinator)
# is allowed to
last_group = False
# The recursive nature of this function relies on keeping track of the
# state of iteration. This iterator will be passed down to recursed calls.
iterator = iter(text)
# Detection for exclamation points. only matters for this situation:
# foo=bar&!(bar=baz)
last_negation = False
for character in iterator:
if character in COMBINATORS:
if last_negation:
buf.write(constants.OPERATOR_NEGATION)
# The string representation of our segment.
val = buf.getvalue()
reset_stringio(buf)
if not last_group and not len(val):
raise ValueError('Unexpected %s.' % character)
# When a group happens, the previous value is empty.
if len(val):
segments.append(parse_segment(val))
combinators.append(COMBINATORS[character])
elif character == constants.GROUP_BEGIN:
# Recursively go into the next group.
if buf.tell():
raise ValueError('Unexpected %s' % character)
seg = split_segments(iterator, True)
if last_negation:
seg = UnarySegmentCombinator(seg)
segments.append(seg)
# Flag that the last entry was a grouping, so that we don't panic
# when the next character is a logical combinator
last_group = True
continue
elif character == constants.GROUP_END:
# Build the segment for anything remaining, and then combine
# all the segments.
val = buf.getvalue()
# Check for unbalanced parens or an empty thing: foo=bar&();bar=baz
if not buf.tell() or not closing_paren:
raise ValueError('Unexpected %s' % character)
segments.append(parse_segment(val))
return combine(segments, combinators)
elif character == constants.OPERATOR_NEGATION and not buf.tell():
last_negation = True
continue
else:
if last_negation:
buf.write(constants.OPERATOR_NEGATION)
if last_group:
raise ValueError('Unexpected %s' % character)
buf.write(character)
last_negation = False
last_group = False
else:
# Check and see if the iterator exited early (unbalanced parens)
if closing_paren:
raise ValueError('Expected %s.' % constants.GROUP_END)
if not last_group:
# Add the final segment.
segments.append(parse_segment(buf.getvalue()))
# Everything completed normally, combine all the segments into one
# and return them.
return combine(segments, combinators)
