def _parse_operator(segment, iterator): """Parses the operator (eg. '==' or '<').""" stream = StringIO() for character in iterator: if character == constants.NEGATION[1]: if stream.tell(): # Negation can only occur at the start of an operator. raise ValueError('Unexpected negation.') # We've been negated. segment.negated = not segment.negated continue if (stream.getvalue() + character not in OPERATOR_SYMBOL_MAP and stream.getvalue() + character not in OPERATOR_BEGIN_CHARS): # We're no longer an operator. break # Expand the operator stream.write(character) # Check for existance. text = stream.getvalue() if text not in OPERATOR_SYMBOL_MAP: # Doesn't exist because of a mis-placed negation in the middle # of the path. raise ValueError('Unexpected negation.') # Set the found operator. segment.operator = OPERATOR_SYMBOL_MAP[text] # Return the remaining characters. return chain(character, iterator)
def parse(text, encoding='utf8'): """Parse the querystring into a normalized form.""" # Initialize the query object. query = Query() # Decode the text if we got bytes. if isinstance(text, six.binary_type): text = text.decode(encoding) # Iterate through the characters in the query string; one-by-one # in order to perform one-pass parsing. stream = StringIO() for character in text: # We want to stop reading the query and pass it off to someone # when we reach a logical or grouping operator. if character in (constants.LOGICAL_AND, constants.LOGICAL_OR): if not stream.tell(): # There is no content in the stream; a logical operator # was found out of place. raise ValueError('Found `{}` out of place'.format( character)) # Parse the segment up till the combinator segment = parse_segment(stream.getvalue(), character) query.segments.append(segment) stream.truncate(0) stream.seek(0) else: # This isn't a special character, just roll with it. stream.write(character) # TODO: Throw some nonsense here if the query string ended with a # & or ;, because that makes no sense. if stream.tell(): # Append the remainder of the query string. query.segments.append(parse_segment(stream.getvalue())) # Return the constructed query object. return query
def do_http(method, url, body=""): if isinstance(body, str): body = StringIO(body) elif isinstance(body, unicode): raise TypeError("do_http body must be a bytestring, not unicode") else: # We must give a Content-Length header to twisted.web, otherwise it # seems to get a zero-length file. I suspect that "chunked-encoding" # may fix this. assert body.tell assert body.seek assert body.read scheme, host, port, path = parse_url(url) if scheme == "http": c = httplib.HTTPConnection(host, port) elif scheme == "https": c = httplib.HTTPSConnection(host, port) else: raise ValueError("unknown scheme '%s', need http or https" % scheme) c.putrequest(method, path) c.putheader("Hostname", host) c.putheader("User-Agent", allmydata.__full_version__ + " (tahoe-client)") c.putheader("Accept", "text/plain, application/octet-stream") c.putheader("Connection", "close") old = body.tell() body.seek(0, os.SEEK_END) length = body.tell() body.seek(old) c.putheader("Content-Length", str(length)) try: c.endheaders() except socket_error as err: return BadResponse(url, err) while True: data = body.read(8192) if not data: break c.send(data) return c.getresponse()
def _read(sock): preamble = sock.recv(10) if preamble == '': raise socket.error('transmission terminated') mid, size, protocol = struct.unpack('!IIH', preamble) log.debug('reading message id %d with %d bytes', mid, size) sock.settimeout(10) # Ensure we load the entire buffer. This is a blocking operation and # probably should be optimized away. However, this is required for for # reading the raw *.rcx binary data from the client. All other methods I # have tested don't require this loop. message = StringIO() while True: message.write(sock.recv(size)) if message.tell() == size: break return mid, protocol, message.getvalue()
def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) f = StringIO() displaypath = cgi.escape(urllib.unquote(self.path)) f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath) f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath) f.write("<hr>\n<ul>\n") for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / f.write('<li><a href="%s">%s</a>\n' % (urllib.quote(linkname), cgi.escape(displayname))) f.write("</ul>\n<hr>\n</body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) encoding = sys.getfilesystemencoding() self.send_header("Content-type", "text/html; charset=%s" % encoding) self.send_header("Content-Length", str(length)) self.end_headers() return f
def local_changes(self, path=None): # -z is stable like --porcelain; see the git status documentation for details cmd = ["status", "-z", "--ignore-submodules=all"] if path is not None: cmd.extend(["--", path]) rv = {} data = self.git(*cmd) if data == "": return rv assert data[-1] == "\0" f = StringIO(data) while f.tell() < len(data): # First two bytes are the status in the stage (index) and working tree, respectively staged = f.read(1) worktree = f.read(1) assert f.read(1) == " " if staged == "R": # When a file is renamed, there are two files, the source and the destination files = 2 else: files = 1 filenames = [] for i in range(files): filenames.append("") char = f.read(1) while char != "\0": filenames[-1] += char char = f.read(1) if not is_blacklisted(rel_path_to_url(filenames[0], self.url_base)): rv.update(self.local_status(staged, worktree, filenames)) return rv
def parse_segment(text, combinator=constants.LOGICAL_AND): # Initialize a query segment. segment = QuerySegment() # Construct an iterator over the segment text. iterator = iter(text) stream = StringIO() # Iterate through the characters in the segment; one-by-one # in order to perform one-pass parsing. for character in iterator: if (character == constants.NEGATION[1] and not stream.tell() and not segment.path): # We've been negated. segment.negated = not segment.negated continue if character in OPERATOR_BEGIN_CHARS: # Found an operator; pull out what we can. iterator = _parse_operator(segment, chain(character, iterator)) # We're done here; go to the value parser break if character == constants.SEP_PATH: # A path separator, push the current stack into the path segment.path.append(stream.getvalue()) stream.truncate(0) stream.seek(0) # Keep checking for more path segments. continue # Append the text to the stream stream.write(character) # Write any remaining information into the path. segment.path.append(stream.getvalue()) # Attempt to normalize the path. try: # The keyword 'not' can be the last item which # negates this query. if segment.path[-1] == constants.NEGATION[0]: segment.negated = not segment.negated segment.path.pop(-1) # The last keyword can explicitly state the operation; in which # case the operator symbol **must** be `=`. if segment.path[-1] in OPERATOR_KEYWORDS: if segment.operator != constants.OPERATOR_IEQUAL[0]: raise ValueError( 'Explicit operations must use the `=` symbol.') segment.operator = segment.path.pop(-1) # Make sure we still have a path left. if not segment.path: raise IndexError() except IndexError: # Ran out of path items after removing operations and negation. raise ValueError('No path specified in {}'.format(text)) # Values are not complicated (yet) so just slice and dice # until we get a list of possible values. segment.values = ''.join(iterator) if segment.values: segment.values = segment.values.split(constants.SEP_VALUE) # Set the combinator. segment.combinator = COMBINATORS[combinator] # Return the constructed query segment. return segment
class FileCacheObject(CacheObject): _struct = struct.Struct('dII') # double and two ints # timestamp, lifetime, position @classmethod def fromFile(cls, fd): dat = cls._struct.unpack(fd.read(cls._struct.size)) obj = cls(None, None, dat[1], dat[0]) obj.position = dat[2] return obj def __init__(self, *args, **kwargs): self._key = None self._data = None self._size = None self._buff = StringIO() super(FileCacheObject, self).__init__(*args, **kwargs) @property def size(self): if self._size is None: self._buff.seek(0, 2) size = self._buff.tell() if size == 0: if (self._key is None) or (self._data is None): raise RuntimeError json.dump([self.key, self.data], self._buff) self._size = self._buff.tell() self._size = size return self._size @size.setter def size(self, value): self._size = value @property def key(self): if self._key is None: try: self._key, self._data = json.loads(self._buff.getvalue()) except: pass return self._key @key.setter def key(self, value): self._key = value @property def data(self): if self._data is None: self._key, self._data = json.loads(self._buff.getvalue()) return self._data @data.setter def data(self, value): self._data = value def load(self, fd): fd.seek(self.position) self._buff.seek(0) self._buff.write(fd.read(self.size)) def dumpslot(self, fd): pos = fd.tell() fd.write(self._struct.pack(self.creation, self.lifetime, self.position)) def dumpdata(self, fd): self.size fd.seek(self.position) fd.write(self._buff.getvalue())
def split_segments(text, closing_paren=False): """Return objects representing segments.""" buf = StringIO() # The segments we're building, and the combinators used to combine them. # Note that after this is complete, this should be true: # len(segments) == len(combinators) + 1 # Thus we can understand the relationship between segments and combinators # like so: # s1 (c1) s2 (c2) s3 (c3) where sN are segments and cN are combination # functions. # TODO: Figure out exactly where the querystring died and post cool # error messages about it. segments = [] combinators = [] # A flag dictating if the last character we processed was a group. # This is used to determine if the next character (being a combinator) # is allowed to last_group = False # The recursive nature of this function relies on keeping track of the # state of iteration. This iterator will be passed down to recursed calls. iterator = iter(text) # Detection for exclamation points. only matters for this situation: # foo=bar&!(bar=baz) last_negation = False for character in iterator: if character in COMBINATORS: if last_negation: buf.write(constants.OPERATOR_NEGATION) # The string representation of our segment. val = buf.getvalue() reset_stringio(buf) if not last_group and not len(val): raise ValueError('Unexpected %s.' % character) # When a group happens, the previous value is empty. if len(val): segments.append(parse_segment(val)) combinators.append(COMBINATORS[character]) elif character == constants.GROUP_BEGIN: # Recursively go into the next group. if buf.tell(): raise ValueError('Unexpected %s' % character) seg = split_segments(iterator, True) if last_negation: seg = UnarySegmentCombinator(seg) segments.append(seg) # Flag that the last entry was a grouping, so that we don't panic # when the next character is a logical combinator last_group = True continue elif character == constants.GROUP_END: # Build the segment for anything remaining, and then combine # all the segments. val = buf.getvalue() # Check for unbalanced parens or an empty thing: foo=bar&();bar=baz if not buf.tell() or not closing_paren: raise ValueError('Unexpected %s' % character) segments.append(parse_segment(val)) return combine(segments, combinators) elif character == constants.OPERATOR_NEGATION and not buf.tell(): last_negation = True continue else: if last_negation: buf.write(constants.OPERATOR_NEGATION) if last_group: raise ValueError('Unexpected %s' % character) buf.write(character) last_negation = False last_group = False else: # Check and see if the iterator exited early (unbalanced parens) if closing_paren: raise ValueError('Expected %s.' % constants.GROUP_END) if not last_group: # Add the final segment. segments.append(parse_segment(buf.getvalue())) # Everything completed normally, combine all the segments into one # and return them. return combine(segments, combinators)