def handle_flags_token(self, index, token, token_stream, query): flags = token.match_data.group(1).split(",") # Must be the first index if index != 0: raise errors.SyntaxError(expected=[]) called_functions = [] for current_flag in flags: current_flag = current_flag.lower() if current_flag not in self.FLAG_HANDLERS: raise errors.SyntaxError(expected=[]) else: called_functions.append(self.FLAG_HANDLERS[current_flag]) # Call the flag handlers for called_function in called_functions: called_function(self) return token_stream
def _lex(self, query): """ The initial stage of compilation. We lex the input query into a format that is easier for the parser to deal with. """ self.current_compilation = [] self.logic_stack = [self.current_compilation] token_stream = generate_token_stream(query) last_token = None try: current_index = 0 current_stream = token_stream while True: current_token = next(current_stream) current_token_type = type(current_token) if current_token_type is tokens.WhiteSpace: current_index += 1 continue if current_token_type not in self.TOKEN_HANDLERS: raise QueryError("Unknown token type: %s" % current_token.__class__.__name__) # Check if it makes syntactically sense if self.previous_token is not None: previous_token_type = type(self.previous_token) expected_tokens = tokens.SYNTAX_TABLE[previous_token_type] if current_token_type not in expected_tokens: raise errors.SyntaxError( expected=expected_tokens, query=query, previous_token=self.previous_token, found=current_token, token_index=current_index) self.previous_token = current_token current_stream = self.TOKEN_HANDLERS[current_token_type]( self, current_index, current_token, token_stream, query) current_index += 1 except StopIteration as e: pass if len(self.logic_stack) != 1: raise errors.BadParenthesesError() return
def __init__(self, tokens, keep=None, debug=False, _empty=False, num_retries=0): self._stream_name = None self._data_locators = [] self._files = collections.OrderedDict() self._keep = keep self.num_retries = num_retries streamoffset = 0L # parse stream for tok in tokens: if debug: print 'tok', tok if self._stream_name is None: self._stream_name = tok.replace('\\040', ' ') continue s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok) if s: blocksize = long(s.group(1)) self._data_locators.append( Range(tok, streamoffset, blocksize, 0)) streamoffset += blocksize continue s = re.search(r'^(\d+):(\d+):(\S+)', tok) if s: pos = long(s.group(1)) size = long(s.group(2)) name = s.group(3).replace('\\040', ' ') if name not in self._files: self._files[name] = StreamFileReader( self, [Range(pos, 0, size, 0)], name) else: filereader = self._files[name] filereader.segments.append( Range(pos, filereader.size(), size)) continue raise errors.SyntaxError("Invalid manifest format")
def __init__(self, tokens, keep=None, debug=False, _empty=False): self._stream_name = None self._data_locators = [] self._files = collections.OrderedDict() if keep != None: self._keep = keep else: self._keep = Keep.global_client_object() streamoffset = 0L # parse stream for tok in tokens: if debug: print 'tok', tok if self._stream_name == None: self._stream_name = tok.replace('\\040', ' ') continue s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok) if s: blocksize = long(s.group(1)) self._data_locators.append([tok, blocksize, streamoffset]) streamoffset += blocksize continue s = re.search(r'^(\d+):(\d+):(\S+)', tok) if s: pos = long(s.group(1)) size = long(s.group(2)) name = s.group(3).replace('\\040', ' ') if name not in self._files: self._files[name] = StreamFileReader( self, [[pos, size, 0]], name) else: n = self._files[name] n.segments.append([pos, size, n.size()]) continue raise errors.SyntaxError("Invalid manifest format")
def eval(self, bundle, leader): if self.type is CollTypes.VECTOR: ccls = ParseVector ctyp = ExpressionType.VECTOR_COLLECTION elif self.type is CollTypes.LIST: ccls = ParseList ctyp = ExpressionType.LIST_COLLECTION elif self.type is CollTypes.MAP: ccls = ParseMap ctyp = ExpressionType.MAP_COLLECTION elif self.type is CollTypes.SET: ccls = ParseSet ctyp = ExpressionType.SET_COLLECTION else: raise errors.SyntaxError("Unknown collection type {}".format( self.type)) members = [] for c in self.value: c.eval(bundle, members) leader.append(ccls(ctyp, members, self.token, self.ident))