def handle_flags_token(self, index, token, token_stream, query):
flags = token.match_data.group(1).split(",")
# Must be the first index
if index != 0:
raise errors.SyntaxError(expected=[])
called_functions = []
for current_flag in flags:
current_flag = current_flag.lower()
if current_flag not in self.FLAG_HANDLERS:
raise errors.SyntaxError(expected=[])
else:
called_functions.append(self.FLAG_HANDLERS[current_flag])
# Call the flag handlers
for called_function in called_functions:
called_function(self)
return token_stream
def _lex(self, query):
"""
The initial stage of compilation. We lex the input query into a format that is easier for the
parser to deal with.
"""
self.current_compilation = []
self.logic_stack = [self.current_compilation]
token_stream = generate_token_stream(query)
last_token = None
try:
current_index = 0
current_stream = token_stream
while True:
current_token = next(current_stream)
current_token_type = type(current_token)
if current_token_type is tokens.WhiteSpace:
current_index += 1
continue
if current_token_type not in self.TOKEN_HANDLERS:
raise QueryError("Unknown token type: %s" %
current_token.__class__.__name__)
# Check if it makes syntactically sense
if self.previous_token is not None:
previous_token_type = type(self.previous_token)
expected_tokens = tokens.SYNTAX_TABLE[previous_token_type]
if current_token_type not in expected_tokens:
raise errors.SyntaxError(
expected=expected_tokens,
query=query,
previous_token=self.previous_token,
found=current_token,
token_index=current_index)
self.previous_token = current_token
current_stream = self.TOKEN_HANDLERS[current_token_type](
self, current_index, current_token, token_stream, query)
current_index += 1
except StopIteration as e:
pass
if len(self.logic_stack) != 1:
raise errors.BadParenthesesError()
return
def __init__(self,
tokens,
keep=None,
debug=False,
_empty=False,
num_retries=0):
self._stream_name = None
self._data_locators = []
self._files = collections.OrderedDict()
self._keep = keep
self.num_retries = num_retries
streamoffset = 0L
# parse stream
for tok in tokens:
if debug: print 'tok', tok
if self._stream_name is None:
self._stream_name = tok.replace('\\040', ' ')
continue
s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok)
if s:
blocksize = long(s.group(1))
self._data_locators.append(
Range(tok, streamoffset, blocksize, 0))
streamoffset += blocksize
continue
s = re.search(r'^(\d+):(\d+):(\S+)', tok)
if s:
pos = long(s.group(1))
size = long(s.group(2))
name = s.group(3).replace('\\040', ' ')
if name not in self._files:
self._files[name] = StreamFileReader(
self, [Range(pos, 0, size, 0)], name)
else:
filereader = self._files[name]
filereader.segments.append(
Range(pos, filereader.size(), size))
continue
raise errors.SyntaxError("Invalid manifest format")
def __init__(self, tokens, keep=None, debug=False, _empty=False):
self._stream_name = None
self._data_locators = []
self._files = collections.OrderedDict()
if keep != None:
self._keep = keep
else:
self._keep = Keep.global_client_object()
streamoffset = 0L
# parse stream
for tok in tokens:
if debug: print 'tok', tok
if self._stream_name == None:
self._stream_name = tok.replace('\\040', ' ')
continue
s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok)
if s:
blocksize = long(s.group(1))
self._data_locators.append([tok, blocksize, streamoffset])
streamoffset += blocksize
continue
s = re.search(r'^(\d+):(\d+):(\S+)', tok)
if s:
pos = long(s.group(1))
size = long(s.group(2))
name = s.group(3).replace('\\040', ' ')
if name not in self._files:
self._files[name] = StreamFileReader(
self, [[pos, size, 0]], name)
else:
n = self._files[name]
n.segments.append([pos, size, n.size()])
continue
raise errors.SyntaxError("Invalid manifest format")
def eval(self, bundle, leader):
if self.type is CollTypes.VECTOR:
ccls = ParseVector
ctyp = ExpressionType.VECTOR_COLLECTION
elif self.type is CollTypes.LIST:
ccls = ParseList
ctyp = ExpressionType.LIST_COLLECTION
elif self.type is CollTypes.MAP:
ccls = ParseMap
ctyp = ExpressionType.MAP_COLLECTION
elif self.type is CollTypes.SET:
ccls = ParseSet
ctyp = ExpressionType.SET_COLLECTION
else:
raise errors.SyntaxError("Unknown collection type {}".format(
self.type))
members = []
for c in self.value:
c.eval(bundle, members)
leader.append(ccls(ctyp, members, self.token, self.ident))