def namer(stream_in, matcher):
try:
(result, stream_out) = matcher()
except StopIteration:
if show_failures:
stream = \
_adjust(fmt('stream = {rest}', **s_kargs(stream_in)),
right)
str_name = _adjust(name, left // 4, True, True)
match = _adjust(fmt(' {0} failed', str_name), left, True)
# Python bug #4618
print(match + ' ' + stream, file=out, end=str('\n'))
raise StopIteration
else:
try:
try:
rest = fmt('{rest}', **s_kargs(stream_out))
except StopIteration:
rest = '<EOS>'
stream = _adjust(fmt('stream = {0}', rest), right)
str_name = _adjust(name, left // 4, True, True)
match = _adjust(fmt(' {0} = {1}', str_name, result), left, True)
# Python bug #4618
print(match + ' ' + stream, file=out, end=str('\n'))
return (result, stream_out)
except Exception as e:
print('Error in trace', file=out, end=str('\n'))
print(repr(e), file=out, end=str('\n'))
return (result, stream_out)
def namer(stream_in, matcher):
try:
(result, stream_out) = matcher()
except StopIteration:
if show_failures:
stream = \
_adjust(fmt('stream = {rest}', **s_kargs(stream_in)),
right)
str_name = _adjust(name, left // 4, True, True)
match = _adjust(fmt(' {0} failed', str_name), left, True)
# Python bug #4618
print(match + ' ' + stream, file=out, end=str('\n'))
raise StopIteration
else:
try:
try:
rest = fmt('{rest}', **s_kargs(stream_out))
except StopIteration:
rest = '<EOS>'
stream = _adjust(fmt('stream = {0}', rest), right)
str_name = _adjust(name, left // 4, True, True)
match = _adjust(fmt(' {0} = {1}', str_name, result), left,
True)
# Python bug #4618
print(match + ' ' + stream, file=out, end=str('\n'))
return (result, stream_out)
except Exception as e:
print('Error in trace', file=out, end=str('\n'))
print(repr(e), file=out, end=str('\n'))
return (result, stream_out)
def test_random(self):
'''
Compares lepl + python expressions. This runs 'til it fails, and it
always does fail, because lepl's expressions are guaranteed greedy
while python's aren't. This is "normal" (Perl is the same as Python)
but I cannot fathom why it should be - it seems *harder* to make them
work that way...
'''
#basicConfig(level=DEBUG)
log = getLogger('lepl.regexp._test.random')
match_alphabet = '012'
string_alphabet = '013'
for _ in range(100):
expression = random_expression(3, match_alphabet)
string = random_string(3, string_alphabet)
matcher = DfaRegexp(expression)
# matcher = NfaRegexp(expression)
matcher.config.no_full_first_match()
lepl_result = matcher.parse(string)
if lepl_result:
lepl_result = lepl_result[0]
log.debug(fmt('{0} {1} {2}', expression, string, lepl_result))
try:
python_result = compile_(expression).match(string)
if python_result:
python_result = python_result.group()
assert lepl_result == python_result, \
fmt('{0} != {1}\n{2} {3}',
lepl_result, python_result, expression, string)
except:
(e, v, _t) = exc_info()
if repr(v) == "error('nothing to repeat',)":
pass
else:
raise e
def __str__(self):
'''
Example:
0: 3, 4; 1: 2; 2(Tk1); 3: [{u'\x00'}-`b-{u'\U0010ffff'}]->3, 1;
4: {$}->5, 7; 5: 6; 6($); 7: {^}->10; 8: 9; 9(^); 10: 11;
11: [ ]->11, 8
Node 0 leads to 3 and 4 (both empty)
Node 1 leads to 2 (empty)
Node 2 is terminal, labelled with "Tk1"
Node 3 loops back to 3 for a character in the given range, or to 1
etc.
'''
lines = []
for node in self:
edges = []
for (dest, edge) in self.transitions(node):
edges.append(fmt('{0}->{1}', edge, dest))
for dest in self.empty_transitions(node):
edges.append(str(dest))
label = '' if self.terminal(node) is None \
else fmt('({0})', self.terminal(node))
if edges:
lines.append(
fmt('{0}{1}: {2}', node, label, ', '.join(edges)))
else:
lines.append(fmt('{0}{1}', node, label))
return '; '.join(lines)
def __add_limited(self, reference):
'''
Add the new reference, discarding an old entry if possible.
'''
while reference:
candidate = heappushpop(self.__queue, reference)
self._debug(fmt('Exchanged {0} for {1}', reference, candidate))
if candidate.order_epoch == self.epoch:
# even the oldest generator is current
break
elif candidate.deletable(self.epoch):
self._debug(fmt('Closing {0}', candidate))
generator = candidate.generator
if generator:
del self.__known[generator]
candidate.close()
return
else:
# try again (candidate has been updated)
reference = candidate
# if we are here, queue is too small
heappush(self.__queue, candidate)
# this is currently 1 too small, and zero means unlimited, so
# doubling should always be sufficient.
self.queue_len = self.queue_len * 2
self._warn(fmt('Queue is too small - extending to {0}',
self.queue_len))
def __str__(self):
generator = self.generator
if generator:
return fmt('{0} ({1:d}/{2:d})', self.__describe, self.order_epoch,
self.__last_known_epoch)
else:
return fmt('Empty ref to {0}', self.__describe)
def fmt_intervals(self, intervals):
'''
Hide unicode chars because of some strange error that occurs with
Python2.6 on the command line.
This is in StrAlphabet, but for ASCII makes no difference. Having it
here helps LineAwareAlphabet work (the whole idea of subclassing
alphabets etc is not so great).
'''
def pretty(c):
x = self._escape_char(c)
if len(x) > 1 or 32 <= ord(x) <= 127:
return str(x)
elif ord(c) < 0x100:
return fmt('\\x{0:02x}', ord(c))
elif ord(c) < 0x10000:
return fmt('\\u{0:04x}', ord(c))
else:
return fmt('\\U{0:08x}', ord(c))
ranges = []
if len(intervals) == 1:
if intervals[0][0] == intervals[0][1]:
return self._escape_char(intervals[0][0])
elif intervals[0][0] == self.min and intervals[0][1] == self.max:
return '.'
# pylint: disable-msg=C0103
# (sorry. but i use this (a, b) convention throughout the regexp lib)
for (a, b) in intervals:
if a == b:
ranges.append(pretty(a))
else:
ranges.append(fmt('{0!s}-{1!s}', pretty(a), pretty(b)))
return fmt('[{0}]', self.join(ranges))
def _match(self, stream_in):
'''
Check that we match the current level
'''
try:
generator = super(LineStart, self)._match(stream_in)
while True:
(indent, stream) = yield generator
self._debug(fmt('SOL {0!r}', indent))
if indent and indent[0] and indent[0][-1] == '\n':
indent[0] = indent[0][:-1]
# if we're not doing indents, this is empty
if not self.indent:
yield ([], stream)
# if we are doing indents, we need a match or NO_BLOCKS
elif self._current_indent == NO_BLOCKS or \
len(indent[0]) == self._current_indent:
yield (indent, stream)
else:
self._debug(
fmt('Incorrect indent ({0:d} != len({1!r}), {2:d})',
self._current_indent, indent[0],
len(indent[0])))
except StopIteration:
pass
def fmt_intervals(self, intervals):
'''
Hide unicode chars because of some strange error that occurs with
Python2.6 on the command line.
This is in StrAlphabet, but for ASCII makes no difference. Having it
here helps LineAwareAlphabet work (the whole idea of subclassing
alphabets etc is not so great).
'''
def pretty(c):
x = self._escape_char(c)
if len(x) > 1 or 32 <= ord(x) <= 127:
return str(x)
elif ord(c) < 0x100:
return fmt('\\x{0:02x}', ord(c))
elif ord(c) < 0x10000:
return fmt('\\u{0:04x}', ord(c))
else:
return fmt('\\U{0:08x}', ord(c))
ranges = []
if len(intervals) == 1:
if intervals[0][0] == intervals[0][1]:
return self._escape_char(intervals[0][0])
elif intervals[0][0] == self.min and intervals[0][1] == self.max:
return '.'
# pylint: disable-msg=C0103
# (sorry. but i use this (a, b) convention throughout the regexp lib)
for (a, b) in intervals:
if a == b:
ranges.append(pretty(a))
else:
ranges.append(fmt('{0!s}-{1!s}', pretty(a), pretty(b)))
return fmt('[{0}]', self.join(ranges))
def test_random(self):
'''
Compares lepl + python expressions. This runs 'til it fails, and it
always does fail, because lepl's expressions are guaranteed greedy
while python's aren't. This is "normal" (Perl is the same as Python)
but I cannot fathom why it should be - it seems *harder* to make them
work that way...
'''
#basicConfig(level=DEBUG)
log = getLogger('lepl.regexp._test.random')
match_alphabet = '012'
string_alphabet = '013'
for _ in range(100):
expression = random_expression(3, match_alphabet)
string = random_string(3, string_alphabet)
matcher = DfaRegexp(expression)
# matcher = NfaRegexp(expression)
matcher.config.no_full_first_match()
lepl_result = matcher.parse(string)
if lepl_result:
lepl_result = lepl_result[0]
log.debug(fmt('{0} {1} {2}', expression, string, lepl_result))
try:
python_result = compile_(expression).match(string)
if python_result:
python_result = python_result.group()
assert lepl_result == python_result, \
fmt('{0} != {1}\n{2} {3}',
lepl_result, python_result, expression, string)
except:
(e, v, _t) = exc_info()
if repr(v) == "error('nothing to repeat',)":
pass
else:
raise e
def to_regexps(cls, use, possibles, have_add=False):
'''
Convert to regular expressions.
`have_add` indicaes whether the caller can supply an "add".
None - caller doesn't care what lower code needed.
True - caller has add, and caller should need that.
False - caller doesn't have add, and caller should not need it.
'''
regexps = []
for possible in possibles:
if isinstance(possible, RegexpContainer):
cls.log.debug(fmt('unpacking: {0!s}', possible))
if have_add is None or possible.add_reqd == have_add:
regexps.append(possible.regexp)
# this flag indicates that it's "worth" using the regexp
# so we "inherit"
use = use or possible.use
else:
raise Unsuitable('Add inconsistent.')
else:
cls.log.debug(fmt('cannot unpack: {0!s}',
possible.__class__))
raise Unsuitable('Not a container.')
return use, regexps
def next(self, state, count=1):
def add_self(response):
'''
Replace the previous helper with this one, which will then
delegate to the previous when needed.
'''
((tokens, token), (state, _)) = response
self._debug(fmt('Return {0}', tokens))
return ((tokens, token), (state, self))
if count != 1:
raise TypeError('Filtered tokens must be read singly')
discard = list(reversed(self._ids))
start = state
while discard:
((tokens, _), (state, _)) = \
super(FilteredTokenHelper, self).next(state)
if discard[-1] in tokens:
self._debug(fmt('Discarding token {0}', discard[-1]))
discard.pop()
else:
self._debug(fmt('Failed to discard token {0}: {1}',
discard[-1], tokens))
return add_self(super(FilteredTokenHelper, self).next(start))
return add_self(super(FilteredTokenHelper, self).next(state))
def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
try:
id_ = s_id(stream)
while not s_empty(stream):
# avoid conflicts between tokens
id_ += 1
try:
(terminals, match, next_stream) = \
self.t_regexp.match(stream)
self._debug(fmt('Token: {0!r} {1!r} {2!s}',
terminals, match, s_debug(stream)))
yield (terminals, s_stream(stream, match, max=max, id_=id_))
except TypeError:
(terminals, _size, next_stream) = \
self.s_regexp.size_match(stream)
self._debug(fmt('Space: {0!r} {1!s}',
terminals, s_debug(stream)))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream,
'No token for {rest} at {location} of {text}.'))
def __add_limited(self, reference):
'''
Add the new reference, discarding an old entry if possible.
'''
while reference:
candidate = heappushpop(self.__queue, reference)
self._debug(fmt('Exchanged {0} for {1}', reference, candidate))
if candidate.order_epoch == self.epoch:
# even the oldest generator is current
break
elif candidate.deletable(self.epoch):
self._debug(fmt('Closing {0}', candidate))
generator = candidate.generator
if generator:
del self.__known[generator]
candidate.close()
return
else:
# try again (candidate has been updated)
reference = candidate
# if we are here, queue is too small
heappush(self.__queue, candidate)
# this is currently 1 too small, and zero means unlimited, so
# doubling should always be sufficient.
self.queue_len *= 2
self._warn(fmt('Queue is too small - extending to {0}', self.queue_len))
def next(self, state, count=1):
def add_self(response):
'''
Replace the previous helper with this one, which will then
delegate to the previous when needed.
'''
((tokens, token), (state, _)) = response
self._debug(fmt('Return {0}', tokens))
return ((tokens, token), (state, self))
if count != 1:
raise TypeError('Filtered tokens must be read singly')
discard = list(reversed(self._ids))
start = state
while discard:
((tokens, _), (state, _)) = \
super(FilteredTokenHelper, self).next(state)
if discard[-1] in tokens:
self._debug(fmt('Discarding token {0}', discard[-1]))
discard.pop()
else:
self._debug(
fmt('Failed to discard token {0}: {1}', discard[-1],
tokens))
return add_self(super(FilteredTokenHelper, self).next(start))
return add_self(super(FilteredTokenHelper, self).next(state))
def _tokens(self, stream, max):
'''
Generate tokens, on demand.
'''
try:
id_ = s_id(stream)
while not s_empty(stream):
# avoid conflicts between tokens
id_ += 1
try:
(terminals, match, next_stream) = \
self.t_regexp.match(stream)
self._debug(
fmt('Token: {0!r} {1!r} {2!s}', terminals, match,
s_debug(stream)))
yield (terminals, s_stream(stream, match, max=max,
id_=id_))
except TypeError:
(terminals, _size, next_stream) = \
self.s_regexp.size_match(stream)
self._debug(
fmt('Space: {0!r} {1!s}', terminals, s_debug(stream)))
stream = next_stream
except TypeError:
raise RuntimeLexerError(
s_fmt(stream, 'No token for {rest} at {location} of {text}.'))
def clone_wrapper(use, original, *args, **kargs):
factory = original.factory
if factory in map_:
log.debug(fmt('Found {0}', factory))
return map_[factory](use, original, *args, **kargs)
else:
log.debug(fmt('No clone for {0}, {1}', factory, map_.keys()))
return original
def clone_wrapper(use, original, *args, **kargs):
factory = original.factory
if factory in map_:
log.debug(fmt('Found {0}', factory))
return map_[factory](use, original, *args, **kargs)
else:
log.debug(fmt('No clone for {0}, {1}', factory, map_.keys()))
return original
def _typename(self, instance):
if isinstance(instance, list) and instance:
return fmt('<list{0}>', self._typename(instance[0]))
else:
try:
return fmt('<{0}>', instance.__class__.__name__)
except:
return '<unknown>'
def record_success(count, stream_in, result):
(value, stream_out) = result
count_desc = fmt(' ({0})', count) if count > 1 else ''
# Python bug #4618
print(fmt('{0}{1} = {2}\n {3} -> {4}',
name, count_desc, value,
fmt_stream(stream_in), fmt_stream(stream_out)),
file=out, end=str('\n'))
def _typename(self, instance):
if isinstance(instance, list) and instance:
return fmt('<list{0}>', self._typename(instance[0]))
else:
try:
return fmt('<{0}>', instance.__class__.__name__)
except:
return '<unknown>'
def record_success(count, stream_in, result):
(value, stream_out) = result
count_desc = fmt(' ({0})', count) if count > 1 else ''
# Python bug #4618
print(fmt('{0}{1} = {2}\n {3} -> {4}', name, count_desc, value,
fmt_stream(stream_in), fmt_stream(stream_out)),
file=out,
end=str('\n'))
def __str__(self):
generator = self.generator
if generator:
return fmt('{0} ({1:d}/{2:d})',
self.__describe, self.order_epoch,
self.__last_known_epoch)
else:
return fmt('Empty ref to {0}', self.__describe)
def raise_(self, value):
'''
Log when enabled.
'''
if self.enabled > 0:
if type(value) is StopIteration:
self._info(self.fmt_final_result(fmt('raise {0!r}', value)))
else:
self._warn(self.fmt_final_result(fmt('raise {0!r}', value)))
def __str__(self):
counts = fmt('total: {total:3d}\n'
'leaves: {leaves:3d}\n'
'duplicates: {duplicates:3d}\n', **self.__dict__)
keys = list(self.types.keys())
keys.sort(key=repr)
types = '\n'.join([fmt('{0:40s}: {1:3d}', key, self.types[key])
for key in keys])
return counts + types
def nfa(self):
'''
Generate a NFA-based matcher.
'''
self._debug(fmt('compiling to nfa: {0}', self))
graph = NfaGraph(self.alphabet)
self.expression.build(graph, graph.new_node(), graph.new_node())
self._debug(fmt('nfa graph: {0}', graph))
return NfaPattern(graph, self.alphabet)
def __str__(self):
counts = fmt('total: {total:3d}\n'
'leaves: {leaves:3d}\n'
'duplicates: {duplicates:3d}\n', **self.__dict__)
keys = list(self.types.keys())
keys.sort(key=repr)
types = '\n'.join([fmt('{0:40s}: {1:3d}', key, self.types[key])
for key in keys])
return counts + types
def raise_(self, value):
'''
Log when enabled.
'''
if self.enabled > 0:
if type(value) is StopIteration:
self._info(self.fmt_final_result(fmt('raise {0!r}', value)))
else:
self._warn(self.fmt_final_result(fmt('raise {0!r}', value)))
def before_throw(self, generator, value):
'''
Log when enabled.
'''
if self.enabled > 0:
self.generator = generator
if type(value) is StopIteration:
self.action = fmt('stop -> {0}', generator)
else:
self.action = fmt('{1!r} -> {0}', generator, value)
def pretty(c):
x = self._escape_char(c)
if len(x) > 1 or 32 <= ord(x) <= 127:
return str(x)
elif ord(c) < 0x100:
return fmt('\\x{0:02x}', ord(c))
elif ord(c) < 0x10000:
return fmt('\\u{0:04x}', ord(c))
else:
return fmt('\\U{0:08x}', ord(c))
def pretty(c):
x = self._escape_char(c)
if len(x) > 1 or 32 <= ord(x) <= 127:
return str(x)
elif ord(c) < 0x100:
return fmt('\\x{0:02x}', ord(c))
elif ord(c) < 0x10000:
return fmt('\\u{0:04x}', ord(c))
else:
return fmt('\\U{0:08x}', ord(c))
def before_throw(self, generator, value):
'''
Log when enabled.
'''
if self.enabled > 0:
self.generator = generator
if type(value) is StopIteration:
self.action = fmt('stop -> {0}', generator)
else:
self.action = fmt('{1!r} -> {0}', generator, value)
def __args_as_attributes(self):
'''
Validate the arguments passed to the constructor against the spec for
the factory (necessary because we use *args and so the user doesn't
get the feedback they will expect if they make a mistake). As a side
effect we also associated arguments with names and expand defaults
so that attributes are more predictable.
'''
try:
# function wrapper, so we have two levels, and we must construct
# a new, empty function (ie this is a fake function that helps
# us use the code below, even though there's no arguments because
# factory is a dummy generated in make_factory below)
def empty():
return
document(empty, self.factory.factory)
spec = getargspec(empty)
except:
spec = getargspec(self.factory)
names = list(spec.args)
defaults = dict(
zip(names[::-1], spec.defaults[::-1] if spec.defaults else []))
for name in names:
if name in self.__kargs:
self._karg(**{name: self.__kargs[name]})
del self.__kargs[name]
elif self.__args:
self._arg(**{name: self.__args[0]})
self.__args = self.__args[1:]
elif name in defaults:
self._karg(**{name: defaults[name]})
else:
raise TypeError(
fmt("No value for argument '{0}' in "
"{1}(...)", name, self._small_str))
if self.__args:
if spec.varargs:
self._args(**{spec.varargs: self.__args})
else:
raise TypeError(
fmt(
"No parameter matches the argument "
"{0!r} in {1}(...)", self.__args[0], self._small_str))
if self.__kargs:
if spec.keywords:
self.__kargs(**{spec.keywords: self.__kargs})
else:
name = list(self.__kargs.keys())[0]
value = self.__kargs[name]
raise TypeError(
fmt(
"No parameter matches the argument "
"{0}={1!r} in {2}(...)", name, value, self._small_str))
def dfa(self):
'''
Generate a DFA-based matcher (faster than NFA, but returns only a
single, greedy match).
'''
self._debug(fmt('compiling to dfa: {0}', self))
ngraph = NfaGraph(self.alphabet)
self.expression.build(ngraph, ngraph.new_node(), ngraph.new_node())
self._debug(fmt('nfa graph: {0}', ngraph))
dgraph = NfaToDfa(ngraph, self.alphabet).dfa
self._debug(fmt('dfa graph: {0}', dgraph))
return DfaPattern(dgraph, self.alphabet)
def __str__(self):
lines = []
for node in self:
edges = []
for (dest, edge) in self.transitions(node):
edges.append(fmt('{0}->{1}', edge, dest))
nodes = [n for n in self.nfa_nodes(node)]
edges = ' ' + ','.join(edges) if edges else ''
labels = list(self.terminals(node))
labels = fmt('({0})', ','.join(str(label) for label in labels)) \
if labels else ''
lines.append(fmt('{0}{1}: {2}{3}', node, labels, nodes, edges))
return '; '.join(lines)
def __args_as_attributes(self):
'''
Validate the arguments passed to the constructor against the spec for
the factory (necessary because we use *args and so the user doesn't
get the feedback they will expect if they make a mistake). As a side
effect we also associated arguments with names and expand defaults
so that attributes are more predictable.
'''
try:
# function wrapper, so we have two levels, and we must construct
# a new, empty function (ie this is a fake function that helps
# us use the code below, even though there's no arguments because
# factory is a dummy generated in make_factory below)
def empty(): return
document(empty, self.factory.factory)
spec = getargspec(empty)
except:
spec = getargspec(self.factory)
names = list(spec.args)
defaults = dict(zip(names[::-1], spec.defaults[::-1] if spec.defaults else []))
for name in names:
if name in self.__kargs:
self._karg(**{name: self.__kargs[name]})
del self.__kargs[name]
elif self.__args:
self._arg(**{name: self.__args[0]})
self.__args = self.__args[1:]
elif name in defaults:
self._karg(**{name: defaults[name]})
else:
raise TypeError(fmt("No value for argument '{0}' in "
"{1}(...)",
name, self._small_str))
if self.__args:
if spec.varargs:
self._args(**{spec.varargs: self.__args})
else:
raise TypeError(fmt("No parameter matches the argument "
"{0!r} in {1}(...)",
self.__args[0], self._small_str))
if self.__kargs:
if spec.keywords:
self.__kargs(**{spec.keywords: self.__kargs})
else:
name = list(self.__kargs.keys())[0]
value = self.__kargs[name]
raise TypeError(fmt("No parameter matches the argument "
"{0}={1!r} in {2}(...)",
name, value, self._small_str))
def _fmt(self,
sequence,
offset,
max_len=60,
left='',
right='',
index=True):
'''fmt a possibly long subsection of data.'''
if not sequence:
if index:
return fmt('{0!r}[{1:d}]', sequence, offset)
else:
return fmt('{0!r}', sequence)
if 0 <= offset < len(sequence):
centre = offset
elif offset > 0:
centre = len(sequence) - 1
else:
centre = 0
begin, end = centre, centre + 1
longest = None
while True:
if begin > 0:
if end < len(sequence):
template = '{0!s}...{1!s}...{2!s}'
else:
template = '{0!s}...{1!s}{2!s}'
else:
if end < len(sequence):
template = '{0!s}{1!s}...{2!s}'
else:
template = '{0!s}{1!s}{2!s}'
body = repr(sequence[begin:end])[len(left):]
if len(right):
body = body[:-len(right)]
text = fmt(template, left, body, right, offset)
if index:
text = fmt('{0!s}[{1:d}:]', text, offset)
if longest is None or len(text) <= max_len:
longest = text
if len(text) > max_len:
return longest
begin -= 1
end += 1
if begin < 0 and end > len(sequence):
return longest
begin = max(begin, 0)
end = min(end, len(sequence))
def _format_repr(self, indent, key, contents):
return fmt('{0}{1}{2}({3}{4})',
' ' * indent,
key + '=' if key else '',
self._small_str,
'' if self._fmt_compact else '\n',
',\n'.join(contents))
def __init__(self, matcher, tokens, alphabet, discard,
t_regexp=None, s_regexp=None):
'''
matcher is the head of the original matcher graph, which will be called
with a tokenised stream.
tokens is the set of `Token` instances that define the lexer.
alphabet is the alphabet for which the regexps are defined.
discard is the regular expression for spaces (which are silently
dropped if not token can be matcher).
t_regexp and s_regexp are internally compiled state, used in cloning,
and should not be provided by non-cloning callers.
'''
super(Lexer, self).__init__(TOKENS, TokenNamespace)
if t_regexp is None:
unique = {}
for token in tokens:
token.compile(alphabet)
self._debug(fmt('Token: {0}', token))
# this just reduces the work for the regexp compiler
unique[token.id_] = token
t_regexp = Compiler.multiple(alphabet,
[(t.id_, t.regexp)
for t in unique.values() if t.regexp is not None]).dfa()
if s_regexp is None and discard is not None:
s_regexp = Compiler.single(alphabet, discard).dfa()
self._arg(matcher=matcher)
self._arg(tokens=tokens)
self._arg(alphabet=alphabet)
self._arg(discard=discard)
self._karg(t_regexp=t_regexp)
self._karg(s_regexp=s_regexp)
def build(cls,
node,
regexp,
alphabet,
regexp_type,
use,
add_reqd=False,
wrapper=None):
'''
Construct a container or matcher.
'''
if use and not add_reqd:
matcher = single(alphabet, node, regexp, regexp_type, wrapper)
# if matcher is a Transformable with a Transformation other than
# the standard empty_adapter then we must stop
if len(matcher.wrapper.functions) > 1:
cls.log.debug(fmt('Force matcher: {0}', matcher.wrapper))
return matcher
else:
# not good enough to have a regexp as default, so either force
# the original matcher if it has transforms, or keep going in the
# hope we can get more complex later
matcher = node
if hasattr(matcher, 'wrapper') and matcher.wrapper:
return matcher
return RegexpContainer(matcher, regexp, use, add_reqd)
def RepeatWrapper(matcher, start, stop, step, separator, add, reduce):
'''Parse `step` if it is a string.'''
# Handle circular dependencies
from lepl.matchers.derived import Repeat
try:
int(step) # if this works, we may have a var, so keep the instance
limit = step
algorithm = DEPTH_FIRST
except ValueError:
if (isinstance(step, basestring)):
limit = None
algorithm = None
while step:
match = DIGITS.match(step)
if match:
if limit is None:
limit = int(match.group(1))
step = match.group(2)
else:
raise TypeError(fmt('Cannot parse limit/algorithm for []: {}',
step))
else:
if algorithm is None:
algorithm = step[0]
step = step[1:]
else:
raise TypeError('The step of [...] must be an integer limit, or a '
'string to select the algorithm, or both as a string '
'like "d1" for a single value, depth first')
return Repeat(matcher, start=start, stop=stop, limit=limit,
algorithm=algorithm, separator=separator, add_=add,
reduce=reduce)
def before_next(self, generator):
'''
Log when enabled.
'''
if self.enabled > 0:
self.generator = generator
self.action = fmt('next({0})', generator)
def find_tokens(matcher):
'''
Returns a set of Tokens. Also asserts that children of tokens are
not themselves Tokens.
Should we also check that a Token occurs somewhere on every path to a
leaf node?
'''
(tokens, visited, non_tokens) = (set(), set(), set())
stack = deque([matcher])
while stack:
matcher = stack.popleft()
if matcher not in visited:
if is_child(matcher, NonToken):
non_tokens.add(matcher)
if isinstance(matcher, BaseToken):
tokens.add(matcher)
if matcher.content:
assert_not_token(matcher.content, visited)
else:
for child in matcher:
if isinstance(child, Matcher):
stack.append(child)
visited.add(matcher)
if tokens and non_tokens:
raise LexerError(
fmt('The grammar contains a mix of Tokens and non-Token '
'matchers at the top level. If Tokens are used then '
'non-token matchers that consume input must only '
'appear "inside" Tokens. The non-Token matchers '
'include: {0}.',
'; '.join(str(n) for n in non_tokens)))
return tokens
def clone_transform(use, original, matcher, wrapper):
'''
We can assume that wrapper is a transformation. Add joins into
a sequence.
'''
if original.wrapper:
if original.wrapper.functions[0] is add:
have_add = True
wrapper = original.wrapper.functions[1:]
else:
have_add = False
wrapper = original.wrapper.functions
else:
# punt to next level
return matcher
(use, [regexp]) = \
RegexpContainer.to_regexps(use, [matcher], have_add=have_add)
log.debug(fmt('Transform: cloning {0}', regexp))
return RegexpContainer.build(original,
regexp,
alphabet_,
regexp_type,
use,
add_reqd=False,
wrapper=wrapper)
def __init__(self, conservative=None, left=None, right=None, d=0):
super(AutoMemoize, self).__init__(Rewriter.MEMOIZE,
fmt('AutoMemoize({0}, {1}, {2})', conservative, left, right))
self.conservative = conservative
self.left = left
self.right = right
self.d = d
def __init__(self, matcher, tokens, alphabet, discard,
t_regexp=None, s_regexp=None):
'''
matcher is the head of the original matcher graph, which will be called
with a tokenised stream.
tokens is the set of `Token` instances that define the lexer.
alphabet is the alphabet for which the regexps are defined.
discard is the regular expression for spaces (which are silently
dropped if not token can be matcher).
t_regexp and s_regexp are internally compiled state, used in cloning,
and should not be provided by non-cloning callers.
'''
super(Lexer, self).__init__(TOKENS, TokenNamespace)
if t_regexp is None:
unique = {}
for token in tokens:
token.compile(alphabet)
self._debug(fmt('Token: {0}', token))
# this just reduces the work for the regexp compiler
unique[token.id_] = token
t_regexp = Compiler.multiple(alphabet,
[(t.id_, t.regexp)
for t in unique.values() if t.regexp is not None]).dfa()
if s_regexp is None and discard is not None:
s_regexp = Compiler.single(alphabet, discard).dfa()
self._arg(matcher=matcher)
self._arg(tokens=tokens)
self._arg(alphabet=alphabet)
self._arg(discard=discard)
self._karg(t_regexp=t_regexp)
self._karg(s_regexp=s_regexp)
def new_clone(i, j, node, args, kargs):
type_, ok = None, False
for parent in self.spec:
if is_child(node, parent):
type_ = self.spec[parent]
if type_:
ok = True
for arg in args:
if isinstance(arg, Matcher) and not \
isinstance(arg, NoTrampoline):
ok = False
for name in kargs:
arg = kargs[name]
if isinstance(arg, Matcher) and not \
isinstance(arg, NoTrampoline):
ok = False
if not ok:
type_ = type(node)
try:
copy = type_(*args, **kargs)
copy_standard_attributes(node, copy)
return copy
except TypeError as err:
raise TypeError(
fmt('Error cloning {0} with ({1}, {2}): {3}', type_, args,
kargs, err))
def __init__(self, type_, **extra_kargs):
super(SetArguments,
self).__init__(Rewriter.SET_ARGUMENTS,
fmt('SetArguments({0}, {1})', type_, extra_kargs),
False)
self.type = type_
self.extra_kargs = extra_kargs
def before_next(self, generator):
'''
Log when enabled.
'''
if self.enabled > 0:
self.generator = generator
self.action = fmt('next({0})', generator)
def push_level(self, level):
'''
Add a new indent level.
'''
self.__stack.append(level)
self.__state[BlockMonitor] = level
self._debug(fmt('Indent -> {0:d}', level))
def before_send(self, generator, value):
'''
Log when enabled.
'''
if self.enabled > 0:
self.generator = generator
self.action = fmt('{1!r} -> {0}', generator, value)
def push_level(self, level):
'''
Add a new indent level.
'''
self.__stack.append(level)
self.__state[BlockMonitor] = level
self._debug(fmt('Indent -> {0:d}', level))
def __init__(self, conservative=None, left=None, right=None, d=0):
super(AutoMemoize, self).__init__(Rewriter.MEMOIZE,
fmt('AutoMemoize({0}, {1}, {2})', conservative, left, right))
self.conservative = conservative
self.left = left
self.right = right
self.d = d
def TraceVariables(on=True, show_failures=True, width=80, out=stderr):
'''
Add this as a context (`with TraceVariables():`) and you will see
debug logging indicating how variables are bound during matching.
'''
if on:
before = _getframe(2).f_locals.copy()
yield None
if on:
after = _getframe(2).f_locals
for key in after:
value = after[key]
if key not in before or value != before[key]:
try:
try:
value.wrapper.append(
name(key, show_failures, width, out))
except AttributeError:
value.trace_variables = name(key, show_failures, width,
out)
except: # what exception?
print(
'Unfortunately the following matchers cannot '
'be tracked:',
end=str('\n'))
print(fmt(' {0} = {1}', key, value), end=str('\n'))
def new_clone(i, j, node, args, kargs):
type_, ok = None, False
for parent in self.spec:
if is_child(node, parent):
type_ = self.spec[parent]
if type_:
ok = True
for arg in args:
if isinstance(arg, Matcher) and not \
isinstance(arg, NoTrampoline):
ok = False
for name in kargs:
arg = kargs[name]
if isinstance(arg, Matcher) and not \
isinstance(arg, NoTrampoline):
ok = False
if not ok:
type_ = type(node)
try:
copy = type_(*args, **kargs)
copy_standard_attributes(node, copy)
return copy
except TypeError as err:
raise TypeError(fmt('Error cloning {0} with ({1}, {2}): {3}',
type_, args, kargs, err))
def before_send(self, generator, value):
'''
Log when enabled.
'''
if self.enabled > 0:
self.generator = generator
self.action = fmt('{1!r} -> {0}', generator, value)
def apply_modifiers(func, args, kargs, modifiers, margs, mkargs):
'''
Modify values in args and kargs.
'''
spec = getargspec(func)
names = list(spec.args)
defaults = dict(
zip(names[::-1], spec.defaults[::-1] if spec.defaults else []))
newargs = []
newkargs = {}
for name in names:
if name in kargs:
value = kargs[name]
if name in modifiers:
value = modifiers[name](value)
newkargs[name] = value
del kargs[name]
elif args:
(value, args) = (args[0], args[1:])
if name in modifiers:
value = modifiers[name](value)
newargs.append(value)
elif name in defaults:
value = defaults[name]
if name in modifiers:
value = modifiers[name](value)
newkargs[name] = value
else:
raise TypeError(
fmt("No value for argument '{0}' in "
"{1}(...)", name, func.__name__))
# copy across varags
if spec.varargs:
newargs.extend(map(margs, args))
elif args:
raise TypeError(
fmt("Unexpected argument {0!r} for {1}(...)", args[0],
func.__name__))
if spec.keywords:
for name in kargs:
newkargs[name] = mkargs(kargs[name])
elif kargs:
for name in kargs:
raise TypeError(
fmt("Unexpected argument {0}={1!r} for {2}(...)", name,
kargs[name], func.__name__))
return (newargs, newkargs)
