def test_sv_search(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv.search(re.compile(r'5'))
assert not sv.search(re.compile(r' '))
assert sv[5:].search(re.compile(r'5'))
assert not sv[:9].search(re.compile(r'9'))
def test_sv_match(self):
s = " 0123456789 "
sv = StringView(s, 1, -1)
assert sv.match(re.compile(r'\d'))
assert sv.match(re.compile(r'\d+'))
assert not sv.match(re.compile(r' '))
assert sv[4:].match(re.compile(r'45'))
def gen_find_include_func(rx: Union[str, Any],
comment_rx: Optional[Union[str, Any]] = None) -> FindIncludeFunc:
if isinstance(rx, str): rx = re.compile(rx)
if isinstance(comment_rx, str): comment_rx = re.compile(comment_rx)
def find_include(text: str, begin: int) -> IncludeInfo:
nonlocal rx
m = rx.search(text, begin)
if m:
begin = m.start()
return IncludeInfo(begin, m.end() - begin, m.group('name'))
else:
return IncludeInfo(-1, 0, '')
def find_comment(text: str, begin: int) -> Tuple[int, int]:
m = comment_rx.search(text, begin)
return m.span() if m else (-1, -2)
def meta_find_include(text: str, begin: int) -> IncludeInfo:
a, b = find_comment(text, begin)
info = find_include(text, begin)
k, length, name = info
while a < b <= k:
a, b = find_comment(text, b)
while (a < k < b) or (a < k + length < b):
info = find_include(text, b)
k, length, name = info
while a < b <= k:
a, b = find_comment(text, b)
return info
return find_include if comment_rx is None else meta_find_include
def test_generate_find_include_w_comments(self):
rx = re.compile(r'include\((?P<name>[^)\n]*)\)')
comment_rx = re.compile(r'#.*(?:\n|$)')
find = gen_find_include_func(rx, comment_rx)
test = '''a
b # include(alpha)
c include(beta)
# include(gamma)'''
info = find(test, 0)
assert info.file_name == "beta"
info = find(test, info.begin + info.length)
assert info.begin < 0
from DHParser.syntaxtree import Node, RootNode, parse_tree, flatten_sxpr, ZOMBIE_TAG
from DHParser.trace import set_tracer, all_descendants, trace_history
from DHParser.transform import traverse, remove_children
from DHParser.toolkit import load_if_file, re, re_find, concurrent_ident, instantiate_executor
__all__ = ('unit_from_config', 'unit_from_json', 'TEST_READERS',
'unit_from_file', 'get_report', 'TEST_ARTIFACT',
'POSSIBLE_ARTIFACTS', 'grammar_unit', 'unique_name',
'grammar_suite', 'SymbolsDictType', 'extract_symbols',
'create_test_templates', 'reset_unit', 'runner', 'clean_report',
'read_full_content', 'add_header', 'stdio', 'MockStream')
UNIT_STAGES = {'match*', 'match', 'fail', 'ast', 'cst'}
RESULT_STAGES = {'__cst__', '__ast__', '__err__'}
RX_SECTION = re.compile(r'\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]')
RE_VALUE = '(?:"""((?:.|\n)*?)""")|' + "(?:'''((?:.|\n)*?)''')|" + \
r'(?:"(.*?)")|' + "(?:'(.*?)')|" + r'(.*(?:\n(?:\s*\n)* .*)*)'
# the following does not work with pypy3, because pypy's re-engine does not
# support local flags, e.g. '(?s: )'
# RE_VALUE = r'(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \
# r'(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)* .*)*)'
RX_ENTRY = re.compile(
r'\s*(\w+\*?)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE))
RX_COMMENT = re.compile(r'\s*[#;].*(?:\n|$)')
def unit_from_config(config_str, filename):
""" Reads grammar unit tests contained in a file in config file (.ini)
syntax.
Reads a script-template from a template file named `template_name`
in the template-directory and returns it as a string.
"""
with open(os.path.join(DHPARSER_DIR, 'templates', template_name), 'r', encoding='utf-8') as f:
return f.read()
SECTION_MARKER = """\n
#######################################################################
#
# {marker}
#
#######################################################################
\n"""
RX_SECTION_MARKER = re.compile(SECTION_MARKER.format(marker=r'.*?SECTION.*?'))
RX_WHITESPACE = re.compile(r'\s*')
SYMBOLS_SECTION = "SYMBOLS SECTION - Can be edited. Changes will be preserved."
PREPROCESSOR_SECTION = "PREPROCESSOR SECTION - Can be edited. Changes will be preserved."
PARSER_SECTION = "PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!"
AST_SECTION = "AST SECTION - Can be edited. Changes will be preserved."
COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved."
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
class DSLException(Exception):
"""
Base class for DSL-exceptions.
"""
def __init__(self, errors: Union[Sequence[Error], Iterator[Error]]):
def test_generate_find_include_func(self):
rx = re.compile(r'include\((?P<name>[^)\n]*)\)')
find = gen_find_include_func(rx)
info = find('''321include(sub.txt)xyz''', 0)
assert info == IncludeInfo(3, 16, 'sub.txt')
def test_EMPTY_STRING_VIEW(self):
assert len(EMPTY_STRING_VIEW) == 0
assert EMPTY_STRING_VIEW.find('x') < 0
assert not EMPTY_STRING_VIEW.match(re.compile(r'x'))
assert EMPTY_STRING_VIEW.match(re.compile(r'.*'))
assert len(EMPTY_STRING_VIEW[0:1]) == 0
'gen_find_include_func',
'preprocess_includes')
#######################################################################
#
# Types and constants
#
#######################################################################
BEGIN_TOKEN = '\x1b'
TOKEN_DELIMITER = '\x1c'
END_TOKEN = '\x1d'
RESERVED_TOKEN_CHARS = BEGIN_TOKEN + TOKEN_DELIMITER + END_TOKEN
RX_TOKEN_NAME = re.compile(r'\w+')
RX_TOKEN_ARGUMENT = re.compile(r'[^\x1b\x1c\x1d]*')
RX_TOKEN = re.compile(r'\x1b(?P<name>\w+)\x1c(?P<argument>[^\x1b\x1c\x1d]*)\x1d')
# class IncludeInfo(NamedTuple):
# begin: int
# length: int
# file_name: str
# collections.namedtuple needed for Cython compatbility
IncludeInfo = namedtuple('IncludeInfo',
['begin', # type: int
'length', # type: int
'file_name' # type: str
], module = __name__)