def test_sv_search(self): s = " 0123456789 " sv = StringView(s, 1, -1) assert sv.search(re.compile(r'5')) assert not sv.search(re.compile(r' ')) assert sv[5:].search(re.compile(r'5')) assert not sv[:9].search(re.compile(r'9'))
def test_sv_match(self): s = " 0123456789 " sv = StringView(s, 1, -1) assert sv.match(re.compile(r'\d')) assert sv.match(re.compile(r'\d+')) assert not sv.match(re.compile(r' ')) assert sv[4:].match(re.compile(r'45'))
def gen_find_include_func(rx: Union[str, Any], comment_rx: Optional[Union[str, Any]] = None) -> FindIncludeFunc: if isinstance(rx, str): rx = re.compile(rx) if isinstance(comment_rx, str): comment_rx = re.compile(comment_rx) def find_include(text: str, begin: int) -> IncludeInfo: nonlocal rx m = rx.search(text, begin) if m: begin = m.start() return IncludeInfo(begin, m.end() - begin, m.group('name')) else: return IncludeInfo(-1, 0, '') def find_comment(text: str, begin: int) -> Tuple[int, int]: m = comment_rx.search(text, begin) return m.span() if m else (-1, -2) def meta_find_include(text: str, begin: int) -> IncludeInfo: a, b = find_comment(text, begin) info = find_include(text, begin) k, length, name = info while a < b <= k: a, b = find_comment(text, b) while (a < k < b) or (a < k + length < b): info = find_include(text, b) k, length, name = info while a < b <= k: a, b = find_comment(text, b) return info return find_include if comment_rx is None else meta_find_include
def test_generate_find_include_w_comments(self): rx = re.compile(r'include\((?P<name>[^)\n]*)\)') comment_rx = re.compile(r'#.*(?:\n|$)') find = gen_find_include_func(rx, comment_rx) test = '''a b # include(alpha) c include(beta) # include(gamma)''' info = find(test, 0) assert info.file_name == "beta" info = find(test, info.begin + info.length) assert info.begin < 0
from DHParser.syntaxtree import Node, RootNode, parse_tree, flatten_sxpr, ZOMBIE_TAG from DHParser.trace import set_tracer, all_descendants, trace_history from DHParser.transform import traverse, remove_children from DHParser.toolkit import load_if_file, re, re_find, concurrent_ident, instantiate_executor __all__ = ('unit_from_config', 'unit_from_json', 'TEST_READERS', 'unit_from_file', 'get_report', 'TEST_ARTIFACT', 'POSSIBLE_ARTIFACTS', 'grammar_unit', 'unique_name', 'grammar_suite', 'SymbolsDictType', 'extract_symbols', 'create_test_templates', 'reset_unit', 'runner', 'clean_report', 'read_full_content', 'add_header', 'stdio', 'MockStream') UNIT_STAGES = {'match*', 'match', 'fail', 'ast', 'cst'} RESULT_STAGES = {'__cst__', '__ast__', '__err__'} RX_SECTION = re.compile(r'\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]') RE_VALUE = '(?:"""((?:.|\n)*?)""")|' + "(?:'''((?:.|\n)*?)''')|" + \ r'(?:"(.*?)")|' + "(?:'(.*?)')|" + r'(.*(?:\n(?:\s*\n)* .*)*)' # the following does not work with pypy3, because pypy's re-engine does not # support local flags, e.g. '(?s: )' # RE_VALUE = r'(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \ # r'(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)* .*)*)' RX_ENTRY = re.compile( r'\s*(\w+\*?)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE)) RX_COMMENT = re.compile(r'\s*[#;].*(?:\n|$)') def unit_from_config(config_str, filename): """ Reads grammar unit tests contained in a file in config file (.ini) syntax.
Reads a script-template from a template file named `template_name` in the template-directory and returns it as a string. """ with open(os.path.join(DHPARSER_DIR, 'templates', template_name), 'r', encoding='utf-8') as f: return f.read() SECTION_MARKER = """\n ####################################################################### # # {marker} # ####################################################################### \n""" RX_SECTION_MARKER = re.compile(SECTION_MARKER.format(marker=r'.*?SECTION.*?')) RX_WHITESPACE = re.compile(r'\s*') SYMBOLS_SECTION = "SYMBOLS SECTION - Can be edited. Changes will be preserved." PREPROCESSOR_SECTION = "PREPROCESSOR SECTION - Can be edited. Changes will be preserved." PARSER_SECTION = "PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!" AST_SECTION = "AST SECTION - Can be edited. Changes will be preserved." COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved." END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS" class DSLException(Exception): """ Base class for DSL-exceptions. """ def __init__(self, errors: Union[Sequence[Error], Iterator[Error]]):
def test_generate_find_include_func(self): rx = re.compile(r'include\((?P<name>[^)\n]*)\)') find = gen_find_include_func(rx) info = find('''321include(sub.txt)xyz''', 0) assert info == IncludeInfo(3, 16, 'sub.txt')
def test_EMPTY_STRING_VIEW(self): assert len(EMPTY_STRING_VIEW) == 0 assert EMPTY_STRING_VIEW.find('x') < 0 assert not EMPTY_STRING_VIEW.match(re.compile(r'x')) assert EMPTY_STRING_VIEW.match(re.compile(r'.*')) assert len(EMPTY_STRING_VIEW[0:1]) == 0
'gen_find_include_func', 'preprocess_includes') ####################################################################### # # Types and constants # ####################################################################### BEGIN_TOKEN = '\x1b' TOKEN_DELIMITER = '\x1c' END_TOKEN = '\x1d' RESERVED_TOKEN_CHARS = BEGIN_TOKEN + TOKEN_DELIMITER + END_TOKEN RX_TOKEN_NAME = re.compile(r'\w+') RX_TOKEN_ARGUMENT = re.compile(r'[^\x1b\x1c\x1d]*') RX_TOKEN = re.compile(r'\x1b(?P<name>\w+)\x1c(?P<argument>[^\x1b\x1c\x1d]*)\x1d') # class IncludeInfo(NamedTuple): # begin: int # length: int # file_name: str # collections.namedtuple needed for Cython compatbility IncludeInfo = namedtuple('IncludeInfo', ['begin', # type: int 'length', # type: int 'file_name' # type: str ], module = __name__)