def define_molecule_parser(self): """ Defines specific parsers for BNG molecules """ # molecules can have tags self.parsers.tag = pp.Combine( pp.Word("%") + (self.parsers.base_name ^ pp.Word(pp.nums))) # and compartments self.parsers.compartment = pp.Combine( pp.Word("@") + self.parsers.base_name) # combine tag and compartment tag_comp = (pp.Optional(self.parsers.tag) + pp.Optional(self.parsers.compartment)) ^ ( pp.Optional(self.parsers.compartment) + pp.Optional(self.parsers.tag)) # full molecule self.parsers.molecule = ( self.parsers.base_name + tag_comp + pp.Word("(") + pp.Optional(self.parsers.combined_components_parser) + pp.Word(")") + tag_comp) molecule_parser = pp.Combine(self.parsers.molecule) # molecules # components are separated by commas molecule_separator = pp.Word(".") self.parsers.molecules_parser = pp.delimited_list( molecule_parser, delim=molecule_separator) self.parsers.combined_molecules_parser = pp.delimited_list( molecule_parser, delim=molecule_separator, combine=True)
def make_list_t(scalars: Union[Any, List[Any]], *, start: str = "[", end: str = "]", delimiter: str = ",", throw_if_empty: bool = True, multiline: bool = True) -> pp.ParserElement: """Atom for lists. Parameters ---------- scalars: Union[Any, List[Any]] Scalar parser elements, already combined or as a list. The list will be combined using the `^` operator. start: str Left delimiter for the list. Defaults to '['. end: str Right delimiter for the list. Defaults to ']'. delimiter: str List delimiter. Defaults to ','. multiline: bool Whether the list can span multiple lines. Defaults to `True`. Notes ----- The order of the scalar tokens in the `scalars` list **is important**. Returns ------- list: Any """ START, END = map(pp.Suppress, start + end) if isinstance(scalars, list): atoms = functools.reduce(lambda x, y: x ^ y, scalars) else: atoms = scalars if multiline: NEWLINE = pp.Literal("\n").suppress() list_t = (START + pp.Optional( pp.delimited_list(atoms ^ NEWLINE, delim=delimiter)) + END) else: list_t = START + pp.Optional(pp.delimited_list(atoms, delim=delimiter)) + END return list_t.add_condition(bool, message="Empty lists not allowed", fatal=throw_if_empty)
def __init__(self): # build grammar according to Glue.Client.get_partitions(Expression) lpar, rpar = map(Suppress, "()") # NOTE these are AWS Athena column name best practices ident = Forward().set_name("ident") ident <<= Word(alphanums + "._").set_parse_action(_Ident) | lpar + ident + rpar number = Forward().set_name("number") number <<= pyparsing_common.number | lpar + number + rpar string = Forward().set_name("string") string <<= QuotedString(quote_char="'", esc_quote="''") | lpar + string + rpar literal = (number | string).set_name("literal") literal_list = delimited_list(literal, min=1).set_name("list") bin_op = one_of("<> >= <= > < =").set_name("binary op") and_ = Forward() and_ <<= CaselessKeyword("and") | lpar + and_ + rpar or_ = Forward() or_ <<= CaselessKeyword("or") | lpar + or_ + rpar in_, between, like, not_, is_, null = map( CaselessKeyword, "in between like not is null".split() ) not_ = Suppress(not_) # only needed for matching cond = ( (ident + is_ + null).set_parse_action(_IsNull) | (ident + is_ + not_ + null).set_parse_action(_IsNotNull) | (ident + bin_op + literal).set_parse_action(_BinOp) | (ident + like + string).set_parse_action(_Like) | (ident + not_ + like + string).set_parse_action(_NotLike) | (ident + in_ + lpar + literal_list + rpar).set_parse_action(_In) | (ident + not_ + in_ + lpar + literal_list + rpar).set_parse_action(_NotIn) | (ident + between + literal + and_ + literal).set_parse_action(_Between) | (ident + not_ + between + literal + and_ + literal).set_parse_action( _NotBetween ) ).set_name("cond") # conditions can be joined using 2-ary AND and/or OR expr = infix_notation( cond, [ (and_, 2, OpAssoc.LEFT, _BoolAnd), (or_, 2, OpAssoc.LEFT, _BoolOr), ], ) self._expr = expr.set_name("expr") self._cache: Dict[str, _Expr] = {}
def define_component_parser(self): """ Defines specific parsers for BNG components """ # bng names are alpha numericals and _1 self.parsers.base_name = pp.Word(pp.alphas, pp.alphanums + "_") # components have optional states and bonds self.parsers.state = pp.Combine( pp.Word("~") + (self.parsers.base_name ^ pp.Word(pp.nums))) ^ pp.Word("~?") self.parsers.bond = pp.Combine((pp.Word("!") + pp.Word(pp.nums)) ^ (pp.Word("!?")) ^ (pp.Word("!+"))) self.parsers.component = (self.parsers.base_name + pp.Optional(self.parsers.state) + pp.Optional(self.parsers.bond)) component_parser = pp.Combine(self.parsers.component) # components are separated by commas component_separator = pp.Word(",") self.parsers.components_parser = pp.delimited_list( component_parser, delim=component_separator) self.parsers.combined_components_parser = pp.delimited_list( component_parser, delim=component_separator, combine=True)
class Mapping(Fragment): """ Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input sections of the entity will end up. [mapping:<name>] archive: lib1.a entries: obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ... obj2 (scheme2) ... Ultimately, an `entity (scheme)` entry generates an input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html) in the output linker script. It is possible to attach 'flags' to the `entity (scheme)` to generate different output commands or to emit additional keywords in the generated input section description. The input section description, as well as other output commands, is defined in output_commands.py. """ _any = Literal('*') _obj = Word(alphas + '_', alphanums + '-_').set_results_name('object') _sym = Fragment.IDENTIFIER.set_results_name('symbol') # There are three possible patterns for mapping entries: # obj:symbol (scheme) # obj (scheme) # * (scheme) _entry = (((_obj + Opt(Suppress(':') + _sym)) | _any.set_results_name('object')) + Suppress('(') + Fragment.IDENTIFIER.set_results_name('section') + Suppress(')')) ENTRY = _entry + LineEnd().suppress() ARCHIVE = (Word(alphanums + '.-_$+') | Literal('*')) + LineEnd().suppress() # Flags can be specified for section->target in the scheme specified, ex: # obj (scheme); # section->target SURROUND(symbol), # section2->target2 ALIGN(4) ENTRY_WITH_FLAG = (_entry + Suppress(';') + delimited_list(Flag.FLAG.set_parse_action(Flag.parse))) def __init__(self, archive: str, flags: Dict[Any, Flag], *args, **kwargs): super().__init__(*args, **kwargs) self.archive = archive self.flags = flags @staticmethod def parse_archive(s, loc, toks): this = toks[0][0] if len(this) != 1: raise ParseFatalException(s, loc, 'Could only specify one archive file in one mapping fragment') return this[0] @staticmethod def parse_entry(toks): return toks.object, toks.symbol or None, toks.section @staticmethod def parse_entry_with_flag(toks): entry = toks.object, toks.symbol or None, toks.section return { entry: [tok for tok in toks if isinstance(tok, Flag)] } @staticmethod def parse_entries(toks): return toks[0] @staticmethod def parse(toks): this = toks[0] name = this[0] archive = this[1] entries_or_dict_with_flags = this[2] entries = set() flags = dict() for item in entries_or_dict_with_flags: if isinstance(item, Empty): continue elif isinstance(item, dict): # entry with flags for k, v in item.items(): entries.add(k) if k in flags: flags[k].extend(v) else: flags[k] = v else: entries.add(item) return Mapping(archive=archive, name=name, entries=entries, flags=flags)
def rc_statement(): """ Generate a RC statement parser that can be used to parse a RC file :rtype: pyparsing.ParserElement """ one_line_comment = "//" + rest_of_line comments = c_style_comment ^ one_line_comment precompiler = AtLineStart(Word("#", alphanums) + rest_of_line) language_definition = ( "LANGUAGE" + Word(alphas + "_").set_results_name("language") + Optional("," + Word(alphas + "_").set_results_name("sublanguage"))) block_start = (Keyword("{") | Keyword("BEGIN")).set_name("block_start") block_end = (Keyword("}") | Keyword("END")).set_name("block_end") name_id = Group(Word(alphas, alphanums + "_")).set_name("name_id") numbers = Word(nums) integerconstant = numbers ^ Combine("0x" + numbers) constant = Combine( Optional(Keyword("NOT")) + (name_id | integerconstant), adjacent=False, join_string=" ", ) combined_constants = delimited_list(constant, "|") concatenated_string = OneOrMore(quoted_string) block_options = Optional( SkipTo(Keyword("CAPTION"), fail_on=block_start)("pre_caption") + Keyword("CAPTION") + quoted_string("caption")) + SkipTo(block_start)("post_caption") undefined_control = (Group( name_id.set_results_name("id_control") + delimited_list(concatenated_string ^ constant ^ numbers ^ Group( combined_constants)).set_results_name("values_")) | comments) block = (block_start + ZeroOrMore(undefined_control, stop_on=block_end)("controls") + block_end) dialog = (name_id("block_id") + (Keyword("DIALOGEX") | Keyword("DIALOG"))("block_type") + block_options + block) string_table = Keyword("STRINGTABLE")("block_type") + block_options + block menu_item = Keyword("MENUITEM")("block_type") + ( pyparsing_common.comma_separated_list("values_") | Keyword("SEPARATOR")) popup_block = Forward() popup_block <<= Group( Keyword("POPUP")("block_type") + Optional(quoted_string("caption")) + block_start + ZeroOrMore(Group(menu_item | popup_block), stop_on=block_end)("elements") + block_end)("popups*") menu = (name_id("block_id") + Keyword("MENU")("block_type") + block_options + block_start + ZeroOrMore(popup_block, stop_on=block_end) + block_end) return comments ^ precompiler ^ language_definition ^ dialog ^ string_table ^ menu
class TestCommonHelperExpressions(PyparsingExpressionTestCase): tests = [ PpTestSpec( desc="A comma-delimited list of words", expr=pp.delimited_list(pp.Word(pp.alphas)), text="this, that, blah,foo, bar", expected_list=["this", "that", "blah", "foo", "bar"], ), PpTestSpec( desc="A counted array of words", expr=pp.Group(pp.counted_array(pp.Word("ab")))[...], text="2 aaa bbb 0 3 abab bbaa abbab", expected_list=[["aaa", "bbb"], [], ["abab", "bbaa", "abbab"]], ), PpTestSpec( desc="skipping comments with ignore", expr=(pp.pyparsing_common.identifier("lhs") + "=" + pp.pyparsing_common.fnumber("rhs")).ignore( pp.cpp_style_comment), text="abc_100 = /* value to be tested */ 3.1416", expected_list=["abc_100", "=", 3.1416], expected_dict={ "lhs": "abc_100", "rhs": 3.1416 }, ), PpTestSpec( desc= "some pre-defined expressions in pyparsing_common, and building a dotted identifier with delimted_list", expr=(pp.pyparsing_common.number("id_num") + pp.delimitedList( pp.pyparsing_common.identifier, ".", combine=True)("name") + pp.pyparsing_common.ipv4_address("ip_address")), text="1001 www.google.com 192.168.10.199", expected_list=[1001, "www.google.com", "192.168.10.199"], expected_dict={ "id_num": 1001, "name": "www.google.com", "ip_address": "192.168.10.199", }, ), PpTestSpec( desc= "using one_of (shortcut for Literal('a') | Literal('b') | Literal('c'))", expr=pp.one_of("a b c")[...], text="a b a b b a c c a b b", expected_list=[ "a", "b", "a", "b", "b", "a", "c", "c", "a", "b", "b" ], ), PpTestSpec( desc="parsing nested parentheses", expr=pp.nested_expr(), text="(a b (c) d (e f g ()))", expected_list=[["a", "b", ["c"], "d", ["e", "f", "g", []]]], ), PpTestSpec( desc="parsing nested braces", expr=(pp.Keyword("if") + pp.nested_expr()("condition") + pp.nested_expr("{", "}")("body")), text='if ((x == y) || !z) {printf("{}");}', expected_list=[ "if", [["x", "==", "y"], "||", "!z"], ["printf(", '"{}"', ");"], ], expected_dict={ "condition": [[["x", "==", "y"], "||", "!z"]], "body": [["printf(", '"{}"', ");"]], }, ), ]
class TestRepetition(PyparsingExpressionTestCase): tests = [ PpTestSpec( desc="Match several words", expr=(pp.Word("x") | pp.Word("y"))[...], text="xxyxxyyxxyxyxxxy", expected_list=[ "xx", "y", "xx", "yy", "xx", "y", "x", "y", "xxx", "y" ], ), PpTestSpec( desc="Match several words, skipping whitespace", expr=(pp.Word("x") | pp.Word("y"))[...], text="x x y xxy yxx y xyx xxy", expected_list=[ "x", "x", "y", "xx", "y", "y", "xx", "y", "x", "y", "x", "xx", "y", ], ), PpTestSpec( desc="Match several words, skipping whitespace (old style)", expr=pp.OneOrMore(pp.Word("x") | pp.Word("y")), text="x x y xxy yxx y xyx xxy", expected_list=[ "x", "x", "y", "xx", "y", "y", "xx", "y", "x", "y", "x", "xx", "y", ], ), PpTestSpec( desc= "Match words and numbers - show use of results names to collect types of tokens", expr=(pp.Word(pp.alphas)("alpha*") | pp.pyparsing_common.integer("int*"))[...], text="sdlfj23084ksdfs08234kjsdlfkjd0934", expected_list=["sdlfj", 23084, "ksdfs", 8234, "kjsdlfkjd", 934], expected_dict={ "alpha": ["sdlfj", "ksdfs", "kjsdlfkjd"], "int": [23084, 8234, 934], }, ), PpTestSpec( desc="Using delimited_list (comma is the default delimiter)", expr=pp.delimited_list(pp.Word(pp.alphas)), text="xxyx,xy,y,xxyx,yxx, xy", expected_list=["xxyx", "xy", "y", "xxyx", "yxx", "xy"], ), PpTestSpec( desc= "Using delimited_list (comma is the default delimiter) with trailing delimiter", expr=pp.delimited_list(pp.Word(pp.alphas), allow_trailing_delim=True), text="xxyx,xy,y,xxyx,yxx, xy,", expected_list=["xxyx", "xy", "y", "xxyx", "yxx", "xy"], ), PpTestSpec( desc= "Using delimited_list (comma is the default delimiter) with minimum size", expr=pp.delimited_list(pp.Word(pp.alphas), min=3), text="xxyx,xy", expected_fail_locn=7, ), PpTestSpec( desc= "Using delimited_list (comma is the default delimiter) with maximum size", expr=pp.delimited_list(pp.Word(pp.alphas), max=3), text="xxyx,xy,y,xxyx,yxx, xy,", expected_list=["xxyx", "xy", "y"], ), PpTestSpec( desc="Using delimited_list, with ':' delimiter", expr=pp.delimited_list(pp.Word(pp.hexnums, exact=2), delim=":", combine=True), text="0A:4B:73:21:FE:76", expected_list=["0A:4B:73:21:FE:76"], ), PpTestSpec( desc="Using delimited_list, with ':' delimiter", expr=pp.delimited_list( pp.Word(pp.hexnums, exact=2), delim=":", combine=True, allow_trailing_delim=True, ), text="0A:4B:73:21:FE:76:", expected_list=["0A:4B:73:21:FE:76:"], ), ]
braced_text = Forward() braced_text_elems = Forward() braced_text << Suppress('{') + braced_text_elems + Suppress('}') braced_text.set_parse_action(create1(BracedText)) braced_text_elems << ZeroOrMore(nonbraced_text | braced_text) braced_text_elems.set_parse_action(create_rec(BracedTextElements)) quoted_text = QuotedString('"', '\\', unquoteResults=False, multiline=True) quoted_text.set_parse_action(create1(QuotedText)) entry_key = spaces + CharsNotIn('#\\@,={} \t\n') + spaces entry_key.set_parse_action(create1(EntryKey)) tag_value_elems = delimited_list(braced_text | quoted_text | entry_key, '#') tag_value_elems.set_parse_action(create_rec(TagValueElements)) tag_value = tag_value_elems tag_value.set_parse_action(create1(TagValue)) tag_name = spaces + CharsNotIn('#\\@,={} \t\n') + spaces tag_name.set_parse_action(create1(TagName)) tag = tag_name + Suppress('=') + tag_value tag.set_parse_action(create2(Tag)) tags = delimited_list(tag, ',', allow_trailing_delim=True) tags.set_parse_action(create_rec(Tags)) entry_body = entry_key + Optional(Suppress(',') + Optional(tags))
# - invalid_texture_functions.test: no GL texture functions in Runtime Effects # - preprocessor.test: no preprocessor in SkSL import os import pyparsing as pp import re import sys # Each case can contain expected input/output values, sometimes in [bracketed|lists] and # sometimes not. # GLSL code appears in ""double-double quotes"" and is indicated as vert/frag-specific or "both". # Some tests denote that they are expected to pass/fail. (Presumably, "pass" is the default.) # We ignore descriptions and version fields. wordWithUnderscores = pp.Word(pp.alphanums + '_') pipeList = pp.delimited_list(pp.SkipTo(pp.Literal("|") | pp.Literal("]")), delim="|") bracketedPipeList = pp.Group(pp.Literal("[").suppress() + pipeList + pp.Literal("]").suppress()) unbracketedValue = pp.Group(pp.SkipTo(";")) valueList = (pp.Word(pp.alphanums) + # type pp.Word(pp.alphanums) + # varname pp.Literal("=").suppress() + (bracketedPipeList | unbracketedValue) + pp.Literal(";").suppress()) value = pp.Group((pp.Keyword("input") | pp.Keyword("output") | pp.Keyword("uniform")) + valueList) values = (pp.Keyword("values") + pp.Literal("{").suppress() + pp.ZeroOrMore(value) +