def define_molecule_parser(self):
"""
Defines specific parsers for BNG molecules
"""
# molecules can have tags
self.parsers.tag = pp.Combine(
pp.Word("%") + (self.parsers.base_name ^ pp.Word(pp.nums)))
# and compartments
self.parsers.compartment = pp.Combine(
pp.Word("@") + self.parsers.base_name)
# combine tag and compartment
tag_comp = (pp.Optional(self.parsers.tag) +
pp.Optional(self.parsers.compartment)) ^ (
pp.Optional(self.parsers.compartment) +
pp.Optional(self.parsers.tag))
# full molecule
self.parsers.molecule = (
self.parsers.base_name + tag_comp + pp.Word("(") +
pp.Optional(self.parsers.combined_components_parser) +
pp.Word(")") + tag_comp)
molecule_parser = pp.Combine(self.parsers.molecule)
# molecules
# components are separated by commas
molecule_separator = pp.Word(".")
self.parsers.molecules_parser = pp.delimited_list(
molecule_parser, delim=molecule_separator)
self.parsers.combined_molecules_parser = pp.delimited_list(
molecule_parser, delim=molecule_separator, combine=True)
def make_list_t(scalars: Union[Any, List[Any]],
*,
start: str = "[",
end: str = "]",
delimiter: str = ",",
throw_if_empty: bool = True,
multiline: bool = True) -> pp.ParserElement:
"""Atom for lists.
Parameters
----------
scalars: Union[Any, List[Any]]
Scalar parser elements, already combined or as a list. The list will be
combined using the `^` operator.
start: str
Left delimiter for the list. Defaults to '['.
end: str
Right delimiter for the list. Defaults to ']'.
delimiter: str
List delimiter. Defaults to ','.
multiline: bool
Whether the list can span multiple lines. Defaults to `True`.
Notes
-----
The order of the scalar tokens in the `scalars` list **is important**.
Returns
-------
list: Any
"""
START, END = map(pp.Suppress, start + end)
if isinstance(scalars, list):
atoms = functools.reduce(lambda x, y: x ^ y, scalars)
else:
atoms = scalars
if multiline:
NEWLINE = pp.Literal("\n").suppress()
list_t = (START + pp.Optional(
pp.delimited_list(atoms ^ NEWLINE, delim=delimiter)) + END)
else:
list_t = START + pp.Optional(pp.delimited_list(atoms,
delim=delimiter)) + END
return list_t.add_condition(bool,
message="Empty lists not allowed",
fatal=throw_if_empty)
def __init__(self):
# build grammar according to Glue.Client.get_partitions(Expression)
lpar, rpar = map(Suppress, "()")
# NOTE these are AWS Athena column name best practices
ident = Forward().set_name("ident")
ident <<= Word(alphanums + "._").set_parse_action(_Ident) | lpar + ident + rpar
number = Forward().set_name("number")
number <<= pyparsing_common.number | lpar + number + rpar
string = Forward().set_name("string")
string <<= QuotedString(quote_char="'", esc_quote="''") | lpar + string + rpar
literal = (number | string).set_name("literal")
literal_list = delimited_list(literal, min=1).set_name("list")
bin_op = one_of("<> >= <= > < =").set_name("binary op")
and_ = Forward()
and_ <<= CaselessKeyword("and") | lpar + and_ + rpar
or_ = Forward()
or_ <<= CaselessKeyword("or") | lpar + or_ + rpar
in_, between, like, not_, is_, null = map(
CaselessKeyword, "in between like not is null".split()
)
not_ = Suppress(not_) # only needed for matching
cond = (
(ident + is_ + null).set_parse_action(_IsNull)
| (ident + is_ + not_ + null).set_parse_action(_IsNotNull)
| (ident + bin_op + literal).set_parse_action(_BinOp)
| (ident + like + string).set_parse_action(_Like)
| (ident + not_ + like + string).set_parse_action(_NotLike)
| (ident + in_ + lpar + literal_list + rpar).set_parse_action(_In)
| (ident + not_ + in_ + lpar + literal_list + rpar).set_parse_action(_NotIn)
| (ident + between + literal + and_ + literal).set_parse_action(_Between)
| (ident + not_ + between + literal + and_ + literal).set_parse_action(
_NotBetween
)
).set_name("cond")
# conditions can be joined using 2-ary AND and/or OR
expr = infix_notation(
cond,
[
(and_, 2, OpAssoc.LEFT, _BoolAnd),
(or_, 2, OpAssoc.LEFT, _BoolOr),
],
)
self._expr = expr.set_name("expr")
self._cache: Dict[str, _Expr] = {}
def define_component_parser(self):
"""
Defines specific parsers for BNG components
"""
# bng names are alpha numericals and _1
self.parsers.base_name = pp.Word(pp.alphas, pp.alphanums + "_")
# components have optional states and bonds
self.parsers.state = pp.Combine(
pp.Word("~") +
(self.parsers.base_name ^ pp.Word(pp.nums))) ^ pp.Word("~?")
self.parsers.bond = pp.Combine((pp.Word("!") + pp.Word(pp.nums))
^ (pp.Word("!?")) ^ (pp.Word("!+")))
self.parsers.component = (self.parsers.base_name +
pp.Optional(self.parsers.state) +
pp.Optional(self.parsers.bond))
component_parser = pp.Combine(self.parsers.component)
# components are separated by commas
component_separator = pp.Word(",")
self.parsers.components_parser = pp.delimited_list(
component_parser, delim=component_separator)
self.parsers.combined_components_parser = pp.delimited_list(
component_parser, delim=component_separator, combine=True)
class Mapping(Fragment):
"""
Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input
sections of the entity will end up.
[mapping:<name>]
archive: lib1.a
entries:
obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ...
obj2 (scheme2)
...
Ultimately, an `entity (scheme)` entry generates an
input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html)
in the output linker script. It is possible to attach 'flags' to the
`entity (scheme)` to generate different output commands or to
emit additional keywords in the generated input section description. The
input section description, as well as other output commands, is defined in
output_commands.py.
"""
_any = Literal('*')
_obj = Word(alphas + '_', alphanums + '-_').set_results_name('object')
_sym = Fragment.IDENTIFIER.set_results_name('symbol')
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
_entry = (((_obj + Opt(Suppress(':') + _sym)) | _any.set_results_name('object'))
+ Suppress('(')
+ Fragment.IDENTIFIER.set_results_name('section')
+ Suppress(')'))
ENTRY = _entry + LineEnd().suppress()
ARCHIVE = (Word(alphanums + '.-_$+') | Literal('*')) + LineEnd().suppress()
# Flags can be specified for section->target in the scheme specified, ex:
# obj (scheme);
# section->target SURROUND(symbol),
# section2->target2 ALIGN(4)
ENTRY_WITH_FLAG = (_entry + Suppress(';')
+ delimited_list(Flag.FLAG.set_parse_action(Flag.parse)))
def __init__(self, archive: str, flags: Dict[Any, Flag], *args, **kwargs):
super().__init__(*args, **kwargs)
self.archive = archive
self.flags = flags
@staticmethod
def parse_archive(s, loc, toks):
this = toks[0][0]
if len(this) != 1:
raise ParseFatalException(s, loc, 'Could only specify one archive file in one mapping fragment')
return this[0]
@staticmethod
def parse_entry(toks):
return toks.object, toks.symbol or None, toks.section
@staticmethod
def parse_entry_with_flag(toks):
entry = toks.object, toks.symbol or None, toks.section
return {
entry: [tok for tok in toks if isinstance(tok, Flag)]
}
@staticmethod
def parse_entries(toks):
return toks[0]
@staticmethod
def parse(toks):
this = toks[0]
name = this[0]
archive = this[1]
entries_or_dict_with_flags = this[2]
entries = set()
flags = dict()
for item in entries_or_dict_with_flags:
if isinstance(item, Empty):
continue
elif isinstance(item, dict): # entry with flags
for k, v in item.items():
entries.add(k)
if k in flags:
flags[k].extend(v)
else:
flags[k] = v
else:
entries.add(item)
return Mapping(archive=archive, name=name, entries=entries, flags=flags)
def rc_statement():
"""
Generate a RC statement parser that can be used to parse a RC file
:rtype: pyparsing.ParserElement
"""
one_line_comment = "//" + rest_of_line
comments = c_style_comment ^ one_line_comment
precompiler = AtLineStart(Word("#", alphanums) + rest_of_line)
language_definition = (
"LANGUAGE" + Word(alphas + "_").set_results_name("language") +
Optional("," + Word(alphas + "_").set_results_name("sublanguage")))
block_start = (Keyword("{") | Keyword("BEGIN")).set_name("block_start")
block_end = (Keyword("}") | Keyword("END")).set_name("block_end")
name_id = Group(Word(alphas, alphanums + "_")).set_name("name_id")
numbers = Word(nums)
integerconstant = numbers ^ Combine("0x" + numbers)
constant = Combine(
Optional(Keyword("NOT")) + (name_id | integerconstant),
adjacent=False,
join_string=" ",
)
combined_constants = delimited_list(constant, "|")
concatenated_string = OneOrMore(quoted_string)
block_options = Optional(
SkipTo(Keyword("CAPTION"), fail_on=block_start)("pre_caption") +
Keyword("CAPTION") +
quoted_string("caption")) + SkipTo(block_start)("post_caption")
undefined_control = (Group(
name_id.set_results_name("id_control") +
delimited_list(concatenated_string ^ constant ^ numbers ^ Group(
combined_constants)).set_results_name("values_"))
| comments)
block = (block_start +
ZeroOrMore(undefined_control, stop_on=block_end)("controls") +
block_end)
dialog = (name_id("block_id") +
(Keyword("DIALOGEX") | Keyword("DIALOG"))("block_type") +
block_options + block)
string_table = Keyword("STRINGTABLE")("block_type") + block_options + block
menu_item = Keyword("MENUITEM")("block_type") + (
pyparsing_common.comma_separated_list("values_")
| Keyword("SEPARATOR"))
popup_block = Forward()
popup_block <<= Group(
Keyword("POPUP")("block_type") + Optional(quoted_string("caption")) +
block_start +
ZeroOrMore(Group(menu_item
| popup_block), stop_on=block_end)("elements") +
block_end)("popups*")
menu = (name_id("block_id") + Keyword("MENU")("block_type") +
block_options + block_start +
ZeroOrMore(popup_block, stop_on=block_end) + block_end)
return comments ^ precompiler ^ language_definition ^ dialog ^ string_table ^ menu
class TestCommonHelperExpressions(PyparsingExpressionTestCase):
tests = [
PpTestSpec(
desc="A comma-delimited list of words",
expr=pp.delimited_list(pp.Word(pp.alphas)),
text="this, that, blah,foo, bar",
expected_list=["this", "that", "blah", "foo", "bar"],
),
PpTestSpec(
desc="A counted array of words",
expr=pp.Group(pp.counted_array(pp.Word("ab")))[...],
text="2 aaa bbb 0 3 abab bbaa abbab",
expected_list=[["aaa", "bbb"], [], ["abab", "bbaa", "abbab"]],
),
PpTestSpec(
desc="skipping comments with ignore",
expr=(pp.pyparsing_common.identifier("lhs") + "=" +
pp.pyparsing_common.fnumber("rhs")).ignore(
pp.cpp_style_comment),
text="abc_100 = /* value to be tested */ 3.1416",
expected_list=["abc_100", "=", 3.1416],
expected_dict={
"lhs": "abc_100",
"rhs": 3.1416
},
),
PpTestSpec(
desc=
"some pre-defined expressions in pyparsing_common, and building a dotted identifier with delimted_list",
expr=(pp.pyparsing_common.number("id_num") + pp.delimitedList(
pp.pyparsing_common.identifier, ".", combine=True)("name") +
pp.pyparsing_common.ipv4_address("ip_address")),
text="1001 www.google.com 192.168.10.199",
expected_list=[1001, "www.google.com", "192.168.10.199"],
expected_dict={
"id_num": 1001,
"name": "www.google.com",
"ip_address": "192.168.10.199",
},
),
PpTestSpec(
desc=
"using one_of (shortcut for Literal('a') | Literal('b') | Literal('c'))",
expr=pp.one_of("a b c")[...],
text="a b a b b a c c a b b",
expected_list=[
"a", "b", "a", "b", "b", "a", "c", "c", "a", "b", "b"
],
),
PpTestSpec(
desc="parsing nested parentheses",
expr=pp.nested_expr(),
text="(a b (c) d (e f g ()))",
expected_list=[["a", "b", ["c"], "d", ["e", "f", "g", []]]],
),
PpTestSpec(
desc="parsing nested braces",
expr=(pp.Keyword("if") + pp.nested_expr()("condition") +
pp.nested_expr("{", "}")("body")),
text='if ((x == y) || !z) {printf("{}");}',
expected_list=[
"if",
[["x", "==", "y"], "||", "!z"],
["printf(", '"{}"', ");"],
],
expected_dict={
"condition": [[["x", "==", "y"], "||", "!z"]],
"body": [["printf(", '"{}"', ");"]],
},
),
]
class TestRepetition(PyparsingExpressionTestCase):
tests = [
PpTestSpec(
desc="Match several words",
expr=(pp.Word("x") | pp.Word("y"))[...],
text="xxyxxyyxxyxyxxxy",
expected_list=[
"xx", "y", "xx", "yy", "xx", "y", "x", "y", "xxx", "y"
],
),
PpTestSpec(
desc="Match several words, skipping whitespace",
expr=(pp.Word("x") | pp.Word("y"))[...],
text="x x y xxy yxx y xyx xxy",
expected_list=[
"x",
"x",
"y",
"xx",
"y",
"y",
"xx",
"y",
"x",
"y",
"x",
"xx",
"y",
],
),
PpTestSpec(
desc="Match several words, skipping whitespace (old style)",
expr=pp.OneOrMore(pp.Word("x") | pp.Word("y")),
text="x x y xxy yxx y xyx xxy",
expected_list=[
"x",
"x",
"y",
"xx",
"y",
"y",
"xx",
"y",
"x",
"y",
"x",
"xx",
"y",
],
),
PpTestSpec(
desc=
"Match words and numbers - show use of results names to collect types of tokens",
expr=(pp.Word(pp.alphas)("alpha*")
| pp.pyparsing_common.integer("int*"))[...],
text="sdlfj23084ksdfs08234kjsdlfkjd0934",
expected_list=["sdlfj", 23084, "ksdfs", 8234, "kjsdlfkjd", 934],
expected_dict={
"alpha": ["sdlfj", "ksdfs", "kjsdlfkjd"],
"int": [23084, 8234, 934],
},
),
PpTestSpec(
desc="Using delimited_list (comma is the default delimiter)",
expr=pp.delimited_list(pp.Word(pp.alphas)),
text="xxyx,xy,y,xxyx,yxx, xy",
expected_list=["xxyx", "xy", "y", "xxyx", "yxx", "xy"],
),
PpTestSpec(
desc=
"Using delimited_list (comma is the default delimiter) with trailing delimiter",
expr=pp.delimited_list(pp.Word(pp.alphas),
allow_trailing_delim=True),
text="xxyx,xy,y,xxyx,yxx, xy,",
expected_list=["xxyx", "xy", "y", "xxyx", "yxx", "xy"],
),
PpTestSpec(
desc=
"Using delimited_list (comma is the default delimiter) with minimum size",
expr=pp.delimited_list(pp.Word(pp.alphas), min=3),
text="xxyx,xy",
expected_fail_locn=7,
),
PpTestSpec(
desc=
"Using delimited_list (comma is the default delimiter) with maximum size",
expr=pp.delimited_list(pp.Word(pp.alphas), max=3),
text="xxyx,xy,y,xxyx,yxx, xy,",
expected_list=["xxyx", "xy", "y"],
),
PpTestSpec(
desc="Using delimited_list, with ':' delimiter",
expr=pp.delimited_list(pp.Word(pp.hexnums, exact=2),
delim=":",
combine=True),
text="0A:4B:73:21:FE:76",
expected_list=["0A:4B:73:21:FE:76"],
),
PpTestSpec(
desc="Using delimited_list, with ':' delimiter",
expr=pp.delimited_list(
pp.Word(pp.hexnums, exact=2),
delim=":",
combine=True,
allow_trailing_delim=True,
),
text="0A:4B:73:21:FE:76:",
expected_list=["0A:4B:73:21:FE:76:"],
),
]
braced_text = Forward()
braced_text_elems = Forward()
braced_text << Suppress('{') + braced_text_elems + Suppress('}')
braced_text.set_parse_action(create1(BracedText))
braced_text_elems << ZeroOrMore(nonbraced_text | braced_text)
braced_text_elems.set_parse_action(create_rec(BracedTextElements))
quoted_text = QuotedString('"', '\\', unquoteResults=False, multiline=True)
quoted_text.set_parse_action(create1(QuotedText))
entry_key = spaces + CharsNotIn('#\\@,={} \t\n') + spaces
entry_key.set_parse_action(create1(EntryKey))
tag_value_elems = delimited_list(braced_text | quoted_text | entry_key, '#')
tag_value_elems.set_parse_action(create_rec(TagValueElements))
tag_value = tag_value_elems
tag_value.set_parse_action(create1(TagValue))
tag_name = spaces + CharsNotIn('#\\@,={} \t\n') + spaces
tag_name.set_parse_action(create1(TagName))
tag = tag_name + Suppress('=') + tag_value
tag.set_parse_action(create2(Tag))
tags = delimited_list(tag, ',', allow_trailing_delim=True)
tags.set_parse_action(create_rec(Tags))
entry_body = entry_key + Optional(Suppress(',') + Optional(tags))
# - invalid_texture_functions.test: no GL texture functions in Runtime Effects
# - preprocessor.test: no preprocessor in SkSL
import os
import pyparsing as pp
import re
import sys
# Each case can contain expected input/output values, sometimes in [bracketed|lists] and
# sometimes not.
# GLSL code appears in ""double-double quotes"" and is indicated as vert/frag-specific or "both".
# Some tests denote that they are expected to pass/fail. (Presumably, "pass" is the default.)
# We ignore descriptions and version fields.
wordWithUnderscores = pp.Word(pp.alphanums + '_')
pipeList = pp.delimited_list(pp.SkipTo(pp.Literal("|") | pp.Literal("]")), delim="|")
bracketedPipeList = pp.Group(pp.Literal("[").suppress() +
pipeList +
pp.Literal("]").suppress())
unbracketedValue = pp.Group(pp.SkipTo(";"))
valueList = (pp.Word(pp.alphanums) + # type
pp.Word(pp.alphanums) + # varname
pp.Literal("=").suppress() +
(bracketedPipeList | unbracketedValue) +
pp.Literal(";").suppress())
value = pp.Group((pp.Keyword("input") | pp.Keyword("output") | pp.Keyword("uniform")) +
valueList)
values = (pp.Keyword("values") +
pp.Literal("{").suppress() +
pp.ZeroOrMore(value) +
