def _logical_parser(expression):
"""
Return a new parser parsing logical expressions.
This parser recognizes the following grammar, with precedence:
<logical> ::= expression | '~' <logical> | <logical> '&' <logical>
| <logical> '|' <logical> | <logical> '->' <logical>
| <logical> '<->' <logical>
.. note:: The parser uses :mod:`pytlq.ast` module's classes to build ASTs.
.. credit:: Adapted from Simon Busard's parser parsing logical expressions
on atomics.
"""
parser = Forward()
not_strict = Literal('~') + expression
not_strict.setParseAction(lambda tokens: Not(tokens[1]))
not_ = (not_strict | expression)
and_ = not_ + ZeroOrMore(Literal('&') + not_)
and_.setParseAction(lambda tokens: _left(And, tokens))
or_ = and_ + ZeroOrMore(Literal('|') + and_)
or_.setParseAction(lambda tokens: _left(Or, tokens))
imply = ZeroOrMore(or_ + Literal('->')) + or_
imply.setParseAction(lambda tokens: _right(Imply, tokens))
iff = imply + ZeroOrMore(Literal('<->') + imply)
iff.setParseAction(lambda tokens: _left(Iff, tokens))
parser <<= iff
return parser
def _define_valued_characters_section(self, heading, characters,
parse_action, character_type):
"""Returns a parser object for a section specifying characters
and their valued features.
:param heading: section heading
:type heading: `str`
:param characters: valid characters
:type characters: `list` of `str`
:param parse_action: parse action for a character
:type parse_action: `function`
:param character_type: type of characters being described
:type character_type: `str`
"""
heading = Literal('[{}]'.format(heading))
character = Word(''.join(characters), exact=1).setResultsName(
'character')
character.setParseAction(self._handle_character)
feature = Word(alphas).setResultsName('feature')
feature.setParseAction(parse_action)
value = Literal('+') ^ Literal('-') ^ Literal('\N{MINUS SIGN}')
value.setParseAction(self._handle_feature_value)
feature_value = Group(value + feature)
feature_values = Group(delimitedList(feature_value))
character_definition = Dict(Group(character + Suppress(':') +
feature_values))
character_definitions = Group(OneOrMore(character_definition)).setResultsName(character_type)
section = Suppress(heading) + character_definitions
return section
def __init__(self, path, text, state=None):
self.path = path
self.base_path = os.path.dirname(path)
self.text = text
self.state = state
opcode_name = Word(alphanums + '_')
value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))',
re.MULTILINE)
opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value
opcode.setParseAction(self.handle_opcode)
section_name = Literal('<').suppress() + Word(alphas) + Literal(
'>').suppress()
section = section_name
section.setParseAction(self.handle_section)
include = Literal('#include').suppress() + locatedExpr(
QuotedString('"'))
include.setParseAction(self.handle_include)
statement = (section ^ opcode ^ include)
self.sfz_file = ZeroOrMore(statement) + stringEnd
comment = Literal('//') + restOfLine
self.sfz_file.ignore(comment)
def grammar():
parenthesis = Forward()
parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")"
parenthesis.setParseAction(join_string_act)
quoted_string = "'" + OneOrMore(CharsNotIn("'")) + "'"
quoted_string.setParseAction(join_string_act)
quoted_default_value = "DEFAULT" + quoted_string + OneOrMore(CharsNotIn(", \n\t"))
quoted_default_value.setParseAction(quoted_default_value_act)
field_def = OneOrMore(quoted_default_value | Word(alphanums + "_\"'`:-/[]") | parenthesis)
field_def.setParseAction(field_act)
tablename_def = ( Word(alphas + "`_.") | QuotedString("\"") )
field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def)
field_list_def.setParseAction(field_list_act)
create_table_def = Literal("CREATE") + "TABLE" + tablename_def.setResultsName("tableName") + "(" + field_list_def.setResultsName("fields") + ")" + ";"
create_table_def.setParseAction(create_table_act)
add_fkey_def = Literal("ALTER") + "TABLE" + "ONLY" + tablename_def.setResultsName("tableName") + "ADD" + "CONSTRAINT" + Word(alphanums + "_") + "FOREIGN" + "KEY" + "(" + Word(alphanums + "_").setResultsName("keyName") + ")" + "REFERENCES" + Word(alphanums + "._").setResultsName("fkTable") + "(" + Word(alphanums + "_").setResultsName("fkCol") + ")" + Optional(Literal("DEFERRABLE")) + Optional(Literal("ON") + "DELETE" + ( Literal("CASCADE") | Literal("RESTRICT") )) + ";"
add_fkey_def.setParseAction(add_fkey_act)
other_statement_def = OneOrMore(CharsNotIn(";")) + ";"
other_statement_def.setParseAction(other_statement_act)
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
comment_def.setParseAction(other_statement_act)
return OneOrMore(comment_def | create_table_def | add_fkey_def | other_statement_def)
def grammar():
parenthesis = Forward()
parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")"
field_def = OneOrMore(Word(alphanums + "_\"'`:-") | parenthesis)
field_def.setParseAction(field_act)
tablename_def = ( Word(alphas + "`_") | QuotedString("\"") )
field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def)
field_list_def.setParseAction(field_list_act)
create_table_def = Literal("CREATE") + "TABLE" + tablename_def.setResultsName("tableName") + "(" + field_list_def.setResultsName("fields") + ")" + ";"
create_table_def.setParseAction(create_table_act)
add_fkey_def = Literal("ALTER") + "TABLE" + "ONLY" + tablename_def.setResultsName("tableName") + "ADD" + "CONSTRAINT" + Word(alphanums + "_") + "FOREIGN" + "KEY" + "(" + Word(alphanums + "_").setResultsName("keyName") + ")" + "REFERENCES" + Word(alphanums + "_").setResultsName("fkTable") + "(" + Word(alphanums + "_").setResultsName("fkCol") + ")" + ";"
add_fkey_def.setParseAction(add_fkey_act)
other_statement_def = OneOrMore(CharsNotIn(";")) + ";"
other_statement_def.setParseAction(other_statement_act)
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
comment_def.setParseAction(other_statement_act)
return OneOrMore(comment_def | create_table_def | add_fkey_def | other_statement_def)
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen, colon, qmark = map(
Literal, "[]{}():?")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(
c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
reTerm = (reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
reExpr = infixNotation(reTerm, [
(repetition, 1, opAssoc.LEFT, handleRepetition),
(None, 2, opAssoc.LEFT, handleSequence),
(Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
])
_parser = reExpr
return _parser
def grammar():
parenthesis = Forward()
parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")"
field_def = OneOrMore(Word(alphanums + "_\"'`:-") | parenthesis)
field_def.setParseAction(field_act)
field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def)
field_list_def.setParseAction(field_list_act)
create_table_def = Literal("CREATE") + "TABLE" + Word(
alphas + "`_").setResultsName(
"tableName") + "(" + field_list_def.setResultsName(
"fields") + ")" + ";"
create_table_def.setParseAction(create_table_act)
add_fkey_def = Literal("ALTER") + "TABLE" + "ONLY" + Word(
alphanums +
"_").setResultsName("tableName") + "ADD" + "CONSTRAINT" + Word(
alphanums + "_") + "FOREIGN" + "KEY" + "(" + Word(
alphanums +
"_").setResultsName("keyName") + ")" + "REFERENCES" + Word(
alphanums + "_").setResultsName("fkTable") + "(" + Word(
alphanums + "_").setResultsName("fkCol") + ")" + ";"
add_fkey_def.setParseAction(add_fkey_act)
other_statement_def = OneOrMore(CharsNotIn(";")) + ";"
other_statement_def.setParseAction(other_statement_act)
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
comment_def.setParseAction(other_statement_act)
return OneOrMore(comment_def | create_table_def | add_fkey_def
| other_statement_def)
def __init__(self, path, text, state=None):
self.path = path
self.base_path = os.path.dirname(path)
self.text = text
self.state = state
opcode_name = Word(alphanums + '_')
value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE)
opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value
opcode.setParseAction(self.handle_opcode)
section_name = Literal('<').suppress() + Word(alphas) + Literal('>').suppress()
section = section_name
section.setParseAction(self.handle_section)
include = Literal('#include').suppress() + locatedExpr(QuotedString('"'))
include.setParseAction(self.handle_include)
statement = (section
^ opcode
^ include)
self.sfz_file = ZeroOrMore(statement) + stringEnd
comment = Literal('//') + restOfLine
self.sfz_file.ignore(comment)
def parse(self, query):
"""Parses a query string."""
# Parse instructions
quoted_string = QuotedString(quoteChar='"', escChar='\\', unquoteResults=True)
field_name = Word(alphas, alphanums + '_')
subexpression = Forward()
boolean_expression = Forward()
binary_operator = Literal('=') | Literal('<=') | Literal('<') | Literal('>=') | Literal('>')
boolean_operator = CaselessKeyword('AND') | CaselessKeyword('OR')
boolean_not = CaselessKeyword('NOT')
boolean_value = CaselessKeyword("true") ^ CaselessKeyword("false")
integer = Word(nums)
rvalue = quoted_string ^ boolean_value ^ integer
field_to_value = field_name + binary_operator + rvalue
expression = Optional(boolean_not) + ((subexpression + ZeroOrMore(boolean_expression)) | (field_to_value + ZeroOrMore(boolean_expression)))
boolean_expression << boolean_operator + expression
left_bracket = Literal('(')
right_bracket = Literal(')')
subexpression << (left_bracket + expression + right_bracket)
search_query = expression
# Parse actions for emitting special cases
field_to_value.setParseAction(do_field_to_value)
boolean_operator.setParseAction(do_boolean_operator)
boolean_not.setParseAction(do_boolean_operator)
boolean_value.setParseAction(do_boolean_value)
integer.setParseAction(do_integer)
left_bracket.setParseAction(do_bracket)
right_bracket.setParseAction(do_bracket)
self.tokens = search_query.parseString(query)
def _define_context_component (self, cluster, base_feature_set):
placeholder = Literal(SOURCE_PLACEHOLDER)
placeholder.setParseAction(self._handle_placeholder)
context_component = Group(ZeroOrMore(cluster ^ base_feature_set) + \
placeholder + ZeroOrMore(cluster ^ base_feature_set)).setResultsName('context_component')
context_component.setParseAction(self._handle_context_component)
return context_component
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen = map(Literal, "[]{}()")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~ reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = (
(lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) |
oneOf(list("*+?"))
)
reRange.setParseAction(handle_range)
reLiteral.setParseAction(handle_literal)
reMacro.setParseAction(handle_macro)
reDot.setParseAction(handle_dot)
reTerm = (reLiteral | reRange | reMacro | reDot)
reExpr = operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, handle_repetition),
(None, 2, opAssoc.LEFT, handle_sequence),
(Suppress('|'), 2, opAssoc.LEFT, handle_alternative),
])
_parser = reExpr
return _parser
def _logical_parser(expression):
"""
Return a new parser parsing logical expressions.
This parser recognizes the following grammar, with precedence:
<logical> ::= expression | '~' <logical> | <logical> '&' <logical>
| <logical> '|' <logical> | <logical> '->' <logical>
| <logical> '<->' <logical>
.. note:: The parser uses :mod:`pytlq.ast` module's classes to build ASTs.
.. credit:: Adapted from Simon Busard's parser parsing logical expressions
on atomics.
"""
parser = Forward()
not_strict = Literal('~') + expression
not_strict.setParseAction(lambda tokens: Not(tokens[1]))
not_ = (not_strict | expression)
and_ = not_ + ZeroOrMore(Literal('&') + not_)
and_.setParseAction(lambda tokens: _left(And, tokens))
or_ = and_ + ZeroOrMore(Literal('|') + and_)
or_.setParseAction(lambda tokens: _left(Or, tokens))
imply = ZeroOrMore(or_ + Literal('->')) + or_
imply.setParseAction(lambda tokens: _right(Imply, tokens))
iff = imply + ZeroOrMore(Literal('<->') + imply)
iff.setParseAction(lambda tokens: _left(Iff, tokens))
parser <<= iff
return parser
def enumeration_type_definition() -> Token:
enumeration_literal = unqualified_identifier()
positional_enumeration = enumeration_literal + ZeroOrMore(
comma() - enumeration_literal)
positional_enumeration.setParseAction(
lambda t: [(k, Number(v)) for v, k in enumerate(t.asList())])
element_value_association = enumeration_literal + Keyword(
"=>") - numeric_literal()
element_value_association.setParseAction(lambda t: (t[0], t[2]))
named_enumeration = element_value_association + ZeroOrMore(
comma() - element_value_association)
boolean_literal = Keyword("True") | Keyword("False")
boolean_literal.setParseAction(lambda t: t[0] == "True")
boolean_aspect_definition = Optional(Keyword("=>") - boolean_literal)
boolean_aspect_definition.setParseAction(lambda t:
(t if t else ["=>", True]))
always_valid_aspect = Literal("Always_Valid") - boolean_aspect_definition
always_valid_aspect.setParseAction(parse_aspect)
enumeration_aspects = Keyword("with") - delimitedList(
size_aspect() | always_valid_aspect)
enumeration_aspects.setParseAction(parse_aspects)
return (Literal("(") - (named_enumeration | positional_enumeration) -
Literal(")") - enumeration_aspects).setName("Enumeration")
def parseArctl(spec):
"""Parse the spec and return its AST."""
global __arctl
if __arctl is None:
true = Literal("True")
true.setParseAction(lambda tokens: TrueExp())
false = Literal("False")
false.setParseAction(lambda tokens: FalseExp())
atom = "'" + SkipTo("'") + "'"
atom.setParseAction(lambda tokens: Atom(tokens[1]))
action = _logicals_(atom)
__arctl = Forward()
proposition = true | false | atom
notproposition = "~" + proposition
notproposition.setParseAction(lambda tokens: Not(tokens[1]))
formula = (proposition | notproposition
| Suppress("(") + __arctl + Suppress(")"))
temporal = Forward()
e = Literal("E") + "<" + action + ">"
a = Literal("A") + "<" + action + ">"
eax = e + "X" + temporal
eax.setParseAction(lambda tokens: EaX(tokens[2], tokens[5]))
aax = a + "X" + temporal
aax.setParseAction(lambda tokens: AaX(tokens[2], tokens[5]))
eaf = e + "F" + temporal
eaf.setParseAction(lambda tokens: EaF(tokens[2], tokens[5]))
aaf = a + "F" + temporal
aaf.setParseAction(lambda tokens: AaF(tokens[2], tokens[5]))
eag = e + "G" + temporal
eag.setParseAction(lambda tokens: EaG(tokens[2], tokens[5]))
aag = a + "G" + temporal
aag.setParseAction(lambda tokens: AaG(tokens[2], tokens[5]))
eau = e + "[" + __arctl + "U" + __arctl + "]"
eau.setParseAction(lambda tokens: EaU(tokens[2], tokens[5], tokens[7]))
aau = a + "[" + __arctl + "U" + __arctl + "]"
aau.setParseAction(lambda tokens: AaU(tokens[2], tokens[5], tokens[7]))
eaw = e + "[" + __arctl + "W" + __arctl + "]"
eaw.setParseAction(lambda tokens: EaW(tokens[2], tokens[5], tokens[7]))
aaw = a + "[" + __arctl + "W" + __arctl + "]"
aaw.setParseAction(lambda tokens: AaW(tokens[2], tokens[5], tokens[7]))
temporal <<= (formula | eax | aax | eaf | aaf | eag | aag | eau | aau
| eaw | aaw)
logical = _logicals_(temporal)
__arctl <<= logical
return __arctl.parseString(spec, parseAll=True)
def parseArctl(spec):
"""Parse the spec and return its AST."""
global __arctl
if __arctl is None:
true = Literal("True")
true.setParseAction(lambda tokens: TrueExp())
false = Literal("False")
false.setParseAction(lambda tokens: FalseExp())
atom = "'" + SkipTo("'") + "'"
atom.setParseAction(lambda tokens: Atom(tokens[1]))
action = _logicals_(atom)
__arctl = Forward()
proposition = true | false | atom
notproposition = "~" + proposition
notproposition.setParseAction(lambda tokens: Not(tokens[1]))
formula = proposition | notproposition | Suppress("(") + __arctl + Suppress(")")
temporal = Forward()
e = Literal("E") + "<" + action + ">"
a = Literal("A") + "<" + action + ">"
eax = e + "X" + temporal
eax.setParseAction(lambda tokens: EaX(tokens[2], tokens[5]))
aax = a + "X" + temporal
aax.setParseAction(lambda tokens: AaX(tokens[2], tokens[5]))
eaf = e + "F" + temporal
eaf.setParseAction(lambda tokens: EaF(tokens[2], tokens[5]))
aaf = a + "F" + temporal
aaf.setParseAction(lambda tokens: AaF(tokens[2], tokens[5]))
eag = e + "G" + temporal
eag.setParseAction(lambda tokens: EaG(tokens[2], tokens[5]))
aag = a + "G" + temporal
aag.setParseAction(lambda tokens: AaG(tokens[2], tokens[5]))
eau = e + "[" + __arctl + "U" + __arctl + "]"
eau.setParseAction(lambda tokens: EaU(tokens[2], tokens[5], tokens[7]))
aau = a + "[" + __arctl + "U" + __arctl + "]"
aau.setParseAction(lambda tokens: AaU(tokens[2], tokens[5], tokens[7]))
eaw = e + "[" + __arctl + "W" + __arctl + "]"
eaw.setParseAction(lambda tokens: EaW(tokens[2], tokens[5], tokens[7]))
aaw = a + "[" + __arctl + "W" + __arctl + "]"
aaw.setParseAction(lambda tokens: AaW(tokens[2], tokens[5], tokens[7]))
temporal <<= formula | eax | aax | eaf | aaf | eag | aag | eau | aau | eaw | aaw
logical = _logicals_(temporal)
__arctl <<= logical
return __arctl.parseString(spec, parseAll=True)
def _create_primitives():
global binary, ident, rvalue, number, quoted_string, semi, tick_interval, time_interval, slot_id, comp, config_type, stream, comment, stream_trigger, selector
if ident is not None:
return
semi = Literal(u';').suppress()
ident = Word(alphas+u"_", alphas + nums + u"_")
number = Regex(u'((0x[a-fA-F0-9]+)|[+-]?[0-9]+)').setParseAction(lambda s, l, t: [int(t[0], 0)])
binary = Regex(u'hex:([a-fA-F0-9][a-fA-F0-9])+').setParseAction(lambda s, l, t: [unhexlify(t[0][4:])])
quoted_string = QuotedString(quoteChar='"', escChar="\\", unquoteResults=True)
comment = Literal('#') + restOfLine
rvalue = number | quoted_string
# Convert all time intervals into an integer number of seconds
time_unit_multipliers = {
u'second': 1,
u'seconds': 1,
u'minute': 60,
u'minutes': 60,
u'hour': 60*60,
u'hours': 60*60,
u'day': 60*60*24,
u'days': 60*60*24,
u'month': 60*60*24*30,
u'months': 60*60*24*30,
u'year': 60*60*24*365,
u'years': 60*60*24*365,
}
config_type = oneOf('uint8_t uint16_t uint32_t int8_t int16_t int32_t uint8_t[] uint16_t[] uint32_t[] int8_t[] int16_t[] int32_t[] string binary')
comp = oneOf('> < >= <= == ~=')
# Time intervals are all based on internal system clocks so we include a 'system' tag
time_unit = oneOf(u"second seconds minute minutes hour hours day days week weeks month months year years")
time_interval = (number + time_unit).setParseAction(lambda s, l, t: [t[0]*time_unit_multipliers[t[1]], 'system'])
tick_interval = (number + (Literal("tick_1") | Literal("tick_2"))).setParseAction(lambda s, l, t: [t[0], t[1]])
slot_id = Literal(u"controller") | (Literal(u'slot') + number)
slot_id.setParseAction(lambda s,l,t: [SlotIdentifier.FromString(u' '.join([str(x) for x in t]))])
stream_modifier = Literal("system") | Literal("user") | Literal("combined")
stream = Optional(Literal("system")) + oneOf("buffered unbuffered input output counter constant") + number + Optional(Literal("node"))
stream.setParseAction(lambda s,l,t: [DataStream.FromString(u' '.join([str(x) for x in t]))])
all_selector = Optional(Literal("all")) + Optional(stream_modifier) + oneOf("buffered unbuffered inputs outputs counters constants") + Optional(Literal("nodes"))
all_selector.setParseAction(lambda s,l,t: [DataStreamSelector.FromString(u' '.join([str(x) for x in t]))])
one_selector = Optional(Literal("system")) + oneOf("buffered unbuffered input output counter constant") + number + Optional(Literal("node"))
one_selector.setParseAction(lambda s,l,t: [DataStreamSelector.FromString(u' '.join([str(x) for x in t]))])
selector = one_selector | all_selector
trigger_comp = oneOf('> < >= <= ==')
stream_trigger = Group((Literal(u'count') | Literal(u'value')) + Literal(u'(').suppress() - stream - Literal(u')').suppress() - trigger_comp - number).setResultsName('stream_trigger')
def _build_sort_parser():
field = _build_field_expr()
plusorminus = Literal('+') | Literal('-')
plusorminus.setParseAction(lambda x: SortDirection(x[0]))
sort_directive = Group(Optional(plusorminus) +
field).setParseAction(lambda x: SortDirective(x))
statement = Optional(sort_directive) + ZeroOrMore(
Literal(',') + sort_directive)
return statement
def __init__(self):
self.variables = dict()
#_base_var = Literal("$") + (("{" + Word(alphanums + "_-").setResultsName("var_name", listAllMatches=True) + Optional("[" + Word(nums + "*@") + "]") + "}")
#| Word(alphanums))
_simple_var = Literal("$") + Word(alphanums + "_-").setResultsName("varname")
_brace_substitute_part = Optional("/" + (Word(alphanums + "_-").setResultsName("orig"))
+ Optional("/" + Word(alphanums + "_-!?/\\").setResultsName("new")))
_array_access = "[" + Word(nums + "@*").setResultsName("position") + "]"
_brace_var = Literal("${") + Word(alphanums + "_-").setResultsName("text") + _brace_substitute_part + Optional(_array_access) + "}"
_brace_var.setParseAction(lambda x: x if not x.new else re.sub(x.orig, x.new, x.text))
_base_var = _simple_var | _brace_var
self.var = ('"' + _base_var + '"') | _base_var
self.var("variable")
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
tag_begin = Literal("<").suppress()
tag_end = Literal(">").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
ival=Regex('[-]?\d+')
dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?')
lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)')
# Helper definitions
kstr= quotedString.setParseAction(removeQuotes) ^ \
dval ^ ival ^ lval ^ Word(prtable)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | tag_sect ) #| vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
ival.setParseAction(self.conv_ival)
dval.setParseAction(self.conv_dval)
lval.setParseAction(self.conv_lval)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
tag_sect.setParseAction(self.add_sect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
tag_begin = Literal("<").suppress()
tag_end = Literal(">").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
ival=Regex('[-]?\d+')
dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?')
lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)')
# Helper definitions
kstr= quotedString.setParseAction(removeQuotes) ^ \
dval ^ ival ^ lval ^ Word(prtable)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | tag_sect ) #| vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
ival.setParseAction(self.conv_ival)
dval.setParseAction(self.conv_dval)
lval.setParseAction(self.conv_lval)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
tag_sect.setParseAction(self.add_sect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack = Literal("[")
rbrack = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
lparen = Literal("(")
rparen = Literal(")")
reMacro = Suppress("\\") + oneOf(list("dwsZ"))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in string.printable
if c not in r"\[]{}().*?+|")
reRange = Combine(lbrack.suppress() +
SkipTo(rbrack, ignore=escapedChar) +
rbrack.suppress())
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reExpr = Forward()
reGroup = (lparen.suppress() +
Optional(Literal("?").suppress() +
oneOf(list(":P"))).setResultsName("option") +
reExpr.setResultsName("expr") + rparen.suppress())
reTerm = (reLiteral | reRange | reMacro | reDot | reGroup)
reExpr << operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, create(Repetition)),
(None, 2, opAssoc.LEFT, create(Sequence)),
(Suppress('|'), 2, opAssoc.LEFT, create(Alternation)),
])
reGroup.setParseAction(create(Group))
reRange.setParseAction(create(Range))
reLiteral.setParseAction(create(Character))
reMacro.setParseAction(create(Macro))
reDot.setParseAction(create(Dot))
_parser = reExpr
return _parser
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack = Literal("[")
rbrack = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
lparen = Literal("(")
rparen = Literal(")")
reMacro = Suppress("\\") + oneOf(list("dwsZ"))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|")
reRange = Combine(lbrack.suppress() + SkipTo(rbrack,ignore=escapedChar) + rbrack.suppress())
reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
reDot = Literal(".")
repetition = (
( lbrace + Word(nums).setResultsName("count") + rbrace ) |
( lbrace + Word(nums).setResultsName("minCount")+","+ Word(nums).setResultsName("maxCount") + rbrace ) |
oneOf(list("*+?"))
)
reExpr = Forward()
reGroup = (lparen.suppress() +
Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") +
reExpr.setResultsName("expr") +
rparen.suppress())
reTerm = ( reLiteral | reRange | reMacro | reDot | reGroup )
reExpr << operatorPrecedence( reTerm,
[
(repetition, 1, opAssoc.LEFT, create(Repetition)),
(None, 2, opAssoc.LEFT, create(Sequence)),
(Suppress('|'), 2, opAssoc.LEFT, create(Alternation)),
]
)
reGroup.setParseAction(create(Group))
reRange.setParseAction(create(Range))
reLiteral.setParseAction(create(Character))
reMacro.setParseAction(create(Macro))
reDot.setParseAction(create(Dot))
_parser = reExpr
return _parser
def grammar():
parenthesis = Forward()
parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")"
field_def = OneOrMore(Word(alphanums + "._\"'`:-[]") | parenthesis)
field_def.setParseAction(field_act)
tablename_def = (Word(alphas + "`_") | QuotedString("\""))
function_end = Literal("END") + "$$" + Optional(
Literal("LANGUAGE") + "plsql") + ";"
function_statement_def = Literal("CREATE") + Optional(
Literal("OR") + "REPLACE") + "FUNCTION" + SkipTo(
function_end) # + function_end
function_statement_def.setParseAction(function_act)
field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def)
field_list_def.setParseAction(field_list_act)
create_table_def = Literal(
"CREATE") + "TABLE" + tablename_def.setResultsName(
"tableName") + "(" + field_list_def.setResultsName(
"fields") + ")" + ";"
create_table_def.setParseAction(create_table_act)
skip_fkey_def = Literal("ALTER") + "TABLE" + Regex(
r"\"[a-zA-Z_]+_[0-9]{3}\"") + "ADD" + "CONSTRAINT" + QuotedString(
"\"") + "FOREIGN" + "KEY" + "(" + QuotedString(
"\"") + ")" + "REFERENCES" + QuotedString(
"\"") + "(" + QuotedString("\"") + ")" + ZeroOrMore(
CharsNotIn(";")) + ";"
skip_fkey_def.setParseAction(skip_fkey_act)
add_fkey_def = Literal("ALTER") + "TABLE" + tablename_def.setResultsName(
"tableName") + "ADD" + "CONSTRAINT" + QuotedString(
"\"") + "FOREIGN" + "KEY" + "(" + QuotedString(
"\"").setResultsName(
"keyName") + ")" + "REFERENCES" + QuotedString(
"\"").setResultsName("fkTable") + "(" + QuotedString(
"\"").setResultsName("fkCol") + ")" + ZeroOrMore(
CharsNotIn(";")) + ";"
add_fkey_def.setParseAction(add_fkey_act)
other_statement_def = OneOrMore(CharsNotIn(";")) + ";"
other_statement_def.setParseAction(other_statement_act)
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
comment_def.setParseAction(other_statement_act)
mcomment_def = "/*" + SkipTo(Literal("*/"))
mcomment_def.setParseAction(mcomment_act)
return OneOrMore(comment_def | mcomment_def | function_statement_def
| create_table_def | skip_fkey_def | add_fkey_def
| other_statement_def)
def __init__(self):
left_bracket = Literal("{").suppress()
right_bracket = Literal("}").suppress()
semicolon = Literal(";").suppress()
space = White().suppress()
key = Word(alphanums + "+.-_/")
value = ZeroOrMore(
CharsNotIn('{};#"\'') | space |
QuotedString("'", escChar='\\', multiline=True) |
QuotedString('"', escChar='\\', multiline=True))
# modifier for location uri [ = | ~ | ~* | ^~ ]
modifier = Literal("=") | Literal("~*") | Literal("~") | Literal("^~")
comment = Literal('#').suppress() + Optional(restOfLine)
# rules
assignment = Group(
(key | value) + value + semicolon +
Optional(space + comment))
block = Forward()
block << Group(
Group(key + Optional(space + modifier) + Optional(space) +
Optional(value) + Optional(space + value)) +
left_bracket +
Group(ZeroOrMore(assignment | block | comment.suppress())) +
right_bracket)
def comment_handler(t):
result = []
if "promo" in t[0]:
result.append("promo")
if "author: " in t[0]:
try:
email = t[0].split("author: ")[1].strip()
result.append(email)
except Exception:
result.append(t[0])
return result
comment.setParseAction(comment_handler)
self.script = OneOrMore(assignment | block | comment.suppress())
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
arg_begin = Literal("(").suppress()
arg_end = Literal(")").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
# Helper definitions
kstr=Word(prtable) ^ quotedString.setParseAction(removeQuotes)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
key_sect=name+Group(arg_begin+kstr+arg_end)+sect_begin
vec_sect=name+Group(arg_begin+vec+ arg_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | key_sect | vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
key_sect.setParseAction(self.add_sect)
vec_sect.setParseAction(self.add_vecsect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas+"_", alphanums+"_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal( "^" )
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal( "." )
assign = Literal( "=" )
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand,
[
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr,
[
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(comp_expr,
[
(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)
])
pattern = logic_expr + StringEnd()
return pattern
def define_number(self):
"""
Return the syntax definition for a number in Arabic Numerals.
Override this method to support numeral systems other than Arabic
Numerals (0-9).
Do not override this method just to change the character used to
separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR`
and :attr:`T_DECIMAL_SEPARATOR`, respectively.
"""
# Defining the basic tokens:
to_dot = lambda t: "."
to_plus = lambda t: "+"
to_minus = lambda t: "-"
positive_sign = Literal(self._grammar.get_token("positive_sign"))
positive_sign.setParseAction(to_plus)
negative_sign = Literal(self._grammar.get_token("negative_sign"))
negative_sign.setParseAction(to_minus)
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(
self._grammar.get_token("thousands_separator"))
digits = Word(nums)
# Building the integers and decimals:
sign = positive_sign | negative_sign
thousands = Word(nums, max=3) + \
OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
number = Combine(Optional(sign) + integers + Optional(decimals))
number.setParseAction(self.make_number)
number.setName("number")
return number
def define_number(self):
"""
Return the syntax definition for a number in Arabic Numerals.
Override this method to support numeral systems other than Arabic
Numerals (0-9).
Do not override this method just to change the character used to
separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR`
and :attr:`T_DECIMAL_SEPARATOR`, respectively.
"""
# Defining the basic tokens:
to_dot = lambda t: "."
to_plus = lambda t: "+"
to_minus = lambda t: "-"
positive_sign = Literal(self._grammar.get_token("positive_sign"))
positive_sign.setParseAction(to_plus)
negative_sign = Literal(self._grammar.get_token("negative_sign"))
negative_sign.setParseAction(to_minus)
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(self._grammar.get_token("thousands_separator"))
digits = Word(nums)
# Building the integers and decimals:
sign = positive_sign | negative_sign
thousands = Word(nums, max=3) + \
OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
number = Combine(Optional(sign) + integers + Optional(decimals))
number.setParseAction(self.make_number)
number.setName("number")
return number
def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas + "_", alphanums + "_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal("^")
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal(".")
assign = Literal("=")
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(
function.setResultsName('fn') + LPAR +
Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand, [
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr, [
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(
comp_expr, [(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)])
pattern = logic_expr + StringEnd()
return pattern
def _get_bus_array_construct(self):
""" Returns a construct for an array of bus data.
"""
bus_no = integer.setResultsName("bus_no")
v_base = real.setResultsName("v_base") # kV
v_magnitude = Optional(real).setResultsName("v_magnitude")
v_angle = Optional(real).setResultsName("v_angle") # radians
area = Optional(integer).setResultsName("area") # not used yet
region = Optional(integer).setResultsName("region") # not used yet
bus_data = bus_no + v_base + v_magnitude + v_angle + \
area + region + scolon
bus_data.setParseAction(self.push_bus)
bus_array = Literal("Bus.con") + "=" + "[" + "..." + \
ZeroOrMore(bus_data + Optional("]" + scolon))
# Sort buses according to their name (bus_no)
bus_array.setParseAction(self.sort_buses)
return bus_array
def _get_bus_array_construct(self):
""" Returns a construct for an array of bus data.
"""
bus_no = integer.setResultsName("bus_no")
v_base = real.setResultsName("v_base") # kV
v_magnitude = Optional(real).setResultsName("v_magnitude")
v_angle = Optional(real).setResultsName("v_angle") # radians
area = Optional(integer).setResultsName("area") # not used yet
region = Optional(integer).setResultsName("region") # not used yet
bus_data = bus_no + v_base + v_magnitude + v_angle + \
area + region + scolon
bus_data.setParseAction(self.push_bus)
bus_array = Literal("Bus.con") + "=" + "[" + "..." + \
ZeroOrMore(bus_data + Optional("]" + scolon))
# Sort buses according to their name (bus_no)
bus_array.setParseAction(self.sort_buses)
return bus_array
def __init__(self, max_=60):
# define the grammar structure
digits = "0123456789"
star = Literal('*')
number = Word(digits) | Word(alphas)
steps = number
range_ = number + Optional(Literal('-') + number)
numspec = star | range_
expr = Group(numspec) + Optional(Literal('/') + steps)
extra_groups = ZeroOrMore(Literal(',') + expr)
groups = expr + extra_groups + StringEnd()
# define parse actions
star.setParseAction(self._expand_star)
number.setParseAction(self._expand_number)
range_.setParseAction(self._expand_range)
expr.setParseAction(self._filter_steps)
extra_groups.setParseAction(self._ignore_comma)
groups.setParseAction(self._join_to_set)
self.max_ = max_
self.parser = groups
def get_raw_stack(self):
lpar = Literal("(")
rpar = Literal(")")
diff = Literal("-")
intersection = Literal("n")
union = Literal("U")
operand = Word(alphanums)
operator = diff | intersection | union
sexp = Forward()
term = operand.setParseAction(
self.push_first_raw_stack) | lpar.setParseAction(
self.push_first_raw_stack) + sexp + rpar.setParseAction(
self.push_first_raw_stack)
sexp << term + ZeroOrMore(
(operator + term).setParseAction(self.push_first_raw_stack))
sexp.parseString(self.expression)
return self.raw_stack
def __init__(self, max_=60):
# define the grammar structure
digits = "0123456789"
star = Literal('*')
number = Word(digits) | Word(alphas)
steps = number
range_ = number + Optional(Literal('-') + number)
numspec = star | range_
expr = Group(numspec) + Optional(Literal('/') + steps)
extra_groups = ZeroOrMore(Literal(',') + expr)
groups = expr + extra_groups + StringEnd()
# define parse actions
star.setParseAction(self._expand_star)
number.setParseAction(self._expand_number)
range_.setParseAction(self._expand_range)
expr.setParseAction(self._filter_steps)
extra_groups.setParseAction(self._ignore_comma)
groups.setParseAction(self._join_to_set)
self.max_ = max_
self.parser = groups
def define_math(self):
digits = Word(nums)
variable = Word(alphas +
self._grammar.get_token("namespace_separator"))
to_dot = lambda t: "."
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(
self._grammar.get_token("thousands_separator"))
thousands = Word(
nums, max=3) + OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
expop = Literal('^')
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
factop = Literal('!')
modop = Literal('%')
operand = Combine((integers + Optional(decimals)) | variable)
expr = operatorPrecedence(operand, [
(Literal("["), 1, opAssoc.RIGHT),
(Literal("]"), 1, opAssoc.LEFT),
("!", 1, opAssoc.LEFT),
(expop, 2, opAssoc.RIGHT),
(signop, 1, opAssoc.RIGHT),
(multop, 2, opAssoc.LEFT),
(modop, 2, opAssoc.LEFT),
(plusop, 2, opAssoc.LEFT),
])
expr = Combine(expr)
expr.setParseAction(self.make_arithmetic)
expr.setName("arithmetic")
return expr
def make_arithmetic(self, s, l, tokens):
digits = Word(nums)
variable = Word(alphas +
self._grammar.get_token("namespace_separator"))
to_dot = lambda t: "."
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(
self._grammar.get_token("thousands_separator"))
thousands = Word(
nums, max=3) + OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
expop = Literal('^')
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
modop = Literal('%')
factop = Literal("!")
operand = Combine((integers + Optional(decimals)) | variable)
expr = operatorPrecedence(operand, [
(Literal("[").setParseAction(replaceWith("(")), 1, opAssoc.RIGHT),
(Literal("]").setParseAction(replaceWith(")")), 1, opAssoc.LEFT),
("!", 1, opAssoc.LEFT),
(expop, 2, opAssoc.RIGHT),
(signop, 1, opAssoc.RIGHT),
(multop, 2, opAssoc.LEFT),
(modop, 2, opAssoc.LEFT),
(plusop, 2, opAssoc.LEFT),
])
a = expr.parseString(tokens[0], parseAll=True)
return Arithmetic(a[0], self._namespace,
self._grammar.get_token("namespace_separator"))
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack,rbrack,lbrace,rbrace,lparen,rparen,colon,qmark = map(Literal,"[]{}():?")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack,ignore=escapedChar) + rbrack)
reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = (
( lbrace + Word(nums)("count") + rbrace ) |
( lbrace + Word(nums)("minCount")+","+ Word(nums)("maxCount") + rbrace ) |
oneOf(list("*+?"))
)
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
reTerm = ( reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
reExpr = infixNotation( reTerm,
[
(repetition, 1, opAssoc.LEFT, handleRepetition),
(None, 2, opAssoc.LEFT, handleSequence),
(Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
]
)
_parser = reExpr
return _parser
def getToken(self):
"""
Создать элементы из прикрепленных файлов.
Отдельно картинки, отдельно все файлы
"""
attachesAll = []
attaches = Attachment(self.parser.page).attachmentFull
attaches.sort(key=len, reverse=True)
for attach in attaches:
fname = os.path.basename(attach)
if self.filterFile(fname):
attach = Literal(fname)
attachesAll.append(attach)
finalToken = Literal(self.attachString) + concatenate(attachesAll)
finalToken = finalToken.setParseAction(self.convertToLink)("attach")
return finalToken
def rdata_grammar():
pairs = Forward()
comma_separated_values = constant + ZeroOrMore(Literal(",").suppress() + constant)
vector = Literal("c(") + comma_separated_values + Literal(")").setName(') to close c')
vector.setParseAction(lambda s,l,t: [np.array(t[1:-1])])
array = Literal("structure(") + pairs + Literal(")").setName(') to close structure')
array.setParseAction(lambda s,l,t: [_build_array(t[1])])
value = constant | vector | array
pair = variable + Literal("=").suppress() + value
pair.setParseAction(lambda s,l,t: [t[:]])
pairs << (pair + ZeroOrMore(Literal(",").suppress() + pair))
pairs.setParseAction(lambda s,l,t: [dict(t[:])])
data = Literal("list(") + pairs + Literal(")").setName(') to close list')
data.setParseAction(lambda s,l,t: [t[1]])
return data
def parseCTLK(spec):
"""Parse the spec and return the list of possible ASTs."""
global __ctlk
if __ctlk is None:
true = Literal("True")
true.setParseAction(lambda tokens: TrueExp())
false = Literal("False")
false.setParseAction(lambda tokens: FalseExp())
init = Literal("Init")
init.setParseAction(lambda tokens: Init())
reachable = Literal("Reachable")
reachable.setParseAction(lambda tokens: Reachable())
atom = "'" + SkipTo("'") + "'"
atom.setParseAction(lambda tokens: Atom(tokens[1]))
agent = atom
group = Group(ZeroOrMore(agent + Suppress(",")) + agent)
proposition = true | false | init | reachable | atom
__ctlk = Forward()
notproposition = "~" + proposition
notproposition.setParseAction(lambda tokens: Not(tokens[1]))
formula = (proposition | notproposition |
Suppress("(") + __ctlk + Suppress(")"))
logical = Forward()
ex = Literal("E") + "X" + logical
ex.setParseAction(lambda tokens: EX(tokens[2]))
ax = Literal("A") + "X" + logical
ax.setParseAction(lambda tokens: AX(tokens[2]))
ef = Literal("E") + "F" + logical
ef.setParseAction(lambda tokens: EF(tokens[2]))
af = Literal("A") + "F" + logical
af.setParseAction(lambda tokens: AF(tokens[2]))
eg = Literal("E") + "G" + logical
eg.setParseAction(lambda tokens: EG(tokens[2]))
ag = Literal("A") + "G" + logical
ag.setParseAction(lambda tokens: AG(tokens[2]))
eu = Literal("E") + "[" + __ctlk + "U" + __ctlk + "]"
eu.setParseAction(lambda tokens: EU(tokens[2], tokens[4]))
au = Literal("A") + "[" + __ctlk + "U" + __ctlk + "]"
au.setParseAction(lambda tokens: AU(tokens[2], tokens[4]))
ew = Literal("E") + "[" + __ctlk + "W" + __ctlk + "]"
ew.setParseAction(lambda tokens: EW(tokens[2], tokens[4]))
aw = Literal("A") + "[" + __ctlk + "W" + __ctlk + "]"
aw.setParseAction(lambda tokens: AW(tokens[2], tokens[4]))
temporal = (ex | ax | ef | af | eg | ag | eu | au | ew | aw)
nk = Literal("nK") + "<" + agent + ">" + logical
nk.setParseAction(lambda tokens: nK(tokens[2], tokens[4]))
k = Literal("K") + "<" + agent + ">" + logical
k.setParseAction(lambda tokens: K(tokens[2], tokens[4]))
ne = Literal("nE") + "<" + group + ">" + logical
ne.setParseAction(lambda tokens: nE(list(tokens[2]), tokens[4]))
e = Literal("E") + "<" + group + ">" + logical
e.setParseAction(lambda tokens: E(list(tokens[2]), tokens[4]))
nd = Literal("nD") + "<" + group + ">" + logical
nd.setParseAction(lambda tokens: nD(list(tokens[2]), tokens[4]))
d = Literal("D") + "<" + group + ">" + logical
d.setParseAction(lambda tokens: D(list(tokens[2]), tokens[4]))
nc = Literal("nC") + "<" + group + ">" + logical
nc.setParseAction(lambda tokens: nC(list(tokens[2]), tokens[4]))
c = Literal("C") + "<" + group + ">" + logical
c.setParseAction(lambda tokens: C(list(tokens[2]), tokens[4]))
epistemic = (nk | k | ne | e | nd | d | nc | c)
logical <<= (formula | epistemic | temporal)
__ctlk <<= (_logicals_(logical))
return __ctlk.parseString(spec, parseAll = True)
def parseATL(spec):
"""Parse the spec and return the list of possible ASTs."""
global __atl
if __atl is None:
true = Literal("True")
true.setParseAction(lambda tokens: TrueExp())
false = Literal("False")
false.setParseAction(lambda tokens: FalseExp())
atom = "'" + SkipTo("'") + "'"
atom.setParseAction(lambda tokens: Atom(tokens[1]))
agent = atom
group = Group(ZeroOrMore(agent + Suppress(",")) + agent)
proposition = true | false | atom
__atl = Forward()
notproposition = "~" + proposition
notproposition.setParseAction(lambda tokens: Not(tokens[1]))
formula = (proposition | notproposition
| Suppress("(") + __atl + Suppress(")"))
logical = Forward()
cax = Literal("[") + group + "]" + "X" + logical
cax.setParseAction(lambda tokens: CAX(tokens[1], tokens[4]))
cex = Literal("<") + group + ">" + "X" + logical
cex.setParseAction(lambda tokens: CEX(tokens[1], tokens[4]))
caf = Literal("[") + group + "]" + "F" + logical
caf.setParseAction(lambda tokens: CAF(tokens[1], tokens[4]))
cef = Literal("<") + group + ">" + "F" + logical
cef.setParseAction(lambda tokens: CEF(tokens[1], tokens[4]))
cag = Literal("[") + group + "]" + "G" + logical
cag.setParseAction(lambda tokens: CAG(tokens[1], tokens[4]))
ceg = Literal("<") + group + ">" + "G" + logical
ceg.setParseAction(lambda tokens: CEG(tokens[1], tokens[4]))
cau = Literal("[") + group + "]" + "[" + __atl + "U" + __atl + "]"
cau.setParseAction(lambda tokens: CAU(tokens[1], tokens[4], tokens[6]))
ceu = Literal("<") + group + ">" + "[" + __atl + "U" + __atl + "]"
ceu.setParseAction(lambda tokens: CEU(tokens[1], tokens[4], tokens[6]))
caw = Literal("[") + group + "]" + "[" + __atl + "W" + __atl + "]"
caw.setParseAction(lambda tokens: CAW(tokens[1], tokens[4], tokens[6]))
cew = Literal("<") + group + ">" + "[" + __atl + "W" + __atl + "]"
cew.setParseAction(lambda tokens: CEW(tokens[1], tokens[4], tokens[6]))
strategic = (cax | cex | caf | cef | cag | ceg | cau | ceu | caw | cew)
logical <<= (formula | strategic)
__atl <<= (_logicals_(logical))
return __atl.parseString(spec, parseAll=True)
def __init__(self):
self.ae = False
self.local_dict = None
self.f = None
self.user_functions = None
self.expr_stack = []
self.texpr_stack = []
# Define constants
self.constants = {}
# Define Operators
self.opn = {"+": operator.add,
"-": operator.sub,
"*": operator.mul,
"/": operator.truediv,
">": operator.gt,
">=": operator.ge,
"<": operator.lt,
"<=": operator.le,
"==": operator.eq,
"!=": operator.ne,
"|": operator.or_,
"&": operator.and_,
"!": operator.inv}
# Define xarray DataArray operators with 1 input parameter
self.xfn1 = {"angle": xr.ufuncs.angle,
"arccos": xr.ufuncs.arccos,
"arccosh": xr.ufuncs.arccosh,
"arcsin": xr.ufuncs.arcsin,
"arcsinh": xr.ufuncs.arcsinh,
"arctan": xr.ufuncs.arctan,
"arctanh": xr.ufuncs.arctanh,
"ceil": xr.ufuncs.ceil,
"conj": xr.ufuncs.conj,
"cos": xr.ufuncs.cos,
"cosh": xr.ufuncs.cosh,
"deg2rad": xr.ufuncs.deg2rad,
"degrees": xr.ufuncs.degrees,
"exp": xr.ufuncs.exp,
"expm1": xr.ufuncs.expm1,
"fabs": xr.ufuncs.fabs,
"fix": xr.ufuncs.fix,
"floor": xr.ufuncs.floor,
"frexp": xr.ufuncs.frexp,
"imag": xr.ufuncs.imag,
"iscomplex": xr.ufuncs.iscomplex,
"isfinite": xr.ufuncs.isfinite,
"isinf": xr.ufuncs.isinf,
"isnan": xr.ufuncs.isnan,
"isreal": xr.ufuncs.isreal,
"log": xr.ufuncs.log,
"log10": xr.ufuncs.log10,
"log1p": xr.ufuncs.log1p,
"log2": xr.ufuncs.log2,
"rad2deg": xr.ufuncs.rad2deg,
"radians": xr.ufuncs.radians,
"real": xr.ufuncs.real,
"rint": xr.ufuncs.rint,
"sign": xr.ufuncs.sign,
"signbit": xr.ufuncs.signbit,
"sin": xr.ufuncs.sin,
"sinh": xr.ufuncs.sinh,
"sqrt": xr.ufuncs.sqrt,
"square": xr.ufuncs.square,
"tan": xr.ufuncs.tan,
"tanh": xr.ufuncs.tanh,
"trunc": xr.ufuncs.trunc}
# Define xarray DataArray operators with 2 input parameter
self.xfn2 = {"arctan2": xr.ufuncs.arctan2,
"copysign": xr.ufuncs.copysign,
"fmax": xr.ufuncs.fmax,
"fmin": xr.ufuncs.fmin,
"fmod": xr.ufuncs.fmod,
"hypot": xr.ufuncs.hypot,
"ldexp": xr.ufuncs.ldexp,
"logaddexp": xr.ufuncs.logaddexp,
"logaddexp2": xr.ufuncs.logaddexp2,
"logicaland": xr.ufuncs.logical_and,
"logicalnot": xr.ufuncs.logical_not,
"logicalor": xr.ufuncs.logical_or,
"logicalxor": xr.ufuncs.logical_xor,
"maximum": xr.ufuncs.maximum,
"minimum": xr.ufuncs.minimum,
"nextafter": xr.ufuncs.nextafter}
# Define non-xarray DataArray operators with 2 input parameter
self.fn2 = {"percentile": np.percentile}
# Define xarray DataArray reduction operators
self.xrfn = {"all": xr.DataArray.all,
"any": xr.DataArray.any,
"argmax": xr.DataArray.argmax,
"argmin": xr.DataArray.argmin,
"max": xr.DataArray.max,
"mean": xr.DataArray.mean,
"median": xr.DataArray.median,
"min": xr.DataArray.min,
"prod": xr.DataArray.prod,
"sum": xr.DataArray.sum,
"std": xr.DataArray.std,
"var": xr.DataArray.var}
# Define non-xarray DataArray operators with 2 input parameter
self.xcond = {"<": np.percentile}
# Define Grammar
point = Literal(".")
e = CaselessLiteral("E")
fnumber = Combine(Word("+-"+nums, nums) +
Optional(point + Optional(Word(nums))) +
Optional(e + Word("+-"+nums, nums)))
variable = Word(alphas, alphas+nums+"_$")
seq = Literal("=")
b_not = Literal("~")
plus = Literal("+")
minus = Literal("-")
mult = Literal("*")
div = Literal("/")
gt = Literal(">")
gte = Literal(">=")
lt = Literal("<")
lte = Literal("<=")
eq = Literal("==")
neq = Literal("!=")
b_or = Literal("|")
b_and = Literal("&")
l_not = Literal("!")
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
comma = Literal(",")
colon = Literal(":")
lbrac = Literal("[")
rbrac = Literal("]")
lcurl = Literal("{")
rcurl = Literal("}")
qmark = Literal("?")
scolon = Literal(";")
addop = plus | minus
multop = mult | div
sliceop = colon
compop = gte | lte | gt | lt
eqop = eq | neq
bitcompop = b_or | b_and
bitnotop = b_not
logicalnotop = l_not
assignop = seq
expop = Literal("^")
expr = Forward()
indexexpr = Forward()
atom = (Optional("-") +
(variable + seq + expr).setParseAction(self.push_assign) |
indexexpr.setParseAction(self.push_index) |
(lpar + expr + qmark.setParseAction(self.push_ternary1) + expr +
scolon.setParseAction(self.push_ternary2) + expr +
rpar).setParseAction(self.push_ternary) |
(lpar + expr + qmark + expr + scolon + expr +
rpar).setParseAction(self.push_ternary) |
(logicalnotop + expr).setParseAction(self.push_ulnot) |
(bitnotop + expr).setParseAction(self.push_unot) |
(minus + expr).setParseAction(self.push_uminus) |
(variable + lcurl + expr +
rcurl).setParseAction(self.push_mask) |
(variable + lpar + expr + (comma + expr)*3 +
rpar).setParseAction(self.push_expr4) |
(variable + lpar + expr + (comma + expr)*2 +
rpar).setParseAction(self.push_expr3) |
(variable + lpar + expr + comma + expr +
rpar).setParseAction(self.push_expr2) |
(variable + lpar + expr + rpar |
variable).setParseAction(self.push_expr1) |
fnumber.setParseAction(self.push_expr) |
(lpar + expr + ZeroOrMore(comma + expr).setParseAction(self.get_tuple) +
rpar).setParseAction(self.push_tuple) |
(lpar + expr.suppress() +
rpar).setParseAction(self.push_uminus))
# Define order of operations for operators
factor = Forward()
factor << atom + ZeroOrMore((expop + factor).setParseAction(self.push_op))
term = factor + ZeroOrMore((multop + factor).setParseAction(self.push_op))
term2 = term + ZeroOrMore((addop + term).setParseAction(self.push_op))
term3 = term2 + ZeroOrMore((sliceop + term2).setParseAction(self.push_op))
term4 = term3 + ZeroOrMore((compop + term3).setParseAction(self.push_op))
term5 = term4 + ZeroOrMore((eqop + term4).setParseAction(self.push_op))
term6 = term5 + ZeroOrMore((bitcompop + term5).setParseAction(self.push_op))
expr << term6 + ZeroOrMore((assignop + term6).setParseAction(self.push_op))
# Define index operators
colon_expr = (colon + FollowedBy(comma) ^ colon +
FollowedBy(rbrac)).setParseAction(self.push_colon)
range_expr = colon_expr | expr | colon
indexexpr << (variable + lbrac + delimitedList(range_expr, delim=',') +
rbrac).setParseAction(self.push_expr)
self.parser = expr
hexdigits = Word(string.hexdigits, exact=2)
hexdigits.setName('hexdigits')
escaped = Suppress(Literal('\\')) + hexdigits
escaped.setName('escaped')
def _p_escaped(s, l, t):
text = t[0]
return chr(int(text, 16))
escaped.setParseAction(_p_escaped)
value = Combine(OneOrMore(CharsNotIn('*()\\\0') | escaped))
value.setName('value')
equal = Literal("=")
equal.setParseAction(lambda s, l, t: pureldap.LDAPFilter_equalityMatch)
approx = Literal("~=")
approx.setParseAction(lambda s, l, t: pureldap.LDAPFilter_approxMatch)
greater = Literal(">=")
greater.setParseAction(lambda s, l, t: pureldap.LDAPFilter_greaterOrEqual)
less = Literal("<=")
less.setParseAction(lambda s, l, t: pureldap.LDAPFilter_lessOrEqual)
filtertype = equal | approx | greater | less
filtertype.setName('filtertype')
simple = attr + filtertype + value
simple.leaveWhitespace()
simple.setName('simple')
def _p_simple(s, l, t):
attr, filtertype, value = t
return ("<"+tok[0]+"> " + " ".join(tok)).replace("\"","\\\"")
field_def.setParseAction(field_act)
field_list_def = delimitedList( field_def )
def field_list_act(toks):
return " | ".join(toks)
field_list_def.setParseAction(field_list_act)
create_table_def = Literal("CREATE") + "TABLE" + Word(alphas,alphanums+"_").setResultsName("tablename") + \
"("+field_list_def.setResultsName("columns")+")"+ ";"
def create_table_act(toks):
return """"%(tablename)s" [\n\t label="<%(tablename)s> %(tablename)s | %(columns)s"\n\t shape="record"\n];""" % toks
create_table_def.setParseAction(create_table_act)
add_fkey_def=Literal("ALTER")+"TABLE"+"ONLY" + Word(alphanums+"_").setResultsName("fromtable") + "ADD" \
+ "CONSTRAINT" + Word(alphanums+"_") + "FOREIGN"+"KEY"+"("+Word(alphanums+"_").setResultsName("fromcolumn")+")" \
+"REFERENCES"+Word(alphanums+"_").setResultsName("totable")+"("+Word(alphanums+"_").setResultsName("tocolumn")+")"+";"
def add_fkey_act(toks):
return """ "%(fromtable)s":%(fromcolumn)s -> "%(totable)s":%(tocolumn)s """ % toks
add_fkey_def.setParseAction(add_fkey_act)
other_statement_def = ( OneOrMore(CharsNotIn(";") ) + ";")
other_statement_def.setParseAction( replaceWith("") )
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
comment_def.setParseAction( replaceWith("") )
statement_def = comment_def | create_table_def | add_fkey_def | other_statement_def
def _parse_line(self):
""" Parses a single line, and returns a node representing the active context
Further lines processed are expected to be children of the active context, or children of its accestors.
------------------------------------------------
Basic grammar is as follows:
line = <mako>|<nemo>|<string>
<mako>
We don't parse normally parse tags, so the following info is sketchy.
Mako tags are recognized as anythign that starts with:
- <%
- %>
- %CLOSETEXT
- </%
Mako Control tags however are parsed, and required to adhere to the same indentation rules as Nemo tags.
mako_control = <start>|<middle>|<end>
start = (for|if|while) <inner>:
middle = (else|elif):
end = endfor|endwhile
nemo = % ( <mako_control>|<nemo_statement> )
nemo_statement = .<quote><string><quote>|#<quote><string><quote>|<words>
<quote> = '|"
Notes: Quotes are required to be balanced.
Quotes preceded by a \ are ignored.
<string> = *
words = \w+
"""
#if self.debug: print '\t ' + str(self._current_node)
# PyParser setParseAction's actually execute during parsing,
# So we need closures in order to change the current scope
def depth_from_indentation(function):
""" Set the depth as the start of the match """
def wrap(start, values):
#print 'Depth %d | %d %s' %(self._depth, start, values)
#self._depth = start
self._current_node = function(values)
#print self._current_node
return ''
return wrap
def depth_from_match(function):
""" Set the depth as the start of the match """
def wrap(start, values):
#print 'Depth %d | %d %s' %(self._depth, start, values)
#print self._current_node
self._depth = start
self._current_node = function(values)
#print self._current_node
return ''
return wrap
def depth_from_nemo_tag(function):
""" Start of the match is where the nemo tag is. Pass the other values to the wrapped function """
def wrap(start, values):
# print 'Depth %d | %d %s' %(self._depth, start, values)
self._depth = start
tokens = values[1]
self._current_node = function(tokens)
#print self._current_node
return ''
return wrap
# Match HTML
from pyparsing import NotAny, MatchFirst
html = restOfLine
html.setParseAction(depth_from_indentation(self._add_html_node))
# Match Mako control tags
nemo_tag = Literal('%')
begin = Keyword('for') | Keyword('if') | Keyword('while')
middle = Keyword('else') | Keyword('elif')
end = Keyword('endfor') | Keyword('endif') | Keyword('endwhile')
control = nemo_tag + (begin | middle | end)
begin.setParseAction(depth_from_indentation(self._add_nesting_mako_control_node) )
middle.setParseAction(depth_from_indentation(self._add_mako_middle_node))
end.setParseAction(depth_from_indentation(self._add_mako_control_leaf))
# Match Nemo tags
argument_name = Word(alphas,alphanums+"_-:")
argument_value = quotedString
regular_argument = argument_name + Literal('=') + argument_value
class_name = Literal('.').setParseAction(lambda x: 'class=')
id_name = Literal('#').setParseAction(lambda x: 'id=')
special_argument = (class_name | id_name) + argument_value
argument = Combine(special_argument) | Combine(regular_argument)
# Match single Nemo statement (Part of a multi-line)
inline_nemo_html = Word(alphas) + Group(ZeroOrMore(argument))
inline_nemo_html.setParseAction(depth_from_match(self._add_nemo_node))
# Match first nemo tag on the line (the one that may begin a multi-statement expression)
nemo_html = nemo_tag + Group(Word(alphanums+"_-:") + Group(ZeroOrMore(argument)))
nemo_html.setParseAction(depth_from_nemo_tag(self._add_nemo_node))
# Match a multi-statement expression. Nemo statements are seperated by |. Anything after || is treated as html
separator = Literal('|').suppress()
html_separator = Literal('||') # | Literal('|>')
nemo_list = nemo_html + ZeroOrMore( separator + inline_nemo_html )
inline_html = html.copy()
inline_html.setParseAction(depth_from_match(self._add_inline_html_node))
nemo_multi = nemo_list + Optional(html_separator + inline_html)
# Match empty Nemo statement
empty = nemo_tag + Empty()
empty.setParseAction(depth_from_indentation(self._add_blank_nemo_node))
# Match unused Mako tags
mako_tags = Literal('<%') | Literal('%>') | Literal('%CLOSETEXT') | Literal('</%')
mako = mako_tags
mako_tags.setParseAction(depth_from_indentation(self._add_html_node))
# Matches General
nemo = (control | nemo_multi | empty)
line = mako_tags | nemo | html
# Depth Calculation (deprecated?)
self._depth = len(self._c) - len(self._c.strip())
#try:
line.parseString(self._c)
selectcase_kwd.setParseAction(lambda s,loc,toks: [' '.join(toks)])
selectcase_stmt = selectcase_kwd + if_expr + EOLL
selectcase_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'end select'])
where_stmt = Keyword('where') + if_expr + EOLL
where_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'end where'])
arglist = '(' + (delimitedList(NAME, delim=',') | empty) + ')'
arglist.setParseAction(lambda s,loc,toks: '('+', '.join(toks[1:-1]) + ')')
arglist.parseString('(foo,bar)')
name_eq = Literal('name') + '=' + quotedString
bind_keyword = Keyword('bind').setParseAction(lambda s,loc,toks: [' bind'])
bind_attr = bind_keyword + Literal('(') + 'c' + Optional(comma + name_eq) + ')'
result_attr = Literal('result') + '(' + NAME + ')'
result_attr.setParseAction(lambda s,loc,toks: [' '+''.join(toks)])
func_post = eachMostOnce(bind_attr, result_attr)
elem_attr = Keyword('elemental').setName('elemental')
elem_attr.setParseAction(lambda s,loc,toks: [toks[0] + ' '])
pure_attr = Keyword('pure').setName('pure')
pure_attr.setParseAction(lambda s,loc,toks: [toks[0] + ' '])
recu_attr = Keyword('recursive').setName('elemental')
recu_attr.setParseAction(lambda s,loc,toks: [toks[0] + ' '])
func_pre = eachMostOnce(elem_attr, pure_attr, recu_attr)
func_name = NAME.copy().setName('func_name')
func_name.setParseAction(lambda s,loc,toks: [' '+toks[0]])
func_def = Optional(func_pre) + Keyword('function') + func_name + arglist \
STRING_LITERAL1 = Regex(ur"'(?:[^'\n\r\\]|\\['ntbrf\\])*'(?!')", flags=re.U)
STRING_LITERAL1.setParseAction(
lambda x: rdflib.Literal(x[0][1:-1].decode("string-escape") if not rdflib.py3compat.PY3 else x[0][1:-1])
)
# [157] STRING_LITERAL2 ::= '"' ( ([^#x22#x5C#xA#xD]) | ECHAR )* '"'
# STRING_LITERAL2 = Literal('"') + ZeroOrMore ( Regex(u'[^\u0022\u005C\u000A\u000D]',flags=re.U) | ECHAR ) + '"'
STRING_LITERAL2 = Regex(ur'"(?:[^"\n\r\\]|\\["ntbrf\\])*"(?!")', flags=re.U)
STRING_LITERAL2.setParseAction(
lambda x: rdflib.Literal(x[0][1:-1].decode("string-escape") if not rdflib.py3compat.PY3 else x[0][1:-1])
)
# [161] NIL ::= '(' WS* ')'
NIL = Literal("(") + ")"
NIL.setParseAction(lambda x: rdflib.RDF.nil)
# [162] WS ::= #x20 | #x9 | #xD | #xA
# Not needed?
# WS = #x20 | #x9 | #xD | #xA
# [163] ANON ::= '[' WS* ']'
ANON = Literal("[") + "]"
ANON.setParseAction(lambda x: rdflib.BNode())
# A = CaseSensitiveKeyword('a')
A = Literal("a")
A.setParseAction(lambda x: rdflib.RDF.type)
# ------ NON-TERMINALS --------------
def _makeimagemap(tokens):
image = None
for x in tokens:
if isinstance(x, basestring):
image = x
break
return ImageMap(entries=list(tokens), image=image)
comment = (Literal('#') + restOfLine).setParseAction(_makecomment)
integer = Word(nums).setParseAction(lambda s: int(s[0]))
integer_pair = (integer + integer).setParseAction(lambda x: tuple(x))
poly = Literal("poly") + Group(ZeroOrMore(integer_pair)) + restOfLine
poly = poly.setParseAction(_makepoly)
rect = Literal("rect") + integer_pair + integer_pair + restOfLine
rect = rect.setParseAction(_makerect)
circle = Literal("circle") + integer_pair + integer + restOfLine
circle = circle.setParseAction(_makecircle)
desc = Literal("desc") + (Literal("top-right")
| Literal("bottom-right")
| Literal("bottom-left")
| Literal("top-left")
| Literal("none"))
desc = desc.setParseAction(_makedesc)
default = Literal("default") + restOfLine
default.setParseAction(lambda t: Default(caption=t[1].strip()))
def field_list_act(toks):
return " | ".join(toks)
field_list_def.setParseAction(field_list_act)
create_table_def = Literal("CREATE") + "TABLE" + Word(alphas,alphanums+"_").setResultsName("tablename") + \
"("+field_list_def.setResultsName("columns")+")"+ ";"
def create_table_act(toks):
return """"%(tablename)s" [\n\t label="<%(tablename)s> %(tablename)s | %(columns)s"\n\t shape="record"\n];""" % toks
create_table_def.setParseAction(create_table_act)
add_fkey_def=Literal("ALTER")+"TABLE"+"ONLY" + Word(alphanums+"_").setResultsName("fromtable") + "ADD" \
+ "CONSTRAINT" + Word(alphanums+"_") + "FOREIGN"+"KEY"+"("+Word(alphanums+"_").setResultsName("fromcolumn")+")" \
+"REFERENCES"+Word(alphanums+"_").setResultsName("totable")+"("+Word(alphanums+"_").setResultsName("tocolumn")+")"+";"
def add_fkey_act(toks):
return """ "%(fromtable)s":%(fromcolumn)s -> "%(totable)s":%(tocolumn)s """ % toks
add_fkey_def.setParseAction(add_fkey_act)
other_statement_def = (OneOrMore(CharsNotIn(";")) + ";")
other_statement_def.setParseAction(replaceWith(""))
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
class ObfuscatePythonBNF(ObfuscateBNF):
def __init__(self, get_obfuscated):
"""BNF grammar for Python source statements.
Parameters
----------
get_obfuscated : function
Function to return the obfuscated name for an identifier.
"""
super(ObfuscatePythonBNF, self).__init__(get_obfuscated)
self.validator = \
Literal('@') + \
Literal('validate') + \
Literal('(') + \
self.string + \
self.string + \
Literal(')')
# Parse a Kivy load_file statement
self.builder = \
Literal('Builder.load_file(') + \
self.string + \
Literal(')')
self.statement = (
ZeroOrMore(
(self.directive |
self.builder |
self.tab |
self.conseq_idents_numbs |
self.separator |
self.string_or_doc |
self.triple_quote)
) + Optional(self.comment).suppress()
)
self.except_error = (
ZeroOrMore(
(self.tab.suppress() |
Literal('except') |
self.directive.suppress() |
self.tab.suppress() |
self.ident |
self.separator.suppress() |
self.fnumber.suppress() |
self.string_or_doc.suppress() |
self.triple_quote.suppress())
) + Optional(self.comment).suppress()
)
self.from_import = (
ZeroOrMore(
(self.tab.suppress() |
Literal('from') |
self.directive.suppress() |
self.tab.suppress() |
self.ident |
Literal('import') |
self.separator.suppress() |
self.fnumber.suppress() |
self.string_or_doc.suppress() |
self.triple_quote.suppress())
) + Optional(self.comment).suppress()
)
self.except_error.setParseAction(self.add_except_error)
self.builder.setParseAction(self.transform_builder)
self.from_import.setParseAction(self.add_from_import)
###############
# Parse actions
###############
def add_from_import(self, from_import_list):
"""Add imported modules from reserved modules to reserved.
Parameters
----------
from_import_list : list
"""
if not from_import_list or \
from_import_list[0] != 'from' or \
'import' not in from_import_list[:]:
return
reserved_list = set()
import_index = from_import_list[:].index('import')
package_name = ''
is_reserved = False
for reserve_name in from_import_list[1:import_index]:
# Start with first reserved directory in tree (if one exists)
if not is_reserved:
try:
get_reserved_by_name(reserve_name)
is_reserved = True
package_name = reserve_name
except DoesNotExist:
continue
if is_reserved:
if reserve_name[0].isalpha() or reserve_name[0] == '_':
reserved_list.add(reserve_name)
if is_reserved:
# Get imported items
for reserve_name in from_import_list[import_index+1:]:
if reserve_name[0].isalpha() or reserve_name[0] == '_':
reserved_list.add(reserve_name)
add_reserveds(package_name, reserved_list)
def add_except_error(self, except_error_list):
"""Add except Error names to reserved.
Parameters
----------
except_error_list : list
"""
if not except_error_list or except_error_list[0] != 'except':
return
reserved_list = set()
package_name = 'Except'
for reserve_name in except_error_list[1:]:
if reserve_name == 'as':
break
if reserve_name[0].isalpha() or reserve_name[0] == '_':
reserved_list.add(reserve_name)
if reserved_list:
add_reserveds(package_name, reserved_list)
def transform_builder(self, builder_list):
"""Parse a Kivy load_file statement.
Parameters
builder_list : list
Kivy Builder.load_file statement.
"""
return ''.join([
builder_list[0],
"'",
obfuscate_path(builder_list[1].strip("'")),
"'",
builder_list[2]
])
def dept(deptJson):
print "Processing " + deptJson + "..."
deptCourses = json.loads(open(deptJson).read())
for i in deptCourses["result_data"]:
courseId = i["course_id"].replace(" ", "")
preReqs = i["prerequisites"]
prereqList = []
if preReqs == "":
continue
if re.search("\w{3,4}\s{0,1}\d{3}", preReqs):
if courseId not in courseMap:
courseMap[courseId] = {}
# Replace Department names with DEPT sign
for key, val in deptNameMap.iteritems():
if key in preReqs.lower():
preReqs = re.sub(key, val, preReqs, flags=re.I)
# Remove phone numbers
preReqs = re.sub("\d{3}-\d{3}-\d{4}", "", preReqs)
if "/" in preReqs:
preReqs = preReqs.replace("/", " or ")
# Instructor Permission
if "instructor" in (preReqs.lower()) and \
"permission" in (preReqs.lower()):
courseMap[courseId]["instructorPermission"] = True
m = re.findall("((?:\w{2,4}){0,1}\s{0,1}\d{3}),{0,1}\s{0,1}(,|or|and|AND|OR){0,1}", preReqs)
if m:
n = ["".join(i.split()) for sub in m for i in sub]
preReqs = " ".join(n).upper()
preReqs = ", ".join(preReqs.split())
preReqs = preReqs.replace("OR,", "OR")
preReqs = preReqs.replace("AND,", "AND")
preReqs = rchop(preReqs, ", OR")
preReqs = rchop(preReqs, ", AND")
course_name = Word(string.ascii_uppercase + nums) | Word(nums)
comma_separator = Literal(',')
comma_separator.setParseAction(lambda t: "&&")
and_separator = Literal('AND') | Literal(", AND")
and_separator.setParseAction(lambda t: "&&")
or_separator = Literal('OR') | Literal(", OR")
or_separator.setParseAction(lambda t: "||")
course_line = operatorPrecedence(course_name, [
(and_separator, 2, opAssoc.LEFT,),
(or_separator, 2, opAssoc.LEFT),
(comma_separator, 2, opAssoc.LEFT,),
])
results = course_line.parseString(preReqs)
x = results.asList()[0]
if len(x):
results_list = []
results_list.extend(x if type(x) == list else [x])
results_list = fillDeptIds(results_list)
courseMap[courseId]["prerequisites"] = results_list
fillSatisCourse(courseId, results_list)
STRING_LITERAL1 = Regex(ur"'(?:[^'\n\r\\]|\\['ntbrf\\])*'(?!')", flags=re.U)
STRING_LITERAL1.setParseAction(
lambda x: rdflib.Literal(decodeUnicodeEscape(x[0][1:-1])))
# [157] STRING_LITERAL2 ::= '"' ( ([^#x22#x5C#xA#xD]) | ECHAR )* '"'
# STRING_LITERAL2 = Literal('"') + ZeroOrMore (
# Regex(u'[^\u0022\u005C\u000A\u000D]',flags=re.U) | ECHAR ) + '"'
STRING_LITERAL2 = Regex(ur'"(?:[^"\n\r\\]|\\["ntbrf\\])*"(?!")', flags=re.U)
STRING_LITERAL2.setParseAction(
lambda x: rdflib.Literal(decodeUnicodeEscape(x[0][1:-1])))
# [161] NIL ::= '(' WS* ')'
NIL = Literal('(') + ')'
NIL.setParseAction(lambda x: rdflib.RDF.nil)
# [162] WS ::= #x20 | #x9 | #xD | #xA
# Not needed?
# WS = #x20 | #x9 | #xD | #xA
# [163] ANON ::= '[' WS* ']'
ANON = Literal('[') + ']'
ANON.setParseAction(lambda x: rdflib.BNode())
# A = CaseSensitiveKeyword('a')
A = Literal('a')
A.setParseAction(lambda x: rdflib.RDF.type)
# ------ NON-TERMINALS --------------
# [5] BaseDecl ::= 'BASE' IRIREF
# Basic expressions
_basic_expr = Forward()
_conversion = (
Literal("word1") + Suppress("(") + _basic_expr + Suppress(")")
| Literal("bool") + Suppress("(") + _basic_expr + Suppress(")")
| Literal("toint") + Suppress("(") + _basic_expr + Suppress(")")
| Literal("signed") + Suppress("(") + _basic_expr + Suppress(")")
| Literal("unsigned") + Suppress("(") + _basic_expr + Suppress(")")
)
_conversion.setParseAction(lambda s, l, t: Conversion(t[0], t[1]))
_word_function = Literal("extend") + Suppress("(") + _basic_expr + "," + _basic_expr + Suppress(")") | Literal(
"resize"
) + Suppress("(") + _basic_expr + "," + _basic_expr + Suppress(")")
_word_function.setParseAction(lambda s, l, t: WordFunction(t[0], t[1], t[2]))
_count = Literal("count") + Suppress("(") + delimitedList(_basic_expr) + Suppress(")")
_count.setParseAction(lambda s, l, t: Count(t[1]))
_next = Literal("next") + Suppress("(") + _basic_expr + Suppress(")")
_next.setParseAction(lambda s, l, t: Next(t[1]))
_case_case = _basic_expr + Suppress(":") + _basic_expr + Suppress(";")
_case_body = OneOrMore(_case_case)
_case_body.setParseAction(lambda s, l, t: OrderedDict(zip(t[::2], t[1::2])))
_case = Suppress("case") + _case_body + Suppress("esac")
_case.setParseAction(lambda s, l, t: Case(t[0]))
_base = complex_identifier ^ (
_conversion | _word_function | _count | _next | Suppress("(") + _basic_expr + Suppress(")") | _case | constant
def get_grammar(self):
"""
Defines our grammar for mathematical expressions.
Possibly helpful:
- BNF form of context-free grammar https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form
- Some pyparsing docs http://infohost.nmt.edu/~shipman/soft/pyparsing/web/index.html
"""
# Define + and -
plus = Literal("+")
# Also accept unicode emdash
emdash = Literal("\u2014")
emdash.setParseAction(lambda: "-")
minus = Literal("-") | emdash
plus_minus = plus | minus
# 1 or 1.0 or .1
number_part = Word(nums)
inner_number = Combine((number_part +
Optional("." + Optional(number_part)))
| ("." + number_part))
# Combine() joints the matching parts together in a single token,
# and requires that the matching parts be contiguous (no spaces)
# Define our suffixes
suffix = Word(alphas + '%')
suffix.setParseAction(self.suffix_parse_action)
# Construct number as a group consisting of a text string ("num") and an optional suffix.
# num can include a decimal number and numerical exponent, and can be
# converted to a number using float()
# suffix may contain alphas or %
# Spaces are ignored inside numbers
# Group wraps everything up into its own ParseResults object when parsing
number = Group(
Combine(
inner_number + Optional(
CaselessLiteral("E") + Optional(plus_minus) +
number_part), )("num") +
Optional(suffix)("suffix"))("number")
# Note that calling ("name") on the end of a parser is equivalent to calling
# parser.setResultsName, which is used to pull that result out of a parsed
# expression like a dictionary.
# Construct variable and function names
front = Word(alphas, alphanums) # must start with alpha
subscripts = Word(alphanums + '_') + ~FollowedBy('{') # ~ = not
lower_indices = Literal("_{") + Optional("-") + Word(
alphanums) + Literal("}")
upper_indices = Literal("^{") + Optional("-") + Word(
alphanums) + Literal("}")
# Construct an object name in either of two forms:
# 1. front + subscripts + tail
# 2. front + lower_indices + upper_indices + tail
# where:
# front (required):
# starts with alpha, followed by alphanumeric
# subscripts (optional):
# any combination of alphanumeric and underscores
# lower_indices (optional):
# Of form "_{(-)<alphanumeric>}"
# upper_indices (optional):
# Of form "^{(-)<alphanumeric>}"
# tail (optional):
# any number of primes
name = Combine(front +
Optional(subscripts | (Optional(lower_indices) +
Optional(upper_indices))) +
ZeroOrMore("'"))
# Define a variable as a pyparsing result that contains one object name
variable = Group(name("varname"))("variable")
variable.setParseAction(self.variable_parse_action)
# initialize recursive grammar
expression = Forward()
# Construct functions as consisting of funcname and arguments as
# funcname(arguments)
# where arguments is a comma-separated list of arguments, returned as a list
# Must have at least 1 argument
function = Group(
name("funcname") + Suppress("(") +
Group(delimitedList(expression))("arguments") +
Suppress(")"))("function")
function.setParseAction(self.function_parse_action)
# Define parentheses
parentheses = Group(Suppress("(") + expression +
Suppress(")"))('parentheses')
# Define arrays
array = Group(
Suppress("[") + delimitedList(expression) + Suppress("]"))("array")
# atomic units evaluate directly to number or array without binary operations
atom = number | function | variable | parentheses | array
# Define operations in order of precedence
# Define exponentiation, possibly including negative powers
power = atom + ZeroOrMore(Suppress("^") + Optional(minus)("op") + atom)
power.addParseAction(self.group_if_multiple('power'))
# Define negation (e.g., in 5*-3 --> we need to evaluate the -3 first)
# Negation in powers is handled separately
# This has been arbitrarily assigned a higher precedence than parallel
negation = Optional(minus)("op") + power
negation.addParseAction(self.group_if_multiple('negation'))
# Define the parallel operator 1 || 5 == 1/(1/1 + 1/5)
pipes = Literal('|') + Literal('|')
parallel = negation + ZeroOrMore(Suppress(pipes) + negation)
parallel.addParseAction(self.group_if_multiple('parallel'))
# Define multiplication and division
product = parallel + ZeroOrMore((Literal('*') | Literal('/'))("op") +
parallel)
product.addParseAction(self.group_if_multiple('product'))
# Define sums and differences
# Note that leading - signs are treated by negation
sumdiff = Optional(plus) + product + ZeroOrMore(
plus_minus("op") + product)
sumdiff.addParseAction(self.group_if_multiple('sum'))
# Close the recursion
expression << sumdiff
return expression + stringEnd
scopedIdent = ident + OneOrMore( Literal("::").suppress() + ident )
scopedIdent.setParseAction(lambda t: "_".join(t))
print("(replace namespace-scoped names with C-compatible names)")
print(scopedIdent.transformString( testData ))
# or a crude pre-processor (use parse actions to replace matching text)
def substituteMacro(s,l,t):
if t[0] in macros:
return macros[t[0]]
ident.setParseAction( substituteMacro )
ident.ignore(macroDef)
print("(simulate #define pre-processor)")
print(ident.transformString( testData ))
#################
print("Example of a stripper")
print("----------------------")
from pyparsing import dblQuotedString, LineStart
# remove all string macro definitions (after extracting to a string resource table?)
stringMacroDef = Literal("#define") + ident + "=" + dblQuotedString + LineStart()
stringMacroDef.setParseAction( replaceWith("") )
print(stringMacroDef.transformString( testData ))
hexnum = Combine(Literal("0x") + Word(string.hexdigits)).setParseAction(lambda t: int(t[0][2:], 16))
dimension = identifier.setResultsName("name") + \
Optional(Suppress(Literal("(")) +
(Literal("string") | Literal("uint64")) +
Suppress(Literal(")")), default="string").setResultsName("type")
dimension.setParseAction(parse_dimension)
autoregion = Literal("auto") + integer + integer
staticregion = Literal("region") + integer + hexnum + integer
region = ZeroOrMore(Group(staticregion)) + Optional(Group(autoregion))
region.setParseAction(parse_regions)
subspace = Literal("subspace").suppress() + \
Group(delimitedList(identifier)) + \
Optional(Suppress(Literal("nosearch")) +
Group(delimitedList(identifier)), default=[]) + \
Group(region)
subspace.setParseAction(parse_subspace)
space = Literal("space").suppress() + identifier.setResultsName("name") + \
Literal("dimensions").suppress() + Group(delimitedList(dimension)).setResultsName("dimensions") + \
Literal("key").suppress() + identifier.setResultsName("key") + \
Group(region).setResultsName("keyregions") + \
ZeroOrMore(subspace).setResultsName("subspaces")
space.setParseAction(parse_space)
class TestFillToRegion(unittest.TestCase):
def test_already_met(self):
a = 0
b = 1
while a < (1 << 64):
self.assertEqual([], _fill_to_region(a, 0, 4, a))
hexdigits = Word(string.hexdigits, exact=2)
hexdigits.setName('hexdigits')
escaped = Suppress(Literal('\\')) + hexdigits
escaped.setName('escaped')
def _p_escaped(s, l, t):
text = t[0]
return chr(int(text, 16))
escaped.setParseAction(_p_escaped)
value = Combine(OneOrMore(CharsNotIn('*()\\\0') | escaped))
value.setName('value')
equal = Literal("=")
equal.setParseAction(lambda s, l, t: pureldap.LDAPFilter_equalityMatch)
approx = Literal("~=")
approx.setParseAction(lambda s, l, t: pureldap.LDAPFilter_approxMatch)
greater = Literal(">=")
greater.setParseAction(lambda s, l, t: pureldap.LDAPFilter_greaterOrEqual)
less = Literal("<=")
less.setParseAction(lambda s, l, t: pureldap.LDAPFilter_lessOrEqual)
filtertype = equal | approx | greater | less
filtertype.setName('filtertype')
simple = attr + filtertype + value
simple.leaveWhitespace()
simple.setName('simple')
def _p_simple(s, l, t):
attr, filtertype, value = t
