def _logical_parser(expression): """ Return a new parser parsing logical expressions. This parser recognizes the following grammar, with precedence: <logical> ::= expression | '~' <logical> | <logical> '&' <logical> | <logical> '|' <logical> | <logical> '->' <logical> | <logical> '<->' <logical> .. note:: The parser uses :mod:`pytlq.ast` module's classes to build ASTs. .. credit:: Adapted from Simon Busard's parser parsing logical expressions on atomics. """ parser = Forward() not_strict = Literal('~') + expression not_strict.setParseAction(lambda tokens: Not(tokens[1])) not_ = (not_strict | expression) and_ = not_ + ZeroOrMore(Literal('&') + not_) and_.setParseAction(lambda tokens: _left(And, tokens)) or_ = and_ + ZeroOrMore(Literal('|') + and_) or_.setParseAction(lambda tokens: _left(Or, tokens)) imply = ZeroOrMore(or_ + Literal('->')) + or_ imply.setParseAction(lambda tokens: _right(Imply, tokens)) iff = imply + ZeroOrMore(Literal('<->') + imply) iff.setParseAction(lambda tokens: _left(Iff, tokens)) parser <<= iff return parser
def _define_valued_characters_section(self, heading, characters, parse_action, character_type): """Returns a parser object for a section specifying characters and their valued features. :param heading: section heading :type heading: `str` :param characters: valid characters :type characters: `list` of `str` :param parse_action: parse action for a character :type parse_action: `function` :param character_type: type of characters being described :type character_type: `str` """ heading = Literal('[{}]'.format(heading)) character = Word(''.join(characters), exact=1).setResultsName( 'character') character.setParseAction(self._handle_character) feature = Word(alphas).setResultsName('feature') feature.setParseAction(parse_action) value = Literal('+') ^ Literal('-') ^ Literal('\N{MINUS SIGN}') value.setParseAction(self._handle_feature_value) feature_value = Group(value + feature) feature_values = Group(delimitedList(feature_value)) character_definition = Dict(Group(character + Suppress(':') + feature_values)) character_definitions = Group(OneOrMore(character_definition)).setResultsName(character_type) section = Suppress(heading) + character_definitions return section
def __init__(self, path, text, state=None): self.path = path self.base_path = os.path.dirname(path) self.text = text self.state = state opcode_name = Word(alphanums + '_') value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE) opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value opcode.setParseAction(self.handle_opcode) section_name = Literal('<').suppress() + Word(alphas) + Literal( '>').suppress() section = section_name section.setParseAction(self.handle_section) include = Literal('#include').suppress() + locatedExpr( QuotedString('"')) include.setParseAction(self.handle_include) statement = (section ^ opcode ^ include) self.sfz_file = ZeroOrMore(statement) + stringEnd comment = Literal('//') + restOfLine self.sfz_file.ignore(comment)
def grammar(): parenthesis = Forward() parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")" parenthesis.setParseAction(join_string_act) quoted_string = "'" + OneOrMore(CharsNotIn("'")) + "'" quoted_string.setParseAction(join_string_act) quoted_default_value = "DEFAULT" + quoted_string + OneOrMore(CharsNotIn(", \n\t")) quoted_default_value.setParseAction(quoted_default_value_act) field_def = OneOrMore(quoted_default_value | Word(alphanums + "_\"'`:-/[]") | parenthesis) field_def.setParseAction(field_act) tablename_def = ( Word(alphas + "`_.") | QuotedString("\"") ) field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def) field_list_def.setParseAction(field_list_act) create_table_def = Literal("CREATE") + "TABLE" + tablename_def.setResultsName("tableName") + "(" + field_list_def.setResultsName("fields") + ")" + ";" create_table_def.setParseAction(create_table_act) add_fkey_def = Literal("ALTER") + "TABLE" + "ONLY" + tablename_def.setResultsName("tableName") + "ADD" + "CONSTRAINT" + Word(alphanums + "_") + "FOREIGN" + "KEY" + "(" + Word(alphanums + "_").setResultsName("keyName") + ")" + "REFERENCES" + Word(alphanums + "._").setResultsName("fkTable") + "(" + Word(alphanums + "_").setResultsName("fkCol") + ")" + Optional(Literal("DEFERRABLE")) + Optional(Literal("ON") + "DELETE" + ( Literal("CASCADE") | Literal("RESTRICT") )) + ";" add_fkey_def.setParseAction(add_fkey_act) other_statement_def = OneOrMore(CharsNotIn(";")) + ";" other_statement_def.setParseAction(other_statement_act) comment_def = "--" + ZeroOrMore(CharsNotIn("\n")) comment_def.setParseAction(other_statement_act) return OneOrMore(comment_def | create_table_def | add_fkey_def | other_statement_def)
def grammar(): parenthesis = Forward() parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")" field_def = OneOrMore(Word(alphanums + "_\"'`:-") | parenthesis) field_def.setParseAction(field_act) tablename_def = ( Word(alphas + "`_") | QuotedString("\"") ) field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def) field_list_def.setParseAction(field_list_act) create_table_def = Literal("CREATE") + "TABLE" + tablename_def.setResultsName("tableName") + "(" + field_list_def.setResultsName("fields") + ")" + ";" create_table_def.setParseAction(create_table_act) add_fkey_def = Literal("ALTER") + "TABLE" + "ONLY" + tablename_def.setResultsName("tableName") + "ADD" + "CONSTRAINT" + Word(alphanums + "_") + "FOREIGN" + "KEY" + "(" + Word(alphanums + "_").setResultsName("keyName") + ")" + "REFERENCES" + Word(alphanums + "_").setResultsName("fkTable") + "(" + Word(alphanums + "_").setResultsName("fkCol") + ")" + ";" add_fkey_def.setParseAction(add_fkey_act) other_statement_def = OneOrMore(CharsNotIn(";")) + ";" other_statement_def.setParseAction(other_statement_act) comment_def = "--" + ZeroOrMore(CharsNotIn("\n")) comment_def.setParseAction(other_statement_act) return OneOrMore(comment_def | create_table_def | add_fkey_def | other_statement_def)
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack, rbrack, lbrace, rbrace, lparen, rparen, colon, qmark = map( Literal, "[]{}():?") reMacro = Combine("\\" + oneOf(list("dws"))) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join( c for c in printables if c not in r"\[]{}().*?+|") + " \t" reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack) reLiteral = (escapedChar | oneOf(list(reLiteralChar))) reNonCaptureGroup = Suppress("?:") reDot = Literal(".") repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) | (lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) | oneOf(list("*+?"))) reRange.setParseAction(handleRange) reLiteral.setParseAction(handleLiteral) reMacro.setParseAction(handleMacro) reDot.setParseAction(handleDot) reTerm = (reLiteral | reRange | reMacro | reDot | reNonCaptureGroup) reExpr = infixNotation(reTerm, [ (repetition, 1, opAssoc.LEFT, handleRepetition), (None, 2, opAssoc.LEFT, handleSequence), (Suppress('|'), 2, opAssoc.LEFT, handleAlternative), ]) _parser = reExpr return _parser
def grammar(): parenthesis = Forward() parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")" field_def = OneOrMore(Word(alphanums + "_\"'`:-") | parenthesis) field_def.setParseAction(field_act) field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def) field_list_def.setParseAction(field_list_act) create_table_def = Literal("CREATE") + "TABLE" + Word( alphas + "`_").setResultsName( "tableName") + "(" + field_list_def.setResultsName( "fields") + ")" + ";" create_table_def.setParseAction(create_table_act) add_fkey_def = Literal("ALTER") + "TABLE" + "ONLY" + Word( alphanums + "_").setResultsName("tableName") + "ADD" + "CONSTRAINT" + Word( alphanums + "_") + "FOREIGN" + "KEY" + "(" + Word( alphanums + "_").setResultsName("keyName") + ")" + "REFERENCES" + Word( alphanums + "_").setResultsName("fkTable") + "(" + Word( alphanums + "_").setResultsName("fkCol") + ")" + ";" add_fkey_def.setParseAction(add_fkey_act) other_statement_def = OneOrMore(CharsNotIn(";")) + ";" other_statement_def.setParseAction(other_statement_act) comment_def = "--" + ZeroOrMore(CharsNotIn("\n")) comment_def.setParseAction(other_statement_act) return OneOrMore(comment_def | create_table_def | add_fkey_def | other_statement_def)
def __init__(self, path, text, state=None): self.path = path self.base_path = os.path.dirname(path) self.text = text self.state = state opcode_name = Word(alphanums + '_') value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE) opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value opcode.setParseAction(self.handle_opcode) section_name = Literal('<').suppress() + Word(alphas) + Literal('>').suppress() section = section_name section.setParseAction(self.handle_section) include = Literal('#include').suppress() + locatedExpr(QuotedString('"')) include.setParseAction(self.handle_include) statement = (section ^ opcode ^ include) self.sfz_file = ZeroOrMore(statement) + stringEnd comment = Literal('//') + restOfLine self.sfz_file.ignore(comment)
def parse(self, query): """Parses a query string.""" # Parse instructions quoted_string = QuotedString(quoteChar='"', escChar='\\', unquoteResults=True) field_name = Word(alphas, alphanums + '_') subexpression = Forward() boolean_expression = Forward() binary_operator = Literal('=') | Literal('<=') | Literal('<') | Literal('>=') | Literal('>') boolean_operator = CaselessKeyword('AND') | CaselessKeyword('OR') boolean_not = CaselessKeyword('NOT') boolean_value = CaselessKeyword("true") ^ CaselessKeyword("false") integer = Word(nums) rvalue = quoted_string ^ boolean_value ^ integer field_to_value = field_name + binary_operator + rvalue expression = Optional(boolean_not) + ((subexpression + ZeroOrMore(boolean_expression)) | (field_to_value + ZeroOrMore(boolean_expression))) boolean_expression << boolean_operator + expression left_bracket = Literal('(') right_bracket = Literal(')') subexpression << (left_bracket + expression + right_bracket) search_query = expression # Parse actions for emitting special cases field_to_value.setParseAction(do_field_to_value) boolean_operator.setParseAction(do_boolean_operator) boolean_not.setParseAction(do_boolean_operator) boolean_value.setParseAction(do_boolean_value) integer.setParseAction(do_integer) left_bracket.setParseAction(do_bracket) right_bracket.setParseAction(do_bracket) self.tokens = search_query.parseString(query)
def _define_context_component (self, cluster, base_feature_set): placeholder = Literal(SOURCE_PLACEHOLDER) placeholder.setParseAction(self._handle_placeholder) context_component = Group(ZeroOrMore(cluster ^ base_feature_set) + \ placeholder + ZeroOrMore(cluster ^ base_feature_set)).setResultsName('context_component') context_component.setParseAction(self._handle_context_component) return context_component
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack, rbrack, lbrace, rbrace, lparen, rparen = map(Literal, "[]{}()") reMacro = Combine("\\" + oneOf(list("dws"))) escapedChar = ~ reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t" reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack) reLiteral = (escapedChar | oneOf(list(reLiteralChar))) reDot = Literal(".") repetition = ( (lbrace + Word(nums).setResultsName("count") + rbrace) | (lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) | oneOf(list("*+?")) ) reRange.setParseAction(handle_range) reLiteral.setParseAction(handle_literal) reMacro.setParseAction(handle_macro) reDot.setParseAction(handle_dot) reTerm = (reLiteral | reRange | reMacro | reDot) reExpr = operatorPrecedence(reTerm, [ (repetition, 1, opAssoc.LEFT, handle_repetition), (None, 2, opAssoc.LEFT, handle_sequence), (Suppress('|'), 2, opAssoc.LEFT, handle_alternative), ]) _parser = reExpr return _parser
def _logical_parser(expression): """ Return a new parser parsing logical expressions. This parser recognizes the following grammar, with precedence: <logical> ::= expression | '~' <logical> | <logical> '&' <logical> | <logical> '|' <logical> | <logical> '->' <logical> | <logical> '<->' <logical> .. note:: The parser uses :mod:`pytlq.ast` module's classes to build ASTs. .. credit:: Adapted from Simon Busard's parser parsing logical expressions on atomics. """ parser = Forward() not_strict = Literal('~') + expression not_strict.setParseAction(lambda tokens: Not(tokens[1])) not_ = (not_strict | expression) and_ = not_ + ZeroOrMore(Literal('&') + not_) and_.setParseAction(lambda tokens: _left(And, tokens)) or_ = and_ + ZeroOrMore(Literal('|') + and_) or_.setParseAction(lambda tokens: _left(Or, tokens)) imply = ZeroOrMore(or_ + Literal('->')) + or_ imply.setParseAction(lambda tokens: _right(Imply, tokens)) iff = imply + ZeroOrMore(Literal('<->') + imply) iff.setParseAction(lambda tokens: _left(Iff, tokens)) parser <<= iff return parser
def enumeration_type_definition() -> Token: enumeration_literal = unqualified_identifier() positional_enumeration = enumeration_literal + ZeroOrMore( comma() - enumeration_literal) positional_enumeration.setParseAction( lambda t: [(k, Number(v)) for v, k in enumerate(t.asList())]) element_value_association = enumeration_literal + Keyword( "=>") - numeric_literal() element_value_association.setParseAction(lambda t: (t[0], t[2])) named_enumeration = element_value_association + ZeroOrMore( comma() - element_value_association) boolean_literal = Keyword("True") | Keyword("False") boolean_literal.setParseAction(lambda t: t[0] == "True") boolean_aspect_definition = Optional(Keyword("=>") - boolean_literal) boolean_aspect_definition.setParseAction(lambda t: (t if t else ["=>", True])) always_valid_aspect = Literal("Always_Valid") - boolean_aspect_definition always_valid_aspect.setParseAction(parse_aspect) enumeration_aspects = Keyword("with") - delimitedList( size_aspect() | always_valid_aspect) enumeration_aspects.setParseAction(parse_aspects) return (Literal("(") - (named_enumeration | positional_enumeration) - Literal(")") - enumeration_aspects).setName("Enumeration")
def parseArctl(spec): """Parse the spec and return its AST.""" global __arctl if __arctl is None: true = Literal("True") true.setParseAction(lambda tokens: TrueExp()) false = Literal("False") false.setParseAction(lambda tokens: FalseExp()) atom = "'" + SkipTo("'") + "'" atom.setParseAction(lambda tokens: Atom(tokens[1])) action = _logicals_(atom) __arctl = Forward() proposition = true | false | atom notproposition = "~" + proposition notproposition.setParseAction(lambda tokens: Not(tokens[1])) formula = (proposition | notproposition | Suppress("(") + __arctl + Suppress(")")) temporal = Forward() e = Literal("E") + "<" + action + ">" a = Literal("A") + "<" + action + ">" eax = e + "X" + temporal eax.setParseAction(lambda tokens: EaX(tokens[2], tokens[5])) aax = a + "X" + temporal aax.setParseAction(lambda tokens: AaX(tokens[2], tokens[5])) eaf = e + "F" + temporal eaf.setParseAction(lambda tokens: EaF(tokens[2], tokens[5])) aaf = a + "F" + temporal aaf.setParseAction(lambda tokens: AaF(tokens[2], tokens[5])) eag = e + "G" + temporal eag.setParseAction(lambda tokens: EaG(tokens[2], tokens[5])) aag = a + "G" + temporal aag.setParseAction(lambda tokens: AaG(tokens[2], tokens[5])) eau = e + "[" + __arctl + "U" + __arctl + "]" eau.setParseAction(lambda tokens: EaU(tokens[2], tokens[5], tokens[7])) aau = a + "[" + __arctl + "U" + __arctl + "]" aau.setParseAction(lambda tokens: AaU(tokens[2], tokens[5], tokens[7])) eaw = e + "[" + __arctl + "W" + __arctl + "]" eaw.setParseAction(lambda tokens: EaW(tokens[2], tokens[5], tokens[7])) aaw = a + "[" + __arctl + "W" + __arctl + "]" aaw.setParseAction(lambda tokens: AaW(tokens[2], tokens[5], tokens[7])) temporal <<= (formula | eax | aax | eaf | aaf | eag | aag | eau | aau | eaw | aaw) logical = _logicals_(temporal) __arctl <<= logical return __arctl.parseString(spec, parseAll=True)
def parseArctl(spec): """Parse the spec and return its AST.""" global __arctl if __arctl is None: true = Literal("True") true.setParseAction(lambda tokens: TrueExp()) false = Literal("False") false.setParseAction(lambda tokens: FalseExp()) atom = "'" + SkipTo("'") + "'" atom.setParseAction(lambda tokens: Atom(tokens[1])) action = _logicals_(atom) __arctl = Forward() proposition = true | false | atom notproposition = "~" + proposition notproposition.setParseAction(lambda tokens: Not(tokens[1])) formula = proposition | notproposition | Suppress("(") + __arctl + Suppress(")") temporal = Forward() e = Literal("E") + "<" + action + ">" a = Literal("A") + "<" + action + ">" eax = e + "X" + temporal eax.setParseAction(lambda tokens: EaX(tokens[2], tokens[5])) aax = a + "X" + temporal aax.setParseAction(lambda tokens: AaX(tokens[2], tokens[5])) eaf = e + "F" + temporal eaf.setParseAction(lambda tokens: EaF(tokens[2], tokens[5])) aaf = a + "F" + temporal aaf.setParseAction(lambda tokens: AaF(tokens[2], tokens[5])) eag = e + "G" + temporal eag.setParseAction(lambda tokens: EaG(tokens[2], tokens[5])) aag = a + "G" + temporal aag.setParseAction(lambda tokens: AaG(tokens[2], tokens[5])) eau = e + "[" + __arctl + "U" + __arctl + "]" eau.setParseAction(lambda tokens: EaU(tokens[2], tokens[5], tokens[7])) aau = a + "[" + __arctl + "U" + __arctl + "]" aau.setParseAction(lambda tokens: AaU(tokens[2], tokens[5], tokens[7])) eaw = e + "[" + __arctl + "W" + __arctl + "]" eaw.setParseAction(lambda tokens: EaW(tokens[2], tokens[5], tokens[7])) aaw = a + "[" + __arctl + "W" + __arctl + "]" aaw.setParseAction(lambda tokens: AaW(tokens[2], tokens[5], tokens[7])) temporal <<= formula | eax | aax | eaf | aaf | eag | aag | eau | aau | eaw | aaw logical = _logicals_(temporal) __arctl <<= logical return __arctl.parseString(spec, parseAll=True)
def _create_primitives(): global binary, ident, rvalue, number, quoted_string, semi, tick_interval, time_interval, slot_id, comp, config_type, stream, comment, stream_trigger, selector if ident is not None: return semi = Literal(u';').suppress() ident = Word(alphas+u"_", alphas + nums + u"_") number = Regex(u'((0x[a-fA-F0-9]+)|[+-]?[0-9]+)').setParseAction(lambda s, l, t: [int(t[0], 0)]) binary = Regex(u'hex:([a-fA-F0-9][a-fA-F0-9])+').setParseAction(lambda s, l, t: [unhexlify(t[0][4:])]) quoted_string = QuotedString(quoteChar='"', escChar="\\", unquoteResults=True) comment = Literal('#') + restOfLine rvalue = number | quoted_string # Convert all time intervals into an integer number of seconds time_unit_multipliers = { u'second': 1, u'seconds': 1, u'minute': 60, u'minutes': 60, u'hour': 60*60, u'hours': 60*60, u'day': 60*60*24, u'days': 60*60*24, u'month': 60*60*24*30, u'months': 60*60*24*30, u'year': 60*60*24*365, u'years': 60*60*24*365, } config_type = oneOf('uint8_t uint16_t uint32_t int8_t int16_t int32_t uint8_t[] uint16_t[] uint32_t[] int8_t[] int16_t[] int32_t[] string binary') comp = oneOf('> < >= <= == ~=') # Time intervals are all based on internal system clocks so we include a 'system' tag time_unit = oneOf(u"second seconds minute minutes hour hours day days week weeks month months year years") time_interval = (number + time_unit).setParseAction(lambda s, l, t: [t[0]*time_unit_multipliers[t[1]], 'system']) tick_interval = (number + (Literal("tick_1") | Literal("tick_2"))).setParseAction(lambda s, l, t: [t[0], t[1]]) slot_id = Literal(u"controller") | (Literal(u'slot') + number) slot_id.setParseAction(lambda s,l,t: [SlotIdentifier.FromString(u' '.join([str(x) for x in t]))]) stream_modifier = Literal("system") | Literal("user") | Literal("combined") stream = Optional(Literal("system")) + oneOf("buffered unbuffered input output counter constant") + number + Optional(Literal("node")) stream.setParseAction(lambda s,l,t: [DataStream.FromString(u' '.join([str(x) for x in t]))]) all_selector = Optional(Literal("all")) + Optional(stream_modifier) + oneOf("buffered unbuffered inputs outputs counters constants") + Optional(Literal("nodes")) all_selector.setParseAction(lambda s,l,t: [DataStreamSelector.FromString(u' '.join([str(x) for x in t]))]) one_selector = Optional(Literal("system")) + oneOf("buffered unbuffered input output counter constant") + number + Optional(Literal("node")) one_selector.setParseAction(lambda s,l,t: [DataStreamSelector.FromString(u' '.join([str(x) for x in t]))]) selector = one_selector | all_selector trigger_comp = oneOf('> < >= <= ==') stream_trigger = Group((Literal(u'count') | Literal(u'value')) + Literal(u'(').suppress() - stream - Literal(u')').suppress() - trigger_comp - number).setResultsName('stream_trigger')
def _build_sort_parser(): field = _build_field_expr() plusorminus = Literal('+') | Literal('-') plusorminus.setParseAction(lambda x: SortDirection(x[0])) sort_directive = Group(Optional(plusorminus) + field).setParseAction(lambda x: SortDirective(x)) statement = Optional(sort_directive) + ZeroOrMore( Literal(',') + sort_directive) return statement
def __init__(self): self.variables = dict() #_base_var = Literal("$") + (("{" + Word(alphanums + "_-").setResultsName("var_name", listAllMatches=True) + Optional("[" + Word(nums + "*@") + "]") + "}") #| Word(alphanums)) _simple_var = Literal("$") + Word(alphanums + "_-").setResultsName("varname") _brace_substitute_part = Optional("/" + (Word(alphanums + "_-").setResultsName("orig")) + Optional("/" + Word(alphanums + "_-!?/\\").setResultsName("new"))) _array_access = "[" + Word(nums + "@*").setResultsName("position") + "]" _brace_var = Literal("${") + Word(alphanums + "_-").setResultsName("text") + _brace_substitute_part + Optional(_array_access) + "}" _brace_var.setParseAction(lambda x: x if not x.new else re.sub(x.orig, x.new, x.text)) _base_var = _simple_var | _brace_var self.var = ('"' + _base_var + '"') | _base_var self.var("variable")
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() tag_begin = Literal("<").suppress() tag_end = Literal(">").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' ival=Regex('[-]?\d+') dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?') lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)') # Helper definitions kstr= quotedString.setParseAction(removeQuotes) ^ \ dval ^ ival ^ lval ^ Word(prtable) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | tag_sect ) #| vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions ival.setParseAction(self.conv_ival) dval.setParseAction(self.conv_dval) lval.setParseAction(self.conv_lval) keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) tag_sect.setParseAction(self.add_sect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() tag_begin = Literal("<").suppress() tag_end = Literal(">").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' ival=Regex('[-]?\d+') dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?') lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)') # Helper definitions kstr= quotedString.setParseAction(removeQuotes) ^ \ dval ^ ival ^ lval ^ Word(prtable) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | tag_sect ) #| vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions ival.setParseAction(self.conv_ival) dval.setParseAction(self.conv_dval) lval.setParseAction(self.conv_lval) keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) tag_sect.setParseAction(self.add_sect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack = Literal("[") rbrack = Literal("]") lbrace = Literal("{") rbrace = Literal("}") lparen = Literal("(") rparen = Literal(")") reMacro = Suppress("\\") + oneOf(list("dwsZ")) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|") reRange = Combine(lbrack.suppress() + SkipTo(rbrack, ignore=escapedChar) + rbrack.suppress()) reLiteral = (escapedChar | oneOf(list(reLiteralChar))) reDot = Literal(".") repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) | (lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) | oneOf(list("*+?"))) reExpr = Forward() reGroup = (lparen.suppress() + Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") + reExpr.setResultsName("expr") + rparen.suppress()) reTerm = (reLiteral | reRange | reMacro | reDot | reGroup) reExpr << operatorPrecedence(reTerm, [ (repetition, 1, opAssoc.LEFT, create(Repetition)), (None, 2, opAssoc.LEFT, create(Sequence)), (Suppress('|'), 2, opAssoc.LEFT, create(Alternation)), ]) reGroup.setParseAction(create(Group)) reRange.setParseAction(create(Range)) reLiteral.setParseAction(create(Character)) reMacro.setParseAction(create(Macro)) reDot.setParseAction(create(Dot)) _parser = reExpr return _parser
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack = Literal("[") rbrack = Literal("]") lbrace = Literal("{") rbrace = Literal("}") lparen = Literal("(") rparen = Literal(")") reMacro = Suppress("\\") + oneOf(list("dwsZ")) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|") reRange = Combine(lbrack.suppress() + SkipTo(rbrack,ignore=escapedChar) + rbrack.suppress()) reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) ) reDot = Literal(".") repetition = ( ( lbrace + Word(nums).setResultsName("count") + rbrace ) | ( lbrace + Word(nums).setResultsName("minCount")+","+ Word(nums).setResultsName("maxCount") + rbrace ) | oneOf(list("*+?")) ) reExpr = Forward() reGroup = (lparen.suppress() + Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") + reExpr.setResultsName("expr") + rparen.suppress()) reTerm = ( reLiteral | reRange | reMacro | reDot | reGroup ) reExpr << operatorPrecedence( reTerm, [ (repetition, 1, opAssoc.LEFT, create(Repetition)), (None, 2, opAssoc.LEFT, create(Sequence)), (Suppress('|'), 2, opAssoc.LEFT, create(Alternation)), ] ) reGroup.setParseAction(create(Group)) reRange.setParseAction(create(Range)) reLiteral.setParseAction(create(Character)) reMacro.setParseAction(create(Macro)) reDot.setParseAction(create(Dot)) _parser = reExpr return _parser
def grammar(): parenthesis = Forward() parenthesis <<= "(" + ZeroOrMore(CharsNotIn("()") | parenthesis) + ")" field_def = OneOrMore(Word(alphanums + "._\"'`:-[]") | parenthesis) field_def.setParseAction(field_act) tablename_def = (Word(alphas + "`_") | QuotedString("\"")) function_end = Literal("END") + "$$" + Optional( Literal("LANGUAGE") + "plsql") + ";" function_statement_def = Literal("CREATE") + Optional( Literal("OR") + "REPLACE") + "FUNCTION" + SkipTo( function_end) # + function_end function_statement_def.setParseAction(function_act) field_list_def = field_def + ZeroOrMore(Suppress(",") + field_def) field_list_def.setParseAction(field_list_act) create_table_def = Literal( "CREATE") + "TABLE" + tablename_def.setResultsName( "tableName") + "(" + field_list_def.setResultsName( "fields") + ")" + ";" create_table_def.setParseAction(create_table_act) skip_fkey_def = Literal("ALTER") + "TABLE" + Regex( r"\"[a-zA-Z_]+_[0-9]{3}\"") + "ADD" + "CONSTRAINT" + QuotedString( "\"") + "FOREIGN" + "KEY" + "(" + QuotedString( "\"") + ")" + "REFERENCES" + QuotedString( "\"") + "(" + QuotedString("\"") + ")" + ZeroOrMore( CharsNotIn(";")) + ";" skip_fkey_def.setParseAction(skip_fkey_act) add_fkey_def = Literal("ALTER") + "TABLE" + tablename_def.setResultsName( "tableName") + "ADD" + "CONSTRAINT" + QuotedString( "\"") + "FOREIGN" + "KEY" + "(" + QuotedString( "\"").setResultsName( "keyName") + ")" + "REFERENCES" + QuotedString( "\"").setResultsName("fkTable") + "(" + QuotedString( "\"").setResultsName("fkCol") + ")" + ZeroOrMore( CharsNotIn(";")) + ";" add_fkey_def.setParseAction(add_fkey_act) other_statement_def = OneOrMore(CharsNotIn(";")) + ";" other_statement_def.setParseAction(other_statement_act) comment_def = "--" + ZeroOrMore(CharsNotIn("\n")) comment_def.setParseAction(other_statement_act) mcomment_def = "/*" + SkipTo(Literal("*/")) mcomment_def.setParseAction(mcomment_act) return OneOrMore(comment_def | mcomment_def | function_statement_def | create_table_def | skip_fkey_def | add_fkey_def | other_statement_def)
def __init__(self): left_bracket = Literal("{").suppress() right_bracket = Literal("}").suppress() semicolon = Literal(";").suppress() space = White().suppress() key = Word(alphanums + "+.-_/") value = ZeroOrMore( CharsNotIn('{};#"\'') | space | QuotedString("'", escChar='\\', multiline=True) | QuotedString('"', escChar='\\', multiline=True)) # modifier for location uri [ = | ~ | ~* | ^~ ] modifier = Literal("=") | Literal("~*") | Literal("~") | Literal("^~") comment = Literal('#').suppress() + Optional(restOfLine) # rules assignment = Group( (key | value) + value + semicolon + Optional(space + comment)) block = Forward() block << Group( Group(key + Optional(space + modifier) + Optional(space) + Optional(value) + Optional(space + value)) + left_bracket + Group(ZeroOrMore(assignment | block | comment.suppress())) + right_bracket) def comment_handler(t): result = [] if "promo" in t[0]: result.append("promo") if "author: " in t[0]: try: email = t[0].split("author: ")[1].strip() result.append(email) except Exception: result.append(t[0]) return result comment.setParseAction(comment_handler) self.script = OneOrMore(assignment | block | comment.suppress())
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() arg_begin = Literal("(").suppress() arg_end = Literal(")").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' # Helper definitions kstr=Word(prtable) ^ quotedString.setParseAction(removeQuotes) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin key_sect=name+Group(arg_begin+kstr+arg_end)+sect_begin vec_sect=name+Group(arg_begin+vec+ arg_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | key_sect | vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) key_sect.setParseAction(self.add_sect) vec_sect.setParseAction(self.add_vecsect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def _getPattern(self): arith_expr = Forward() comp_expr = Forward() logic_expr = Forward() LPAR, RPAR, SEMI = map(Suppress, "();") identifier = Word(alphas+"_", alphanums+"_") multop = oneOf('* /') plusop = oneOf('+ -') expop = Literal( "^" ) compop = oneOf('> < >= <= != ==') andop = Literal("AND") orop = Literal("OR") current_value = Literal( "." ) assign = Literal( "=" ) # notop = Literal('NOT') function = oneOf(' '.join(self.FUNCTIONS)) function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR) aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}') single_column = QuotedString(quoteChar='[', endQuoteChar=']') integer = Regex(r"-?\d+") real = Regex(r"-?\d+\.\d*") # quotedString enables strings without quotes to pass operand = \ function_call.setParseAction(self.__evalFunction) | \ aggregate_column.setParseAction(self.__evalAggregateColumn) | \ single_column.setParseAction(self.__evalSingleColumn) | \ ((real | integer).setParseAction(self.__evalConstant)) | \ quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \ current_value.setParseAction(self.__evalCurrentValue) | \ identifier.setParseAction(self.__evalString) arith_expr << operatorPrecedence(operand, [ (expop, 2, opAssoc.LEFT, self.__expOp), (multop, 2, opAssoc.LEFT, self.__multOp), (plusop, 2, opAssoc.LEFT, self.__addOp), ]) # comp_expr = Group(arith_expr + compop + arith_expr) comp_expr << operatorPrecedence(arith_expr, [ (compop, 2, opAssoc.LEFT, self.__evalComparisonOp), ]) logic_expr << operatorPrecedence(comp_expr, [ (andop, 2, opAssoc.LEFT, self.__evalLogicOp), (orop, 2, opAssoc.LEFT, self.__evalLogicOp) ]) pattern = logic_expr + StringEnd() return pattern
def define_number(self): """ Return the syntax definition for a number in Arabic Numerals. Override this method to support numeral systems other than Arabic Numerals (0-9). Do not override this method just to change the character used to separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR` and :attr:`T_DECIMAL_SEPARATOR`, respectively. """ # Defining the basic tokens: to_dot = lambda t: "." to_plus = lambda t: "+" to_minus = lambda t: "-" positive_sign = Literal(self._grammar.get_token("positive_sign")) positive_sign.setParseAction(to_plus) negative_sign = Literal(self._grammar.get_token("negative_sign")) negative_sign.setParseAction(to_minus) decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress( self._grammar.get_token("thousands_separator")) digits = Word(nums) # Building the integers and decimals: sign = positive_sign | negative_sign thousands = Word(nums, max=3) + \ OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits number = Combine(Optional(sign) + integers + Optional(decimals)) number.setParseAction(self.make_number) number.setName("number") return number
def define_number(self): """ Return the syntax definition for a number in Arabic Numerals. Override this method to support numeral systems other than Arabic Numerals (0-9). Do not override this method just to change the character used to separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR` and :attr:`T_DECIMAL_SEPARATOR`, respectively. """ # Defining the basic tokens: to_dot = lambda t: "." to_plus = lambda t: "+" to_minus = lambda t: "-" positive_sign = Literal(self._grammar.get_token("positive_sign")) positive_sign.setParseAction(to_plus) negative_sign = Literal(self._grammar.get_token("negative_sign")) negative_sign.setParseAction(to_minus) decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress(self._grammar.get_token("thousands_separator")) digits = Word(nums) # Building the integers and decimals: sign = positive_sign | negative_sign thousands = Word(nums, max=3) + \ OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits number = Combine(Optional(sign) + integers + Optional(decimals)) number.setParseAction(self.make_number) number.setName("number") return number
def _getPattern(self): arith_expr = Forward() comp_expr = Forward() logic_expr = Forward() LPAR, RPAR, SEMI = map(Suppress, "();") identifier = Word(alphas + "_", alphanums + "_") multop = oneOf('* /') plusop = oneOf('+ -') expop = Literal("^") compop = oneOf('> < >= <= != ==') andop = Literal("AND") orop = Literal("OR") current_value = Literal(".") assign = Literal("=") # notop = Literal('NOT') function = oneOf(' '.join(self.FUNCTIONS)) function_call = Group( function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR) aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}') single_column = QuotedString(quoteChar='[', endQuoteChar=']') integer = Regex(r"-?\d+") real = Regex(r"-?\d+\.\d*") # quotedString enables strings without quotes to pass operand = \ function_call.setParseAction(self.__evalFunction) | \ aggregate_column.setParseAction(self.__evalAggregateColumn) | \ single_column.setParseAction(self.__evalSingleColumn) | \ ((real | integer).setParseAction(self.__evalConstant)) | \ quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \ current_value.setParseAction(self.__evalCurrentValue) | \ identifier.setParseAction(self.__evalString) arith_expr << operatorPrecedence(operand, [ (expop, 2, opAssoc.LEFT, self.__expOp), (multop, 2, opAssoc.LEFT, self.__multOp), (plusop, 2, opAssoc.LEFT, self.__addOp), ]) # comp_expr = Group(arith_expr + compop + arith_expr) comp_expr << operatorPrecedence(arith_expr, [ (compop, 2, opAssoc.LEFT, self.__evalComparisonOp), ]) logic_expr << operatorPrecedence( comp_expr, [(andop, 2, opAssoc.LEFT, self.__evalLogicOp), (orop, 2, opAssoc.LEFT, self.__evalLogicOp)]) pattern = logic_expr + StringEnd() return pattern
def _get_bus_array_construct(self): """ Returns a construct for an array of bus data. """ bus_no = integer.setResultsName("bus_no") v_base = real.setResultsName("v_base") # kV v_magnitude = Optional(real).setResultsName("v_magnitude") v_angle = Optional(real).setResultsName("v_angle") # radians area = Optional(integer).setResultsName("area") # not used yet region = Optional(integer).setResultsName("region") # not used yet bus_data = bus_no + v_base + v_magnitude + v_angle + \ area + region + scolon bus_data.setParseAction(self.push_bus) bus_array = Literal("Bus.con") + "=" + "[" + "..." + \ ZeroOrMore(bus_data + Optional("]" + scolon)) # Sort buses according to their name (bus_no) bus_array.setParseAction(self.sort_buses) return bus_array
def _get_bus_array_construct(self): """ Returns a construct for an array of bus data. """ bus_no = integer.setResultsName("bus_no") v_base = real.setResultsName("v_base") # kV v_magnitude = Optional(real).setResultsName("v_magnitude") v_angle = Optional(real).setResultsName("v_angle") # radians area = Optional(integer).setResultsName("area") # not used yet region = Optional(integer).setResultsName("region") # not used yet bus_data = bus_no + v_base + v_magnitude + v_angle + \ area + region + scolon bus_data.setParseAction(self.push_bus) bus_array = Literal("Bus.con") + "=" + "[" + "..." + \ ZeroOrMore(bus_data + Optional("]" + scolon)) # Sort buses according to their name (bus_no) bus_array.setParseAction(self.sort_buses) return bus_array
def __init__(self, max_=60): # define the grammar structure digits = "0123456789" star = Literal('*') number = Word(digits) | Word(alphas) steps = number range_ = number + Optional(Literal('-') + number) numspec = star | range_ expr = Group(numspec) + Optional(Literal('/') + steps) extra_groups = ZeroOrMore(Literal(',') + expr) groups = expr + extra_groups + StringEnd() # define parse actions star.setParseAction(self._expand_star) number.setParseAction(self._expand_number) range_.setParseAction(self._expand_range) expr.setParseAction(self._filter_steps) extra_groups.setParseAction(self._ignore_comma) groups.setParseAction(self._join_to_set) self.max_ = max_ self.parser = groups
def get_raw_stack(self): lpar = Literal("(") rpar = Literal(")") diff = Literal("-") intersection = Literal("n") union = Literal("U") operand = Word(alphanums) operator = diff | intersection | union sexp = Forward() term = operand.setParseAction( self.push_first_raw_stack) | lpar.setParseAction( self.push_first_raw_stack) + sexp + rpar.setParseAction( self.push_first_raw_stack) sexp << term + ZeroOrMore( (operator + term).setParseAction(self.push_first_raw_stack)) sexp.parseString(self.expression) return self.raw_stack
def __init__(self, max_=60): # define the grammar structure digits = "0123456789" star = Literal('*') number = Word(digits) | Word(alphas) steps = number range_ = number + Optional(Literal('-') + number) numspec = star | range_ expr = Group(numspec) + Optional(Literal('/') + steps) extra_groups = ZeroOrMore(Literal(',') + expr) groups = expr + extra_groups + StringEnd() # define parse actions star.setParseAction(self._expand_star) number.setParseAction(self._expand_number) range_.setParseAction(self._expand_range) expr.setParseAction(self._filter_steps) extra_groups.setParseAction(self._ignore_comma) groups.setParseAction(self._join_to_set) self.max_ = max_ self.parser = groups
def define_math(self): digits = Word(nums) variable = Word(alphas + self._grammar.get_token("namespace_separator")) to_dot = lambda t: "." decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress( self._grammar.get_token("thousands_separator")) thousands = Word( nums, max=3) + OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits expop = Literal('^') signop = oneOf('+ -') multop = oneOf('* /') plusop = oneOf('+ -') factop = Literal('!') modop = Literal('%') operand = Combine((integers + Optional(decimals)) | variable) expr = operatorPrecedence(operand, [ (Literal("["), 1, opAssoc.RIGHT), (Literal("]"), 1, opAssoc.LEFT), ("!", 1, opAssoc.LEFT), (expop, 2, opAssoc.RIGHT), (signop, 1, opAssoc.RIGHT), (multop, 2, opAssoc.LEFT), (modop, 2, opAssoc.LEFT), (plusop, 2, opAssoc.LEFT), ]) expr = Combine(expr) expr.setParseAction(self.make_arithmetic) expr.setName("arithmetic") return expr
def make_arithmetic(self, s, l, tokens): digits = Word(nums) variable = Word(alphas + self._grammar.get_token("namespace_separator")) to_dot = lambda t: "." decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress( self._grammar.get_token("thousands_separator")) thousands = Word( nums, max=3) + OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits expop = Literal('^') signop = oneOf('+ -') multop = oneOf('* /') plusop = oneOf('+ -') modop = Literal('%') factop = Literal("!") operand = Combine((integers + Optional(decimals)) | variable) expr = operatorPrecedence(operand, [ (Literal("[").setParseAction(replaceWith("(")), 1, opAssoc.RIGHT), (Literal("]").setParseAction(replaceWith(")")), 1, opAssoc.LEFT), ("!", 1, opAssoc.LEFT), (expop, 2, opAssoc.RIGHT), (signop, 1, opAssoc.RIGHT), (multop, 2, opAssoc.LEFT), (modop, 2, opAssoc.LEFT), (plusop, 2, opAssoc.LEFT), ]) a = expr.parseString(tokens[0], parseAll=True) return Arithmetic(a[0], self._namespace, self._grammar.get_token("namespace_separator"))
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack,rbrack,lbrace,rbrace,lparen,rparen,colon,qmark = map(Literal,"[]{}():?") reMacro = Combine("\\" + oneOf(list("dws"))) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t" reRange = Combine(lbrack + SkipTo(rbrack,ignore=escapedChar) + rbrack) reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) ) reNonCaptureGroup = Suppress("?:") reDot = Literal(".") repetition = ( ( lbrace + Word(nums)("count") + rbrace ) | ( lbrace + Word(nums)("minCount")+","+ Word(nums)("maxCount") + rbrace ) | oneOf(list("*+?")) ) reRange.setParseAction(handleRange) reLiteral.setParseAction(handleLiteral) reMacro.setParseAction(handleMacro) reDot.setParseAction(handleDot) reTerm = ( reLiteral | reRange | reMacro | reDot | reNonCaptureGroup) reExpr = infixNotation( reTerm, [ (repetition, 1, opAssoc.LEFT, handleRepetition), (None, 2, opAssoc.LEFT, handleSequence), (Suppress('|'), 2, opAssoc.LEFT, handleAlternative), ] ) _parser = reExpr return _parser
def getToken(self): """ Создать элементы из прикрепленных файлов. Отдельно картинки, отдельно все файлы """ attachesAll = [] attaches = Attachment(self.parser.page).attachmentFull attaches.sort(key=len, reverse=True) for attach in attaches: fname = os.path.basename(attach) if self.filterFile(fname): attach = Literal(fname) attachesAll.append(attach) finalToken = Literal(self.attachString) + concatenate(attachesAll) finalToken = finalToken.setParseAction(self.convertToLink)("attach") return finalToken
def rdata_grammar(): pairs = Forward() comma_separated_values = constant + ZeroOrMore(Literal(",").suppress() + constant) vector = Literal("c(") + comma_separated_values + Literal(")").setName(') to close c') vector.setParseAction(lambda s,l,t: [np.array(t[1:-1])]) array = Literal("structure(") + pairs + Literal(")").setName(') to close structure') array.setParseAction(lambda s,l,t: [_build_array(t[1])]) value = constant | vector | array pair = variable + Literal("=").suppress() + value pair.setParseAction(lambda s,l,t: [t[:]]) pairs << (pair + ZeroOrMore(Literal(",").suppress() + pair)) pairs.setParseAction(lambda s,l,t: [dict(t[:])]) data = Literal("list(") + pairs + Literal(")").setName(') to close list') data.setParseAction(lambda s,l,t: [t[1]]) return data
def parseCTLK(spec): """Parse the spec and return the list of possible ASTs.""" global __ctlk if __ctlk is None: true = Literal("True") true.setParseAction(lambda tokens: TrueExp()) false = Literal("False") false.setParseAction(lambda tokens: FalseExp()) init = Literal("Init") init.setParseAction(lambda tokens: Init()) reachable = Literal("Reachable") reachable.setParseAction(lambda tokens: Reachable()) atom = "'" + SkipTo("'") + "'" atom.setParseAction(lambda tokens: Atom(tokens[1])) agent = atom group = Group(ZeroOrMore(agent + Suppress(",")) + agent) proposition = true | false | init | reachable | atom __ctlk = Forward() notproposition = "~" + proposition notproposition.setParseAction(lambda tokens: Not(tokens[1])) formula = (proposition | notproposition | Suppress("(") + __ctlk + Suppress(")")) logical = Forward() ex = Literal("E") + "X" + logical ex.setParseAction(lambda tokens: EX(tokens[2])) ax = Literal("A") + "X" + logical ax.setParseAction(lambda tokens: AX(tokens[2])) ef = Literal("E") + "F" + logical ef.setParseAction(lambda tokens: EF(tokens[2])) af = Literal("A") + "F" + logical af.setParseAction(lambda tokens: AF(tokens[2])) eg = Literal("E") + "G" + logical eg.setParseAction(lambda tokens: EG(tokens[2])) ag = Literal("A") + "G" + logical ag.setParseAction(lambda tokens: AG(tokens[2])) eu = Literal("E") + "[" + __ctlk + "U" + __ctlk + "]" eu.setParseAction(lambda tokens: EU(tokens[2], tokens[4])) au = Literal("A") + "[" + __ctlk + "U" + __ctlk + "]" au.setParseAction(lambda tokens: AU(tokens[2], tokens[4])) ew = Literal("E") + "[" + __ctlk + "W" + __ctlk + "]" ew.setParseAction(lambda tokens: EW(tokens[2], tokens[4])) aw = Literal("A") + "[" + __ctlk + "W" + __ctlk + "]" aw.setParseAction(lambda tokens: AW(tokens[2], tokens[4])) temporal = (ex | ax | ef | af | eg | ag | eu | au | ew | aw) nk = Literal("nK") + "<" + agent + ">" + logical nk.setParseAction(lambda tokens: nK(tokens[2], tokens[4])) k = Literal("K") + "<" + agent + ">" + logical k.setParseAction(lambda tokens: K(tokens[2], tokens[4])) ne = Literal("nE") + "<" + group + ">" + logical ne.setParseAction(lambda tokens: nE(list(tokens[2]), tokens[4])) e = Literal("E") + "<" + group + ">" + logical e.setParseAction(lambda tokens: E(list(tokens[2]), tokens[4])) nd = Literal("nD") + "<" + group + ">" + logical nd.setParseAction(lambda tokens: nD(list(tokens[2]), tokens[4])) d = Literal("D") + "<" + group + ">" + logical d.setParseAction(lambda tokens: D(list(tokens[2]), tokens[4])) nc = Literal("nC") + "<" + group + ">" + logical nc.setParseAction(lambda tokens: nC(list(tokens[2]), tokens[4])) c = Literal("C") + "<" + group + ">" + logical c.setParseAction(lambda tokens: C(list(tokens[2]), tokens[4])) epistemic = (nk | k | ne | e | nd | d | nc | c) logical <<= (formula | epistemic | temporal) __ctlk <<= (_logicals_(logical)) return __ctlk.parseString(spec, parseAll = True)
def parseATL(spec): """Parse the spec and return the list of possible ASTs.""" global __atl if __atl is None: true = Literal("True") true.setParseAction(lambda tokens: TrueExp()) false = Literal("False") false.setParseAction(lambda tokens: FalseExp()) atom = "'" + SkipTo("'") + "'" atom.setParseAction(lambda tokens: Atom(tokens[1])) agent = atom group = Group(ZeroOrMore(agent + Suppress(",")) + agent) proposition = true | false | atom __atl = Forward() notproposition = "~" + proposition notproposition.setParseAction(lambda tokens: Not(tokens[1])) formula = (proposition | notproposition | Suppress("(") + __atl + Suppress(")")) logical = Forward() cax = Literal("[") + group + "]" + "X" + logical cax.setParseAction(lambda tokens: CAX(tokens[1], tokens[4])) cex = Literal("<") + group + ">" + "X" + logical cex.setParseAction(lambda tokens: CEX(tokens[1], tokens[4])) caf = Literal("[") + group + "]" + "F" + logical caf.setParseAction(lambda tokens: CAF(tokens[1], tokens[4])) cef = Literal("<") + group + ">" + "F" + logical cef.setParseAction(lambda tokens: CEF(tokens[1], tokens[4])) cag = Literal("[") + group + "]" + "G" + logical cag.setParseAction(lambda tokens: CAG(tokens[1], tokens[4])) ceg = Literal("<") + group + ">" + "G" + logical ceg.setParseAction(lambda tokens: CEG(tokens[1], tokens[4])) cau = Literal("[") + group + "]" + "[" + __atl + "U" + __atl + "]" cau.setParseAction(lambda tokens: CAU(tokens[1], tokens[4], tokens[6])) ceu = Literal("<") + group + ">" + "[" + __atl + "U" + __atl + "]" ceu.setParseAction(lambda tokens: CEU(tokens[1], tokens[4], tokens[6])) caw = Literal("[") + group + "]" + "[" + __atl + "W" + __atl + "]" caw.setParseAction(lambda tokens: CAW(tokens[1], tokens[4], tokens[6])) cew = Literal("<") + group + ">" + "[" + __atl + "W" + __atl + "]" cew.setParseAction(lambda tokens: CEW(tokens[1], tokens[4], tokens[6])) strategic = (cax | cex | caf | cef | cag | ceg | cau | ceu | caw | cew) logical <<= (formula | strategic) __atl <<= (_logicals_(logical)) return __atl.parseString(spec, parseAll=True)
def __init__(self): self.ae = False self.local_dict = None self.f = None self.user_functions = None self.expr_stack = [] self.texpr_stack = [] # Define constants self.constants = {} # Define Operators self.opn = {"+": operator.add, "-": operator.sub, "*": operator.mul, "/": operator.truediv, ">": operator.gt, ">=": operator.ge, "<": operator.lt, "<=": operator.le, "==": operator.eq, "!=": operator.ne, "|": operator.or_, "&": operator.and_, "!": operator.inv} # Define xarray DataArray operators with 1 input parameter self.xfn1 = {"angle": xr.ufuncs.angle, "arccos": xr.ufuncs.arccos, "arccosh": xr.ufuncs.arccosh, "arcsin": xr.ufuncs.arcsin, "arcsinh": xr.ufuncs.arcsinh, "arctan": xr.ufuncs.arctan, "arctanh": xr.ufuncs.arctanh, "ceil": xr.ufuncs.ceil, "conj": xr.ufuncs.conj, "cos": xr.ufuncs.cos, "cosh": xr.ufuncs.cosh, "deg2rad": xr.ufuncs.deg2rad, "degrees": xr.ufuncs.degrees, "exp": xr.ufuncs.exp, "expm1": xr.ufuncs.expm1, "fabs": xr.ufuncs.fabs, "fix": xr.ufuncs.fix, "floor": xr.ufuncs.floor, "frexp": xr.ufuncs.frexp, "imag": xr.ufuncs.imag, "iscomplex": xr.ufuncs.iscomplex, "isfinite": xr.ufuncs.isfinite, "isinf": xr.ufuncs.isinf, "isnan": xr.ufuncs.isnan, "isreal": xr.ufuncs.isreal, "log": xr.ufuncs.log, "log10": xr.ufuncs.log10, "log1p": xr.ufuncs.log1p, "log2": xr.ufuncs.log2, "rad2deg": xr.ufuncs.rad2deg, "radians": xr.ufuncs.radians, "real": xr.ufuncs.real, "rint": xr.ufuncs.rint, "sign": xr.ufuncs.sign, "signbit": xr.ufuncs.signbit, "sin": xr.ufuncs.sin, "sinh": xr.ufuncs.sinh, "sqrt": xr.ufuncs.sqrt, "square": xr.ufuncs.square, "tan": xr.ufuncs.tan, "tanh": xr.ufuncs.tanh, "trunc": xr.ufuncs.trunc} # Define xarray DataArray operators with 2 input parameter self.xfn2 = {"arctan2": xr.ufuncs.arctan2, "copysign": xr.ufuncs.copysign, "fmax": xr.ufuncs.fmax, "fmin": xr.ufuncs.fmin, "fmod": xr.ufuncs.fmod, "hypot": xr.ufuncs.hypot, "ldexp": xr.ufuncs.ldexp, "logaddexp": xr.ufuncs.logaddexp, "logaddexp2": xr.ufuncs.logaddexp2, "logicaland": xr.ufuncs.logical_and, "logicalnot": xr.ufuncs.logical_not, "logicalor": xr.ufuncs.logical_or, "logicalxor": xr.ufuncs.logical_xor, "maximum": xr.ufuncs.maximum, "minimum": xr.ufuncs.minimum, "nextafter": xr.ufuncs.nextafter} # Define non-xarray DataArray operators with 2 input parameter self.fn2 = {"percentile": np.percentile} # Define xarray DataArray reduction operators self.xrfn = {"all": xr.DataArray.all, "any": xr.DataArray.any, "argmax": xr.DataArray.argmax, "argmin": xr.DataArray.argmin, "max": xr.DataArray.max, "mean": xr.DataArray.mean, "median": xr.DataArray.median, "min": xr.DataArray.min, "prod": xr.DataArray.prod, "sum": xr.DataArray.sum, "std": xr.DataArray.std, "var": xr.DataArray.var} # Define non-xarray DataArray operators with 2 input parameter self.xcond = {"<": np.percentile} # Define Grammar point = Literal(".") e = CaselessLiteral("E") fnumber = Combine(Word("+-"+nums, nums) + Optional(point + Optional(Word(nums))) + Optional(e + Word("+-"+nums, nums))) variable = Word(alphas, alphas+nums+"_$") seq = Literal("=") b_not = Literal("~") plus = Literal("+") minus = Literal("-") mult = Literal("*") div = Literal("/") gt = Literal(">") gte = Literal(">=") lt = Literal("<") lte = Literal("<=") eq = Literal("==") neq = Literal("!=") b_or = Literal("|") b_and = Literal("&") l_not = Literal("!") lpar = Literal("(").suppress() rpar = Literal(")").suppress() comma = Literal(",") colon = Literal(":") lbrac = Literal("[") rbrac = Literal("]") lcurl = Literal("{") rcurl = Literal("}") qmark = Literal("?") scolon = Literal(";") addop = plus | minus multop = mult | div sliceop = colon compop = gte | lte | gt | lt eqop = eq | neq bitcompop = b_or | b_and bitnotop = b_not logicalnotop = l_not assignop = seq expop = Literal("^") expr = Forward() indexexpr = Forward() atom = (Optional("-") + (variable + seq + expr).setParseAction(self.push_assign) | indexexpr.setParseAction(self.push_index) | (lpar + expr + qmark.setParseAction(self.push_ternary1) + expr + scolon.setParseAction(self.push_ternary2) + expr + rpar).setParseAction(self.push_ternary) | (lpar + expr + qmark + expr + scolon + expr + rpar).setParseAction(self.push_ternary) | (logicalnotop + expr).setParseAction(self.push_ulnot) | (bitnotop + expr).setParseAction(self.push_unot) | (minus + expr).setParseAction(self.push_uminus) | (variable + lcurl + expr + rcurl).setParseAction(self.push_mask) | (variable + lpar + expr + (comma + expr)*3 + rpar).setParseAction(self.push_expr4) | (variable + lpar + expr + (comma + expr)*2 + rpar).setParseAction(self.push_expr3) | (variable + lpar + expr + comma + expr + rpar).setParseAction(self.push_expr2) | (variable + lpar + expr + rpar | variable).setParseAction(self.push_expr1) | fnumber.setParseAction(self.push_expr) | (lpar + expr + ZeroOrMore(comma + expr).setParseAction(self.get_tuple) + rpar).setParseAction(self.push_tuple) | (lpar + expr.suppress() + rpar).setParseAction(self.push_uminus)) # Define order of operations for operators factor = Forward() factor << atom + ZeroOrMore((expop + factor).setParseAction(self.push_op)) term = factor + ZeroOrMore((multop + factor).setParseAction(self.push_op)) term2 = term + ZeroOrMore((addop + term).setParseAction(self.push_op)) term3 = term2 + ZeroOrMore((sliceop + term2).setParseAction(self.push_op)) term4 = term3 + ZeroOrMore((compop + term3).setParseAction(self.push_op)) term5 = term4 + ZeroOrMore((eqop + term4).setParseAction(self.push_op)) term6 = term5 + ZeroOrMore((bitcompop + term5).setParseAction(self.push_op)) expr << term6 + ZeroOrMore((assignop + term6).setParseAction(self.push_op)) # Define index operators colon_expr = (colon + FollowedBy(comma) ^ colon + FollowedBy(rbrac)).setParseAction(self.push_colon) range_expr = colon_expr | expr | colon indexexpr << (variable + lbrac + delimitedList(range_expr, delim=',') + rbrac).setParseAction(self.push_expr) self.parser = expr
hexdigits = Word(string.hexdigits, exact=2) hexdigits.setName('hexdigits') escaped = Suppress(Literal('\\')) + hexdigits escaped.setName('escaped') def _p_escaped(s, l, t): text = t[0] return chr(int(text, 16)) escaped.setParseAction(_p_escaped) value = Combine(OneOrMore(CharsNotIn('*()\\\0') | escaped)) value.setName('value') equal = Literal("=") equal.setParseAction(lambda s, l, t: pureldap.LDAPFilter_equalityMatch) approx = Literal("~=") approx.setParseAction(lambda s, l, t: pureldap.LDAPFilter_approxMatch) greater = Literal(">=") greater.setParseAction(lambda s, l, t: pureldap.LDAPFilter_greaterOrEqual) less = Literal("<=") less.setParseAction(lambda s, l, t: pureldap.LDAPFilter_lessOrEqual) filtertype = equal | approx | greater | less filtertype.setName('filtertype') simple = attr + filtertype + value simple.leaveWhitespace() simple.setName('simple') def _p_simple(s, l, t): attr, filtertype, value = t
return ("<"+tok[0]+"> " + " ".join(tok)).replace("\"","\\\"") field_def.setParseAction(field_act) field_list_def = delimitedList( field_def ) def field_list_act(toks): return " | ".join(toks) field_list_def.setParseAction(field_list_act) create_table_def = Literal("CREATE") + "TABLE" + Word(alphas,alphanums+"_").setResultsName("tablename") + \ "("+field_list_def.setResultsName("columns")+")"+ ";" def create_table_act(toks): return """"%(tablename)s" [\n\t label="<%(tablename)s> %(tablename)s | %(columns)s"\n\t shape="record"\n];""" % toks create_table_def.setParseAction(create_table_act) add_fkey_def=Literal("ALTER")+"TABLE"+"ONLY" + Word(alphanums+"_").setResultsName("fromtable") + "ADD" \ + "CONSTRAINT" + Word(alphanums+"_") + "FOREIGN"+"KEY"+"("+Word(alphanums+"_").setResultsName("fromcolumn")+")" \ +"REFERENCES"+Word(alphanums+"_").setResultsName("totable")+"("+Word(alphanums+"_").setResultsName("tocolumn")+")"+";" def add_fkey_act(toks): return """ "%(fromtable)s":%(fromcolumn)s -> "%(totable)s":%(tocolumn)s """ % toks add_fkey_def.setParseAction(add_fkey_act) other_statement_def = ( OneOrMore(CharsNotIn(";") ) + ";") other_statement_def.setParseAction( replaceWith("") ) comment_def = "--" + ZeroOrMore(CharsNotIn("\n")) comment_def.setParseAction( replaceWith("") ) statement_def = comment_def | create_table_def | add_fkey_def | other_statement_def
def _parse_line(self): """ Parses a single line, and returns a node representing the active context Further lines processed are expected to be children of the active context, or children of its accestors. ------------------------------------------------ Basic grammar is as follows: line = <mako>|<nemo>|<string> <mako> We don't parse normally parse tags, so the following info is sketchy. Mako tags are recognized as anythign that starts with: - <% - %> - %CLOSETEXT - </% Mako Control tags however are parsed, and required to adhere to the same indentation rules as Nemo tags. mako_control = <start>|<middle>|<end> start = (for|if|while) <inner>: middle = (else|elif): end = endfor|endwhile nemo = % ( <mako_control>|<nemo_statement> ) nemo_statement = .<quote><string><quote>|#<quote><string><quote>|<words> <quote> = '|" Notes: Quotes are required to be balanced. Quotes preceded by a \ are ignored. <string> = * words = \w+ """ #if self.debug: print '\t ' + str(self._current_node) # PyParser setParseAction's actually execute during parsing, # So we need closures in order to change the current scope def depth_from_indentation(function): """ Set the depth as the start of the match """ def wrap(start, values): #print 'Depth %d | %d %s' %(self._depth, start, values) #self._depth = start self._current_node = function(values) #print self._current_node return '' return wrap def depth_from_match(function): """ Set the depth as the start of the match """ def wrap(start, values): #print 'Depth %d | %d %s' %(self._depth, start, values) #print self._current_node self._depth = start self._current_node = function(values) #print self._current_node return '' return wrap def depth_from_nemo_tag(function): """ Start of the match is where the nemo tag is. Pass the other values to the wrapped function """ def wrap(start, values): # print 'Depth %d | %d %s' %(self._depth, start, values) self._depth = start tokens = values[1] self._current_node = function(tokens) #print self._current_node return '' return wrap # Match HTML from pyparsing import NotAny, MatchFirst html = restOfLine html.setParseAction(depth_from_indentation(self._add_html_node)) # Match Mako control tags nemo_tag = Literal('%') begin = Keyword('for') | Keyword('if') | Keyword('while') middle = Keyword('else') | Keyword('elif') end = Keyword('endfor') | Keyword('endif') | Keyword('endwhile') control = nemo_tag + (begin | middle | end) begin.setParseAction(depth_from_indentation(self._add_nesting_mako_control_node) ) middle.setParseAction(depth_from_indentation(self._add_mako_middle_node)) end.setParseAction(depth_from_indentation(self._add_mako_control_leaf)) # Match Nemo tags argument_name = Word(alphas,alphanums+"_-:") argument_value = quotedString regular_argument = argument_name + Literal('=') + argument_value class_name = Literal('.').setParseAction(lambda x: 'class=') id_name = Literal('#').setParseAction(lambda x: 'id=') special_argument = (class_name | id_name) + argument_value argument = Combine(special_argument) | Combine(regular_argument) # Match single Nemo statement (Part of a multi-line) inline_nemo_html = Word(alphas) + Group(ZeroOrMore(argument)) inline_nemo_html.setParseAction(depth_from_match(self._add_nemo_node)) # Match first nemo tag on the line (the one that may begin a multi-statement expression) nemo_html = nemo_tag + Group(Word(alphanums+"_-:") + Group(ZeroOrMore(argument))) nemo_html.setParseAction(depth_from_nemo_tag(self._add_nemo_node)) # Match a multi-statement expression. Nemo statements are seperated by |. Anything after || is treated as html separator = Literal('|').suppress() html_separator = Literal('||') # | Literal('|>') nemo_list = nemo_html + ZeroOrMore( separator + inline_nemo_html ) inline_html = html.copy() inline_html.setParseAction(depth_from_match(self._add_inline_html_node)) nemo_multi = nemo_list + Optional(html_separator + inline_html) # Match empty Nemo statement empty = nemo_tag + Empty() empty.setParseAction(depth_from_indentation(self._add_blank_nemo_node)) # Match unused Mako tags mako_tags = Literal('<%') | Literal('%>') | Literal('%CLOSETEXT') | Literal('</%') mako = mako_tags mako_tags.setParseAction(depth_from_indentation(self._add_html_node)) # Matches General nemo = (control | nemo_multi | empty) line = mako_tags | nemo | html # Depth Calculation (deprecated?) self._depth = len(self._c) - len(self._c.strip()) #try: line.parseString(self._c)
selectcase_kwd.setParseAction(lambda s,loc,toks: [' '.join(toks)]) selectcase_stmt = selectcase_kwd + if_expr + EOLL selectcase_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'end select']) where_stmt = Keyword('where') + if_expr + EOLL where_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'end where']) arglist = '(' + (delimitedList(NAME, delim=',') | empty) + ')' arglist.setParseAction(lambda s,loc,toks: '('+', '.join(toks[1:-1]) + ')') arglist.parseString('(foo,bar)') name_eq = Literal('name') + '=' + quotedString bind_keyword = Keyword('bind').setParseAction(lambda s,loc,toks: [' bind']) bind_attr = bind_keyword + Literal('(') + 'c' + Optional(comma + name_eq) + ')' result_attr = Literal('result') + '(' + NAME + ')' result_attr.setParseAction(lambda s,loc,toks: [' '+''.join(toks)]) func_post = eachMostOnce(bind_attr, result_attr) elem_attr = Keyword('elemental').setName('elemental') elem_attr.setParseAction(lambda s,loc,toks: [toks[0] + ' ']) pure_attr = Keyword('pure').setName('pure') pure_attr.setParseAction(lambda s,loc,toks: [toks[0] + ' ']) recu_attr = Keyword('recursive').setName('elemental') recu_attr.setParseAction(lambda s,loc,toks: [toks[0] + ' ']) func_pre = eachMostOnce(elem_attr, pure_attr, recu_attr) func_name = NAME.copy().setName('func_name') func_name.setParseAction(lambda s,loc,toks: [' '+toks[0]]) func_def = Optional(func_pre) + Keyword('function') + func_name + arglist \
STRING_LITERAL1 = Regex(ur"'(?:[^'\n\r\\]|\\['ntbrf\\])*'(?!')", flags=re.U) STRING_LITERAL1.setParseAction( lambda x: rdflib.Literal(x[0][1:-1].decode("string-escape") if not rdflib.py3compat.PY3 else x[0][1:-1]) ) # [157] STRING_LITERAL2 ::= '"' ( ([^#x22#x5C#xA#xD]) | ECHAR )* '"' # STRING_LITERAL2 = Literal('"') + ZeroOrMore ( Regex(u'[^\u0022\u005C\u000A\u000D]',flags=re.U) | ECHAR ) + '"' STRING_LITERAL2 = Regex(ur'"(?:[^"\n\r\\]|\\["ntbrf\\])*"(?!")', flags=re.U) STRING_LITERAL2.setParseAction( lambda x: rdflib.Literal(x[0][1:-1].decode("string-escape") if not rdflib.py3compat.PY3 else x[0][1:-1]) ) # [161] NIL ::= '(' WS* ')' NIL = Literal("(") + ")" NIL.setParseAction(lambda x: rdflib.RDF.nil) # [162] WS ::= #x20 | #x9 | #xD | #xA # Not needed? # WS = #x20 | #x9 | #xD | #xA # [163] ANON ::= '[' WS* ']' ANON = Literal("[") + "]" ANON.setParseAction(lambda x: rdflib.BNode()) # A = CaseSensitiveKeyword('a') A = Literal("a") A.setParseAction(lambda x: rdflib.RDF.type) # ------ NON-TERMINALS --------------
def _makeimagemap(tokens): image = None for x in tokens: if isinstance(x, basestring): image = x break return ImageMap(entries=list(tokens), image=image) comment = (Literal('#') + restOfLine).setParseAction(_makecomment) integer = Word(nums).setParseAction(lambda s: int(s[0])) integer_pair = (integer + integer).setParseAction(lambda x: tuple(x)) poly = Literal("poly") + Group(ZeroOrMore(integer_pair)) + restOfLine poly = poly.setParseAction(_makepoly) rect = Literal("rect") + integer_pair + integer_pair + restOfLine rect = rect.setParseAction(_makerect) circle = Literal("circle") + integer_pair + integer + restOfLine circle = circle.setParseAction(_makecircle) desc = Literal("desc") + (Literal("top-right") | Literal("bottom-right") | Literal("bottom-left") | Literal("top-left") | Literal("none")) desc = desc.setParseAction(_makedesc) default = Literal("default") + restOfLine default.setParseAction(lambda t: Default(caption=t[1].strip()))
def field_list_act(toks): return " | ".join(toks) field_list_def.setParseAction(field_list_act) create_table_def = Literal("CREATE") + "TABLE" + Word(alphas,alphanums+"_").setResultsName("tablename") + \ "("+field_list_def.setResultsName("columns")+")"+ ";" def create_table_act(toks): return """"%(tablename)s" [\n\t label="<%(tablename)s> %(tablename)s | %(columns)s"\n\t shape="record"\n];""" % toks create_table_def.setParseAction(create_table_act) add_fkey_def=Literal("ALTER")+"TABLE"+"ONLY" + Word(alphanums+"_").setResultsName("fromtable") + "ADD" \ + "CONSTRAINT" + Word(alphanums+"_") + "FOREIGN"+"KEY"+"("+Word(alphanums+"_").setResultsName("fromcolumn")+")" \ +"REFERENCES"+Word(alphanums+"_").setResultsName("totable")+"("+Word(alphanums+"_").setResultsName("tocolumn")+")"+";" def add_fkey_act(toks): return """ "%(fromtable)s":%(fromcolumn)s -> "%(totable)s":%(tocolumn)s """ % toks add_fkey_def.setParseAction(add_fkey_act) other_statement_def = (OneOrMore(CharsNotIn(";")) + ";") other_statement_def.setParseAction(replaceWith("")) comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
class ObfuscatePythonBNF(ObfuscateBNF): def __init__(self, get_obfuscated): """BNF grammar for Python source statements. Parameters ---------- get_obfuscated : function Function to return the obfuscated name for an identifier. """ super(ObfuscatePythonBNF, self).__init__(get_obfuscated) self.validator = \ Literal('@') + \ Literal('validate') + \ Literal('(') + \ self.string + \ self.string + \ Literal(')') # Parse a Kivy load_file statement self.builder = \ Literal('Builder.load_file(') + \ self.string + \ Literal(')') self.statement = ( ZeroOrMore( (self.directive | self.builder | self.tab | self.conseq_idents_numbs | self.separator | self.string_or_doc | self.triple_quote) ) + Optional(self.comment).suppress() ) self.except_error = ( ZeroOrMore( (self.tab.suppress() | Literal('except') | self.directive.suppress() | self.tab.suppress() | self.ident | self.separator.suppress() | self.fnumber.suppress() | self.string_or_doc.suppress() | self.triple_quote.suppress()) ) + Optional(self.comment).suppress() ) self.from_import = ( ZeroOrMore( (self.tab.suppress() | Literal('from') | self.directive.suppress() | self.tab.suppress() | self.ident | Literal('import') | self.separator.suppress() | self.fnumber.suppress() | self.string_or_doc.suppress() | self.triple_quote.suppress()) ) + Optional(self.comment).suppress() ) self.except_error.setParseAction(self.add_except_error) self.builder.setParseAction(self.transform_builder) self.from_import.setParseAction(self.add_from_import) ############### # Parse actions ############### def add_from_import(self, from_import_list): """Add imported modules from reserved modules to reserved. Parameters ---------- from_import_list : list """ if not from_import_list or \ from_import_list[0] != 'from' or \ 'import' not in from_import_list[:]: return reserved_list = set() import_index = from_import_list[:].index('import') package_name = '' is_reserved = False for reserve_name in from_import_list[1:import_index]: # Start with first reserved directory in tree (if one exists) if not is_reserved: try: get_reserved_by_name(reserve_name) is_reserved = True package_name = reserve_name except DoesNotExist: continue if is_reserved: if reserve_name[0].isalpha() or reserve_name[0] == '_': reserved_list.add(reserve_name) if is_reserved: # Get imported items for reserve_name in from_import_list[import_index+1:]: if reserve_name[0].isalpha() or reserve_name[0] == '_': reserved_list.add(reserve_name) add_reserveds(package_name, reserved_list) def add_except_error(self, except_error_list): """Add except Error names to reserved. Parameters ---------- except_error_list : list """ if not except_error_list or except_error_list[0] != 'except': return reserved_list = set() package_name = 'Except' for reserve_name in except_error_list[1:]: if reserve_name == 'as': break if reserve_name[0].isalpha() or reserve_name[0] == '_': reserved_list.add(reserve_name) if reserved_list: add_reserveds(package_name, reserved_list) def transform_builder(self, builder_list): """Parse a Kivy load_file statement. Parameters builder_list : list Kivy Builder.load_file statement. """ return ''.join([ builder_list[0], "'", obfuscate_path(builder_list[1].strip("'")), "'", builder_list[2] ])
def dept(deptJson): print "Processing " + deptJson + "..." deptCourses = json.loads(open(deptJson).read()) for i in deptCourses["result_data"]: courseId = i["course_id"].replace(" ", "") preReqs = i["prerequisites"] prereqList = [] if preReqs == "": continue if re.search("\w{3,4}\s{0,1}\d{3}", preReqs): if courseId not in courseMap: courseMap[courseId] = {} # Replace Department names with DEPT sign for key, val in deptNameMap.iteritems(): if key in preReqs.lower(): preReqs = re.sub(key, val, preReqs, flags=re.I) # Remove phone numbers preReqs = re.sub("\d{3}-\d{3}-\d{4}", "", preReqs) if "/" in preReqs: preReqs = preReqs.replace("/", " or ") # Instructor Permission if "instructor" in (preReqs.lower()) and \ "permission" in (preReqs.lower()): courseMap[courseId]["instructorPermission"] = True m = re.findall("((?:\w{2,4}){0,1}\s{0,1}\d{3}),{0,1}\s{0,1}(,|or|and|AND|OR){0,1}", preReqs) if m: n = ["".join(i.split()) for sub in m for i in sub] preReqs = " ".join(n).upper() preReqs = ", ".join(preReqs.split()) preReqs = preReqs.replace("OR,", "OR") preReqs = preReqs.replace("AND,", "AND") preReqs = rchop(preReqs, ", OR") preReqs = rchop(preReqs, ", AND") course_name = Word(string.ascii_uppercase + nums) | Word(nums) comma_separator = Literal(',') comma_separator.setParseAction(lambda t: "&&") and_separator = Literal('AND') | Literal(", AND") and_separator.setParseAction(lambda t: "&&") or_separator = Literal('OR') | Literal(", OR") or_separator.setParseAction(lambda t: "||") course_line = operatorPrecedence(course_name, [ (and_separator, 2, opAssoc.LEFT,), (or_separator, 2, opAssoc.LEFT), (comma_separator, 2, opAssoc.LEFT,), ]) results = course_line.parseString(preReqs) x = results.asList()[0] if len(x): results_list = [] results_list.extend(x if type(x) == list else [x]) results_list = fillDeptIds(results_list) courseMap[courseId]["prerequisites"] = results_list fillSatisCourse(courseId, results_list)
STRING_LITERAL1 = Regex(ur"'(?:[^'\n\r\\]|\\['ntbrf\\])*'(?!')", flags=re.U) STRING_LITERAL1.setParseAction( lambda x: rdflib.Literal(decodeUnicodeEscape(x[0][1:-1]))) # [157] STRING_LITERAL2 ::= '"' ( ([^#x22#x5C#xA#xD]) | ECHAR )* '"' # STRING_LITERAL2 = Literal('"') + ZeroOrMore ( # Regex(u'[^\u0022\u005C\u000A\u000D]',flags=re.U) | ECHAR ) + '"' STRING_LITERAL2 = Regex(ur'"(?:[^"\n\r\\]|\\["ntbrf\\])*"(?!")', flags=re.U) STRING_LITERAL2.setParseAction( lambda x: rdflib.Literal(decodeUnicodeEscape(x[0][1:-1]))) # [161] NIL ::= '(' WS* ')' NIL = Literal('(') + ')' NIL.setParseAction(lambda x: rdflib.RDF.nil) # [162] WS ::= #x20 | #x9 | #xD | #xA # Not needed? # WS = #x20 | #x9 | #xD | #xA # [163] ANON ::= '[' WS* ']' ANON = Literal('[') + ']' ANON.setParseAction(lambda x: rdflib.BNode()) # A = CaseSensitiveKeyword('a') A = Literal('a') A.setParseAction(lambda x: rdflib.RDF.type) # ------ NON-TERMINALS -------------- # [5] BaseDecl ::= 'BASE' IRIREF
# Basic expressions _basic_expr = Forward() _conversion = ( Literal("word1") + Suppress("(") + _basic_expr + Suppress(")") | Literal("bool") + Suppress("(") + _basic_expr + Suppress(")") | Literal("toint") + Suppress("(") + _basic_expr + Suppress(")") | Literal("signed") + Suppress("(") + _basic_expr + Suppress(")") | Literal("unsigned") + Suppress("(") + _basic_expr + Suppress(")") ) _conversion.setParseAction(lambda s, l, t: Conversion(t[0], t[1])) _word_function = Literal("extend") + Suppress("(") + _basic_expr + "," + _basic_expr + Suppress(")") | Literal( "resize" ) + Suppress("(") + _basic_expr + "," + _basic_expr + Suppress(")") _word_function.setParseAction(lambda s, l, t: WordFunction(t[0], t[1], t[2])) _count = Literal("count") + Suppress("(") + delimitedList(_basic_expr) + Suppress(")") _count.setParseAction(lambda s, l, t: Count(t[1])) _next = Literal("next") + Suppress("(") + _basic_expr + Suppress(")") _next.setParseAction(lambda s, l, t: Next(t[1])) _case_case = _basic_expr + Suppress(":") + _basic_expr + Suppress(";") _case_body = OneOrMore(_case_case) _case_body.setParseAction(lambda s, l, t: OrderedDict(zip(t[::2], t[1::2]))) _case = Suppress("case") + _case_body + Suppress("esac") _case.setParseAction(lambda s, l, t: Case(t[0])) _base = complex_identifier ^ ( _conversion | _word_function | _count | _next | Suppress("(") + _basic_expr + Suppress(")") | _case | constant
def get_grammar(self): """ Defines our grammar for mathematical expressions. Possibly helpful: - BNF form of context-free grammar https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form - Some pyparsing docs http://infohost.nmt.edu/~shipman/soft/pyparsing/web/index.html """ # Define + and - plus = Literal("+") # Also accept unicode emdash emdash = Literal("\u2014") emdash.setParseAction(lambda: "-") minus = Literal("-") | emdash plus_minus = plus | minus # 1 or 1.0 or .1 number_part = Word(nums) inner_number = Combine((number_part + Optional("." + Optional(number_part))) | ("." + number_part)) # Combine() joints the matching parts together in a single token, # and requires that the matching parts be contiguous (no spaces) # Define our suffixes suffix = Word(alphas + '%') suffix.setParseAction(self.suffix_parse_action) # Construct number as a group consisting of a text string ("num") and an optional suffix. # num can include a decimal number and numerical exponent, and can be # converted to a number using float() # suffix may contain alphas or % # Spaces are ignored inside numbers # Group wraps everything up into its own ParseResults object when parsing number = Group( Combine( inner_number + Optional( CaselessLiteral("E") + Optional(plus_minus) + number_part), )("num") + Optional(suffix)("suffix"))("number") # Note that calling ("name") on the end of a parser is equivalent to calling # parser.setResultsName, which is used to pull that result out of a parsed # expression like a dictionary. # Construct variable and function names front = Word(alphas, alphanums) # must start with alpha subscripts = Word(alphanums + '_') + ~FollowedBy('{') # ~ = not lower_indices = Literal("_{") + Optional("-") + Word( alphanums) + Literal("}") upper_indices = Literal("^{") + Optional("-") + Word( alphanums) + Literal("}") # Construct an object name in either of two forms: # 1. front + subscripts + tail # 2. front + lower_indices + upper_indices + tail # where: # front (required): # starts with alpha, followed by alphanumeric # subscripts (optional): # any combination of alphanumeric and underscores # lower_indices (optional): # Of form "_{(-)<alphanumeric>}" # upper_indices (optional): # Of form "^{(-)<alphanumeric>}" # tail (optional): # any number of primes name = Combine(front + Optional(subscripts | (Optional(lower_indices) + Optional(upper_indices))) + ZeroOrMore("'")) # Define a variable as a pyparsing result that contains one object name variable = Group(name("varname"))("variable") variable.setParseAction(self.variable_parse_action) # initialize recursive grammar expression = Forward() # Construct functions as consisting of funcname and arguments as # funcname(arguments) # where arguments is a comma-separated list of arguments, returned as a list # Must have at least 1 argument function = Group( name("funcname") + Suppress("(") + Group(delimitedList(expression))("arguments") + Suppress(")"))("function") function.setParseAction(self.function_parse_action) # Define parentheses parentheses = Group(Suppress("(") + expression + Suppress(")"))('parentheses') # Define arrays array = Group( Suppress("[") + delimitedList(expression) + Suppress("]"))("array") # atomic units evaluate directly to number or array without binary operations atom = number | function | variable | parentheses | array # Define operations in order of precedence # Define exponentiation, possibly including negative powers power = atom + ZeroOrMore(Suppress("^") + Optional(minus)("op") + atom) power.addParseAction(self.group_if_multiple('power')) # Define negation (e.g., in 5*-3 --> we need to evaluate the -3 first) # Negation in powers is handled separately # This has been arbitrarily assigned a higher precedence than parallel negation = Optional(minus)("op") + power negation.addParseAction(self.group_if_multiple('negation')) # Define the parallel operator 1 || 5 == 1/(1/1 + 1/5) pipes = Literal('|') + Literal('|') parallel = negation + ZeroOrMore(Suppress(pipes) + negation) parallel.addParseAction(self.group_if_multiple('parallel')) # Define multiplication and division product = parallel + ZeroOrMore((Literal('*') | Literal('/'))("op") + parallel) product.addParseAction(self.group_if_multiple('product')) # Define sums and differences # Note that leading - signs are treated by negation sumdiff = Optional(plus) + product + ZeroOrMore( plus_minus("op") + product) sumdiff.addParseAction(self.group_if_multiple('sum')) # Close the recursion expression << sumdiff return expression + stringEnd
scopedIdent = ident + OneOrMore( Literal("::").suppress() + ident ) scopedIdent.setParseAction(lambda t: "_".join(t)) print("(replace namespace-scoped names with C-compatible names)") print(scopedIdent.transformString( testData )) # or a crude pre-processor (use parse actions to replace matching text) def substituteMacro(s,l,t): if t[0] in macros: return macros[t[0]] ident.setParseAction( substituteMacro ) ident.ignore(macroDef) print("(simulate #define pre-processor)") print(ident.transformString( testData )) ################# print("Example of a stripper") print("----------------------") from pyparsing import dblQuotedString, LineStart # remove all string macro definitions (after extracting to a string resource table?) stringMacroDef = Literal("#define") + ident + "=" + dblQuotedString + LineStart() stringMacroDef.setParseAction( replaceWith("") ) print(stringMacroDef.transformString( testData ))
hexnum = Combine(Literal("0x") + Word(string.hexdigits)).setParseAction(lambda t: int(t[0][2:], 16)) dimension = identifier.setResultsName("name") + \ Optional(Suppress(Literal("(")) + (Literal("string") | Literal("uint64")) + Suppress(Literal(")")), default="string").setResultsName("type") dimension.setParseAction(parse_dimension) autoregion = Literal("auto") + integer + integer staticregion = Literal("region") + integer + hexnum + integer region = ZeroOrMore(Group(staticregion)) + Optional(Group(autoregion)) region.setParseAction(parse_regions) subspace = Literal("subspace").suppress() + \ Group(delimitedList(identifier)) + \ Optional(Suppress(Literal("nosearch")) + Group(delimitedList(identifier)), default=[]) + \ Group(region) subspace.setParseAction(parse_subspace) space = Literal("space").suppress() + identifier.setResultsName("name") + \ Literal("dimensions").suppress() + Group(delimitedList(dimension)).setResultsName("dimensions") + \ Literal("key").suppress() + identifier.setResultsName("key") + \ Group(region).setResultsName("keyregions") + \ ZeroOrMore(subspace).setResultsName("subspaces") space.setParseAction(parse_space) class TestFillToRegion(unittest.TestCase): def test_already_met(self): a = 0 b = 1 while a < (1 << 64): self.assertEqual([], _fill_to_region(a, 0, 4, a))
hexdigits = Word(string.hexdigits, exact=2) hexdigits.setName('hexdigits') escaped = Suppress(Literal('\\')) + hexdigits escaped.setName('escaped') def _p_escaped(s, l, t): text = t[0] return chr(int(text, 16)) escaped.setParseAction(_p_escaped) value = Combine(OneOrMore(CharsNotIn('*()\\\0') | escaped)) value.setName('value') equal = Literal("=") equal.setParseAction(lambda s, l, t: pureldap.LDAPFilter_equalityMatch) approx = Literal("~=") approx.setParseAction(lambda s, l, t: pureldap.LDAPFilter_approxMatch) greater = Literal(">=") greater.setParseAction(lambda s, l, t: pureldap.LDAPFilter_greaterOrEqual) less = Literal("<=") less.setParseAction(lambda s, l, t: pureldap.LDAPFilter_lessOrEqual) filtertype = equal | approx | greater | less filtertype.setName('filtertype') simple = attr + filtertype + value simple.leaveWhitespace() simple.setName('simple') def _p_simple(s, l, t): attr, filtertype, value = t