def genParseObject(self,cLocale): sym = DateFormatSymbols(cLocale) val = sym.getEras() frag = Or(CaselessLiteral(val[0].encode('utf-8'))) for item in val[1:]: frag.append(CaselessLiteral(item.encode('utf-8'))) return frag.setResultsName('era')
def _angular_distance(): _m = (usn + Literal("\'").leaveWhitespace()) _s = (usn + Literal("\"").leaveWhitespace()) ms = Or([_m + Optional(_s), _s]) ms = ms.setParseAction(lambda s, l, tok: AngularDistance(tok)) return ms
def create_post_timedelta_literal(tok): """Detects <plus/minus> <number> <timescale>""" delta = Or( [CaselessLiteral(t) for t in ['+', '-', 'plus', 'minus']] ) + originalTextFor(Or([ Word(nums) + ZeroOrMore(',' + Word(nums+',')) + ZeroOrMore('.' + Word(nums)), CaselessLiteral('an'), CaselessLiteral('a') ])) + CaselessLiteral(tok) + StringEnd() delta.setName('post' + tok).\ setParseAction(DateParser.generate_post_timedelta) return delta
def create_post_timedelta_literal(tok): """ Detects <plus/minus> <number> <timescale> :param tok: the token we want to produce a detector for :type tok: str :return: the caseless literal :rtype: pyparsing.Or """ delta = Or( [CaselessLiteral(t) for t in ['+', '-', 'plus', 'minus']] ) + originalTextFor(Or([ Word(nums) + ZeroOrMore(',' + Word(nums+',')) + ZeroOrMore('.' + Word(nums)), CaselessLiteral('an'), CaselessLiteral('a') ])) + CaselessLiteral(tok) + StringEnd() delta.setName('post' + tok).\ setParseAction(DateParser.generate_post_timedelta) return delta
def get_precedence_list(self, parse_unary, parse_binary): """Build a precedence list to be used with pyparsing.infixNotation.""" precedence_list = [] for symbol_list in self.__ordered_operators: expression = Or(symbol_list) if type(self.__operators[symbol_list[0]]) is UnaryOperator: arity = 1 parsing_function = parse_unary associativity = opAssoc.RIGHT else: arity = 2 parsing_function = parse_binary associativity = opAssoc.LEFT entry = (expression, arity, associativity, parsing_function) precedence_list.append(entry) return precedence_list
def instance(): lit_e = CaselessLiteral('E') plusorminus = Literal('+') | Literal('-') number = Word(nums) integer = Combine(Optional(plusorminus) + number).setParseAction(lambda t: int(t[0])) index = integer.copy().addParseAction(index_check(0)) floatnumber = Combine(integer + Optional(Literal('.') + Optional(number)) + Optional(lit_e + integer)).setParseAction( lambda t: float(t[0])) #comment = Suppress("%") + Word(alphanums + " ") comment = Regex(r"%.*").setName("comment").suppress() linend = Or([comment, LineEnd()]).suppress() section_end = (Literal('#') + LineEnd()).suppress() vertex = (Group( OneOrMore(floatnumber('point') + OneOrMore(White()).suppress())) + linend)('vertex') vertex_header = (Keyword('VERTEX') + linend).suppress() vertex_section = (vertex_header + Group(OneOrMore(vertex))('vertices') + section_end) simplex = ( Group(OneOrMore(index('index') + OneOrMore(White()).suppress())) + linend)('simplex') simplex_header = (Keyword('SIMPLEX') + linend).suppress() simplex_section = (simplex_header + Group(OneOrMore(simplex))('simplices') + section_end) boundarysegment = (Group( index('id') + OneOrMore(index('index') + OneOrMore(White()).suppress())) + linend)('boundarysegment') boundarysegment_header = (Keyword('BOUNDARYSEGMENTS') + linend).suppress() boundarysegment_section = ( boundarysegment_header + Dict(OneOrMore(boundarysegment))('boundarysegments') + section_end) sections = Each([vertex_section, simplex_section, boundarysegment_section]) dgf_header = (Keyword('DGF') + linend).suppress() dgf = (dgf_header + Dict(sections) + OneOrMore(section_end))('dgf') return dgf
def __init__(self): RegionPusher.__init__(self) self.shape_definition = ds9_shape_defs regionShape = define_shape_helper(self.shape_definition) regionShape = regionShape.setParseAction( lambda s, l, tok: Shape(tok[0], tok[1:])) regionExpr = define_expr( regionShape, negate_func=lambda s, l, tok: tok[-1].set_exclude(), ) coord_command_keys = "PHYSICAL IMAGE FK4 B1950 FK5 J2000 GALACTIC ECLIPTIC ICRS LINEAR AMPLIFIER DETECTOR".split( ) coordCommand = define_simple_literals( coord_command_keys, parseAction=lambda s, l, tok: CoordCommand(tok[-1])) regionGlobal = comment_shell_like(CaselessKeyword("global"), lambda s, l, tok: Global(tok[-1])) regionAtom = (regionExpr | coordCommand | regionGlobal) regionAtom = regionAtom.setParseAction(self.pushAtom) regionComment = comment_shell_like(Literal("#"), parseAction=self.pushComment) line_simple = define_line(atom=regionAtom, separator=Literal(";"), comment=regionComment) line_w_composite = And([regionAtom, CaselessKeyword("||").setParseAction(self.set_continued) ]) \ + Optional(regionComment) line = Or([line_simple, line_w_composite]) self.parser = Optional(line) + StringEnd()
def property_grammar(): ParserElement.setDefaultWhitespaceChars(' ') model_props = [ Regex(r'LIBINPUT_MODEL_[_0-9A-Z]+')('NAME') - Suppress('=') - (Literal('1'))('VALUE') ] dimension = INTEGER('X') + Suppress('x') + INTEGER('Y') sz_props = ( ('LIBINPUT_ATTR_SIZE_HINT', Group(dimension('SETTINGS*'))), ('LIBINPUT_ATTR_RESOLUTION_HINT', Group(dimension('SETTINGS*'))), ) size_props = [ Literal(name)('NAME') - Suppress('=') - val('VALUE') for name, val in sz_props ] grammar = Or(model_props + size_props) return grammar
def make_one_header_assignment_parser(struct_parsers): """Create a parser that can parse a single header assignment :Parameters: - `struct_parsers`: a dictionary of parsers for structs @return: a parser that can parse on header assignment >>> test_parser = make_one_header_assignment_parser({}) >>> >>> test_string = "a 42 32" >>> parsed_assignments = test_parser.parseString(test_string) >>> print parsed_assignments[0]['name'] a >>> print parsed_assignments[0]['value'] 42 32 """ all_struct_parsers = Or([struct_parsers[s] for s in struct_parsers.keys()]) one_header_assignment_parser = NotAny(all_struct_parsers) + Group( value_name('name') + restOfLine('value')) one_header_assignment_parser.ignore(hash_comment) return one_header_assignment_parser
def make_date_parser(): date_expr = ppc.iso8601_date.copy() date_expr.setParseAction(ppc.convertToDate()) expr_last = ( CaselessKeyword('LAST') & ppc.integer.setResultsName('n') & StringEnd()).setResultsName('interval').setParseAction(handle_last) expr_prev = (CaselessKeyword('PREVIOUS') & Or( CaselessKeyword('DAY').setResultsName('day') | CaselessKeyword('WEEK').setResultsName('week') | CaselessKeyword('MONTH').setResultsName('month')) + StringEnd() ).setResultsName('interval').setParseAction(handle_previous) expr_fromto_date = ( CaselessKeyword('FROM') + date_expr.setResultsName('start') + CaselessKeyword('TO') + date_expr.setResultsName('end') + StringEnd()).setResultsName('interval').setParseAction(handle_fromto) parser = expr_fromto_date | expr_last | expr_prev return parser
def _make_mouse_parser(): from pyparsing import (Optional, Literal, Word, Group, Keyword, Or, ZeroOrMore, Regex, Suppress) double = Regex(r'-?\d+(\.\d*)?([eE]\d+)?') coords = double + Suppress(Optional(Literal(','))) + double integer = Word('0123456789') move = ((Literal('(') + coords + Suppress(Literal(')'))) | (Literal('[') + coords + Suppress(Literal(']'))) | (Literal('<') + coords + Suppress(Literal('>')))) buttons = ('left', 'middle', 'right', 'wheelup', 'wheeldown') key = (Or([Keyword(sym) for sym in buttons]) | integer) press = ( key + Optional(Literal(':') + (integer | (Literal('up') | Literal('down')))) + Optional(Literal('/') + integer)) list_element = Group(move | press) list_parser = list_element + ZeroOrMore(Suppress(',') + list_element) return list_parser
def ruleParser(): global typedversion # ------------------------------------------------------ # Atomic # ------------------------------------------------------ # Tokens lbrX = Literal("(") rbrX = Literal(")") commaX = Literal(",") lbr = Literal("(").suppress() rbr = Literal(")").suppress() comma = Literal(",").suppress() hash = Literal("#") equ = Literal("=") implies = Literal("=>").suppress() dot = Literal(".").suppress() eol = Literal("\n").suppress() # Basic constructors Alfabet = alphas + nums + "_$" Number = Word(nums) # Typeinfo/Constant TypeInfo = oneOf("mr nonce pk sk fu table") Constant = Word(alphas, Alfabet) # Time nTime = Number xTime = Literal("xTime") sTime = Literal("s") + lbrX + Number + rbrX Time = Or([nTime, xTime, sTime]) # Const Const = Forward() ConstC = Literal("c") + lbrX + Constant + commaX + Time + rbrX ConstF = Literal("c(ni,ni)") Const << Or([Constant, ConstC, ConstF]) def stringize(s, l, t): return ["".join(t)] Const.setParseAction(stringize) # Optional prime def optprimeaction(s, l, t): if len(t) == 0: return [""] else: return t optprime = Optional(Literal("'")) optprime.setParseAction(optprimeaction) # Two versions if typedversion: Variable = Word("x", Alfabet) Variable = TypeInfo + lbr + Variable + rbr + optprime Variable.setParseAction(lambda s, l, t: [If.Variable(t[0], t[1], t[2])]) else: Variable = Word("x", Alfabet) + optprime Variable.setParseAction(lambda s, l, t: [If.Variable("untyped", t[0], t[1])]) # Atomic ## DEVIANT : below there is an optprime after the atom. This ## is not in the BNF. TypedConstant = TypeInfo + lbr + Const + rbr + optprime TypedConstant.setParseAction(lambda s, l, t: [If.Constant(t[0], t[1], t[2])]) Atomic = Or(TypedConstant, Variable) ### TEST # print Atomic.parseString("mr(Cas)'") # print Atomic.parseString("nonce(Koen)") # ------------------------------------------------------ # Messages # ------------------------------------------------------ # Base forward declaration Message = Forward() # Agents etc AgentMr = Literal("mr") + lbr + Const + rbr AgentMr.setParseAction(lambda s, l, t: [If.Constant(t[0], t[1])]) Agent = Or([AgentMr, Variable]) # TODO Not implemented yet KeyTable = Or([Literal("table") + lbr + Const + rbr, Variable]) KeyTableApp = Literal("tb") + lbr + KeyTable + comma + Agent + rbr + optprime # Crypto pkterm = Literal("pk") + lbr + Const + rbr + optprime pkterm.setParseAction(lambda s, l, t: [If.PublicKey(t[0], t[1], t[2])]) ##varterm = Variable + optprime ### Variable already has an optprime varterm = Variable Invertible = Or([pkterm, KeyTableApp, varterm]) PublicCypher = Literal("crypt") + lbr + Invertible + comma + Message + rbr PublicCypher.setParseAction(lambda s, l, t: [If.PublicCrypt(t[1], t[2])]) XOR = Literal("rcrypt") + lbr + Message + comma + Message + rbr XOR.setParseAction(lambda s, l, t: [If.XOR(t[1], t[2])]) SymmetricCypher = Literal("scrypt") + lbr + Message + comma + Message + rbr SymmetricCypher.setParseAction(lambda s, l, t: [If.SymmetricCrypt(t[1], t[2])]) # TODO Not implemented yet futerm = Or([Literal("fu") + lbr + Const + rbr, Variable]) Function = Literal("funct") + lbr + futerm + comma + Message + rbr Concatenation = Literal("c").suppress() + lbr + Message + comma + Message + rbr Concatenation.setParseAction(lambda s, l, t: [If.Composed(t[0], t[1])]) # Message composition Composed = Or([Concatenation, SymmetricCypher, XOR, PublicCypher, Function, KeyTable, KeyTableApp]) Message << Or([Composed, Atomic]) ### TEST # print Message.parseString("nonce(c(Na,xTime))") # ------------------------------------------------------ # Model of honest agents # ------------------------------------------------------ Boolean = Or([Literal("true"), Literal("false"), Variable]) Session = Forward() Session << Or([Literal("s") + lbr + Session + rbr, Number, Variable]) MsgEtc = Literal("etc") MsgEtc.setParseAction(lambda s, l, t: [If.MsgList([If.Constant("special", "etc")])]) MsgVar = Group(Variable) MsgVar.setParseAction(lambda s, l, t: [If.MsgList(t)]) MsgList = Forward() MsgComp = Literal("c") + lbr + Message + comma + MsgList + rbr MsgComp.setParseAction(lambda s, l, t: [If.MsgList([t[1]] + t[2].getList())]) MsgList << Or([MsgEtc, Variable, MsgComp]) Step = Or([Number, Variable]) ### TEST # print Message.parseString("xKb") # print Message.parseString("mr(Cas)") # print MsgList.parseString("etc") # print MsgList.parseString("c(xKb,etc)") # print MsgList.parseString("c(xA,c(xB,c(xKa,c(xKa',c(xKb,etc)))))") # Principal fact Principal = ( Literal("w") + lbr + Step + comma + Agent + comma + Agent + comma + MsgList + comma + MsgList + comma + Boolean + comma + Session + rbr ) Principal.setParseAction(lambda s, l, t: [If.PrincipalFact(t[1:])]) # Message fact MessageFact = ( Literal("m") + lbr + Step + comma + Agent + comma + Agent + comma + Agent + comma + Message + comma + Session + rbr ) MessageFact.setParseAction(lambda s, l, t: [If.MessageFact(t[1:])]) # Goal fact Secret = Literal("secret") + lbr + Message + Literal("f") + lbr + Session + rbr + rbr Give = Literal("give") + lbr + Message + Literal("f") + lbr + Session + rbr + rbr Witness = Literal("witness") + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr Request = Literal("request") + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr GoalFact = Or([Secret, Give, Witness, Request]) GoalFact.setParseAction(lambda s, l, t: [If.GoalFact(t)]) # Goal State # It actually yields a rule (not a state per se) Correspondence = Principal + dot + Principal Correspondence.setParseAction(lambda s, l, t: [If.CorrespondenceRule(t)]) Secrecy = ( Literal("secret") + lbr + Literal("xsecret") + comma + Literal("f") + lbr + Session + rbr + rbr + dot + Literal("i") + lbr + Literal("xsecret") + rbr ) Secrecy.setParseAction(lambda s, l, t: [If.SecrecyRule(t)]) STSecrecy = Literal("give(xsecret,f(xc)).secret(xsecret,f(xc))") + implies + Literal("i(xsecret)") STSecrecy.setParseAction(lambda s, l, t: [If.STSecrecyRule(t)]) Authenticate = Literal("request") + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr Authenticate.setParseAction(lambda s, l, t: [If.AuthenticateRule(t)]) GoalState = Or([Correspondence, Secrecy, STSecrecy, Authenticate]) # TimeFact TimeFact = Literal("h") + lbr + Message + rbr TimeFact.setParseAction(lambda s, l, t: [If.TimeFact(t[1])]) # Intruder knowledge IntruderKnowledge = Literal("i") + lbr + Message + rbr # Facts and states Fact = Or([Principal, MessageFact, IntruderKnowledge, TimeFact, Secret, Give, Witness, Request]) Fact.setParseAction(lambda s, l, t: [If.Fact(t)]) State = Group(delimitedList(Fact, ".")) ## From initial part of document, not in detailed BNF State.setParseAction(lambda s, l, t: [If.State(t)]) # Rules MFPrincipal = Or([MessageFact + dot + Principal, Principal]) mr1 = Literal("h") + lbr + Literal("s") + lbr + Literal("xTime") + rbr + rbr + dot + MFPrincipal mr2 = implies mr3 = Literal("h") + lbr + Literal("xTime") + rbr + dot + MFPrincipal + Optional(dot + delimitedList(GoalFact, ".")) MessageRule = Group(mr1) + mr2 + Group(mr3) ## DEVIANT : BNF requires newlines MessageRule.setParseAction(lambda s, l, t: [If.MessageRule(t[0][3:], t[1][2:])]) InitialState = Literal("h") + lbr + Literal("xTime") + rbr + dot + State ## DEVIANT : BNF requires newlines InitialState.setParseAction(lambda s, l, t: [If.InitialRule(t[2])]) # Intruder IntruderRule = Literal("nogniet") # Simplification f_simplif = ( Literal("f") + lbr + Literal("s") + lbr + Literal("xc") + rbr + rbr + implies + Literal("f") + lbr + Literal("xc") + rbr ) ## DEVIANT : EOL removed matching_request = Witness + dot + Request + implies no_auth_intruder = Request + implies SimplificationRule = Or([f_simplif, matching_request, no_auth_intruder]) # Compose all rules Rule = Or([InitialState, MessageRule, IntruderRule, GoalState, SimplificationRule]) return Rule
def parse(cls, content, basedir=None, resolve=True, unresolved_value=DEFAULT_SUBSTITUTION): """parse a HOCON content :param content: HOCON content to parse :type content: basestring :param resolve: if true, resolve substitutions :type resolve: boolean :param unresolved_value: assigned value to unresolved substitution. If overriden with a default value, it will replace all unresolved values by the default value. If it is set to pyhocon.STR_SUBSTITUTION then it will replace the value by its substitution expression (e.g., ${x}) :type unresolved_value: boolean :return: a ConfigTree or a list """ unescape_pattern = re.compile(r'\\.') def replace_escape_sequence(match): value = match.group(0) return cls.REPLACEMENTS.get(value, value) def norm_string(value): return unescape_pattern.sub(replace_escape_sequence, value) def unescape_string(tokens): return ConfigUnquotedString(norm_string(tokens[0])) def parse_multi_string(tokens): # remove the first and last 3 " return tokens[0][3:-3] def convert_number(tokens): n = tokens[0] try: return int(n, 10) except ValueError: return float(n) def convert_period(tokens): period_value = int(tokens.value) period_identifier = tokens.unit period_unit = next((single_unit for single_unit, values in cls.get_supported_period_type_map().items() if period_identifier in values)) return period(period_value, period_unit) # ${path} or ${?path} for optional substitution SUBSTITUTION_PATTERN = r"\$\{(?P<optional>\?)?(?P<variable>[^}]+)\}(?P<ws>[ \t]*)" def create_substitution(instring, loc, token): # remove the ${ and } match = re.match(SUBSTITUTION_PATTERN, token[0]) variable = match.group('variable') ws = match.group('ws') optional = match.group('optional') == '?' substitution = ConfigSubstitution(variable, optional, ws, instring, loc) return substitution # ${path} or ${?path} for optional substitution STRING_PATTERN = '"(?P<value>(?:[^"\\\\]|\\\\.)*)"(?P<ws>[ \t]*)' def create_quoted_string(instring, loc, token): # remove the ${ and } match = re.match(STRING_PATTERN, token[0]) value = norm_string(match.group('value')) ws = match.group('ws') return ConfigQuotedString(value, ws, instring, loc) def include_config(instring, loc, token): url = None file = None required = False if token[0] == 'required': required = True final_tokens = token[1:] else: final_tokens = token if len(final_tokens) == 1: # include "test" value = final_tokens[0].value if isinstance( final_tokens[0], ConfigQuotedString) else final_tokens[0] if value.startswith("http://") or value.startswith( "https://") or value.startswith("file://"): url = value else: file = value elif len(final_tokens) == 2: # include url("test") or file("test") value = final_tokens[1].value if isinstance( final_tokens[1], ConfigQuotedString) else final_tokens[1] if final_tokens[0] == 'url': url = value elif final_tokens[0] == 'package': file = cls.resolve_package_path(value) else: file = value if url is not None: logger.debug('Loading config from url %s', url) obj = ConfigFactory.parse_URL(url, resolve=False, required=required, unresolved_value=NO_SUBSTITUTION) elif file is not None: path = file if basedir is None else os.path.join(basedir, file) def _make_prefix(path): return ('<root>' if path is None else '[%s]' % path).ljust(55).replace('\\', '/') _prefix = _make_prefix(path) def _load(path): _prefix = _make_prefix(path) logger.debug('%s Loading config from file %r', _prefix, path) obj = ConfigFactory.parse_file( path, resolve=False, required=required, unresolved_value=NO_SUBSTITUTION) logger.debug('%s Result: %s', _prefix, obj) return obj if '*' in path or '?' in path: paths = glob(path, recursive=True) obj = None def _merge(a, b): if a is None or b is None: return a or b elif isinstance(a, ConfigTree) and isinstance( b, ConfigTree): return ConfigTree.merge_configs(a, b) elif isinstance(a, list) and isinstance(b, list): return a + b else: raise ConfigException( 'Unable to make such include (merging unexpected types: {a} and {b}', a=type(a), b=type(b)) logger.debug('%s Loading following configs: %s', _prefix, paths) for p in paths: obj = _merge(obj, _load(p)) logger.debug('%s Result: %s', _prefix, obj) else: logger.debug('%s Loading single config: %s', _prefix, path) obj = _load(path) else: raise ConfigException( 'No file or URL specified at: {loc}: {instring}', loc=loc, instring=instring) return ConfigInclude(obj if isinstance(obj, list) else obj.items()) @contextlib.contextmanager def set_default_white_spaces(): default = ParserElement.DEFAULT_WHITE_CHARS ParserElement.setDefaultWhitespaceChars(' \t') yield ParserElement.setDefaultWhitespaceChars(default) with set_default_white_spaces(): assign_expr = Forward() true_expr = Keyword("true", caseless=True).setParseAction( replaceWith(True)) false_expr = Keyword("false", caseless=True).setParseAction( replaceWith(False)) null_expr = Keyword("null", caseless=True).setParseAction( replaceWith(NoneValue())) key = QuotedString( '"', escChar='\\', unquoteResults=False) | Word(alphanums + alphas8bit + '._- /') eol = Word('\n\r').suppress() eol_comma = Word('\n\r,').suppress() comment = (Literal('#') | Literal('//')) - SkipTo(eol | StringEnd()) comment_eol = Suppress(Optional(eol_comma) + comment) comment_no_comma_eol = (comment | eol).suppress() number_expr = Regex( r'[+-]?(\d*\.\d+|\d+(\.\d+)?)([eE][+\-]?\d+)?(?=$|[ \t]*([\$\}\],#\n\r]|//))', re.DOTALL).setParseAction(convert_number) # Flatten the list of lists with unit strings. period_types = list( itertools.chain(*cls.get_supported_period_type_map().values())) # `Or()` tries to match the longest expression if more expressions # are matching. We employ this to match e.g.: 'weeks' so that we # don't end up with 'w' and 'eeks'. Note that 'weeks' but also 'w' # are valid unit identifiers. # Allow only spaces as a valid separator between value and unit. # E.g. \t as a separator is invalid: '10<TAB>weeks'. period_expr = ( Word(nums)('value') + ZeroOrMore(White(ws=' ')).suppress() + Or(period_types)('unit') + WordEnd(alphanums).suppress()).setParseAction(convert_period) # multi line string using """ # Using fix described in http://pyparsing.wikispaces.com/share/view/3778969 multiline_string = Regex( '""".*?"*"""', re.DOTALL | re.UNICODE).setParseAction(parse_multi_string) # single quoted line string quoted_string = Regex( r'"(?:[^"\\\n]|\\.)*"[ \t]*', re.UNICODE).setParseAction(create_quoted_string) # unquoted string that takes the rest of the line until an optional comment # we support .properties multiline support which is like this: # line1 \ # line2 \ # so a backslash precedes the \n unquoted_string = Regex( r'(?:[^^`+?!@*&"\[\{\s\]\}#,=\$\\]|\\.)+[ \t]*', re.UNICODE).setParseAction(unescape_string) substitution_expr = Regex(r'[ \t]*\$\{[^\}]+\}[ \t]*' ).setParseAction(create_substitution) string_expr = multiline_string | quoted_string | unquoted_string value_expr = period_expr | number_expr | true_expr | false_expr | null_expr | string_expr include_content = (quoted_string | ( (Keyword('url') | Keyword('file') | Keyword('package')) - Literal('(').suppress() - quoted_string - Literal(')').suppress())) include_expr = (Keyword("include", caseless=True).suppress() + (include_content | (Keyword("required") - Literal('(').suppress() - include_content - Literal(')').suppress())) ).setParseAction(include_config) root_dict_expr = Forward() dict_expr = Forward() list_expr = Forward() multi_value_expr = ZeroOrMore(comment_eol | include_expr | substitution_expr | dict_expr | list_expr | value_expr | (Literal('\\') - eol).suppress()) # for a dictionary : or = is optional # last zeroOrMore is because we can have t = {a:4} {b: 6} {c: 7} which is dictionary concatenation inside_dict_expr = ConfigTreeParser( ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma)) inside_root_dict_expr = ConfigTreeParser( ZeroOrMore(comment_eol | include_expr | assign_expr | eol_comma), root=True) dict_expr << Suppress('{') - inside_dict_expr - Suppress('}') root_dict_expr << Suppress('{') - inside_root_dict_expr - Suppress( '}') list_entry = ConcatenatedValueParser(multi_value_expr) list_expr << Suppress('[') - ListParser(list_entry - ZeroOrMore( eol_comma - list_entry)) - Suppress(']') # special case when we have a value assignment where the string can potentially be the remainder of the line assign_expr << Group(key - ZeroOrMore(comment_no_comma_eol) - ( dict_expr | (Literal('=') | Literal(':') | Literal('+=')) - ZeroOrMore(comment_no_comma_eol) - ConcatenatedValueParser(multi_value_expr))) # the file can be { ... } where {} can be omitted or [] config_expr = ZeroOrMore(comment_eol | eol) + ( list_expr | root_dict_expr | inside_root_dict_expr) + ZeroOrMore(comment_eol | eol_comma) config = config_expr.parseString(content, parseAll=True)[0] if resolve: allow_unresolved = resolve and unresolved_value is not DEFAULT_SUBSTITUTION and unresolved_value is not MANDATORY_SUBSTITUTION has_unresolved = cls.resolve_substitutions( config, allow_unresolved) if has_unresolved and unresolved_value is MANDATORY_SUBSTITUTION: raise ConfigSubstitutionException( 'resolve cannot be set to True and unresolved_value to MANDATORY_SUBSTITUTION' ) if unresolved_value is not NO_SUBSTITUTION and unresolved_value is not DEFAULT_SUBSTITUTION: cls.unresolve_substitutions_to_value(config, unresolved_value) return config
replaceWith, upcaseTokens, Word, WordEnd, WordStart, ZeroOrMore, ) from data_lists import tlds, schemes alphanum_word_start = WordStart(wordChars=alphanums) alphanum_word_end = WordEnd(wordChars=alphanums) # the label definition ignores the fact that labels should not end in an hyphen label = Word(initChars=alphanums, bodyChars=alphanums + '-', max=63) domain_tld = Or(tlds) domain_name = (alphanum_word_start + Combine( Combine(OneOrMore(label + ('.' + FollowedBy(Word(alphanums + '-'))))) ('domain_labels') + domain_tld('tld')) + alphanum_word_end).setParseAction(downcaseTokens) ipv4_section = (Word( nums, asKeyword=True, max=3).setParseAction(lambda x: str(int(x[0]))).addCondition( lambda tokens: int(tokens[0]) < 256)) # basically, the grammar below says: start any words that start with a '.' or a number; I want to match words that start with a '.' because this will fail later in the grammar and I do not want to match anything that start with a '.' ipv4_address = (alphanum_word_start + WordStart('.' + nums) + Combine((ipv4_section + '.') * 3 + ipv4_section) + NotAny(Regex('\.\S')) + alphanum_word_end) hexadectet = Word(hexnums, min=1, max=4)
BOM = '\xef\xbb\xbf' bom = Regex(BOM) msg_utf8 = bom + utf_8_string msg_any = utf_8_string msg = Combine(Or([msg_utf8, msg_any])).setResultsName('MSG') sd_name = CharsNotIn('= ]"', 1, 32) param_name = sd_name.setResultsName('SD_PARAM_NAME') param_value = QuotedString(quoteChar='"', escChar='\\', multiline=True) param_value = param_value.setResultsName('SD_PARAM_VALUE') sd_id = sd_name.setResultsName('SD_ID') sd_param = Group(param_name + Regex('=') + param_value) sd_params = Group(ZeroOrMore(Group(sp+sd_param.setResultsName('SD_PARAM')))) sd_element = Group('['+sd_id+sd_params.setResultsName('SD_PARAMS')+']') sd_element = sd_element.setResultsName('SD_ELEMENT') sd_elements = Group(OneOrMore(sd_element)) structured_data = Or([nilvalue, sd_elements.setResultsName('SD_ELEMENTS')]) structured_data = structured_data.setResultsName('STRUCTURED_DATA') time_hour = Regex('0[0-9]|1[0-9]|2[0-3]') time_minute = Regex('[0-5][0-9]') time_second = time_minute time_secfrac = Regex('\.[0-9]{1,6}') time_numoffset = Or([Regex('\+'), Regex('-')]) + \ time_hour + ':' + time_minute time_offset = Or([Regex('Z'), time_numoffset]) partial_time = time_hour + ':' + time_minute + ':' + time_second + \ Optional(time_secfrac) full_time = partial_time + time_offset date_mday = Regex('[0-9]{2}') date_month = Regex('0[1-9]|1[0-2]') date_fullyear = Regex('[0-9]{4}') full_date = date_fullyear + '-' + date_month + '-' + date_mday
See the LICENSE file in the root of this project for license details. ''' from pyparsing import ( Word, alphas, alphanums, nums, cppStyleComment, Keyword, Group, Optional, Or, OneOrMore, delimitedList, ZeroOrMore, hexnums, Combine, FollowedBy, ParseException as pyparsingParseException ) ParseException = pyparsingParseException identifier = Word(alphas + "_", alphas + alphanums + "_") qualified_identifier = Word(alphas + "_.", alphas + alphanums + "_.") BASIC_TYPES = ["bool", "i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64"] basic_type = Or([Keyword(t) for t in BASIC_TYPES]) bit_width = Word(nums) dec_literal = Word(nums) hex_literal = Combine("0x" + Word(hexnums)) signed_literal = Combine(Optional('-') + (dec_literal ^ hex_literal)) comment = cppStyleComment enumValue = Group( Optional(comment)("doc") + identifier("name") + Optional('=' + signed_literal("constant")) )
+ Suppress(Literal(":"))) \ + Optional(name.setResultsName("interface") \ + Suppress(Literal("."))) \ + name_with_alias \ + Optional(slice_enclosed \ | index_enclosed)).setResultsName("simple_mapping", listAllMatches=True) ## Broadcast of port mappings separeted by '|' # and an optional '|' in the end (to support generated simple_mapping strings) broadcast_mapping = simple_mapping \ + ZeroOrMore(Suppress(Literal("|")) \ + simple_mapping) + Optional("|") ## Index slice mapping of the form "index|slice=broadcast_mapping" # Usually used for mapping signal indices and slices index_slice_mapping = Group(Or([index.setResultsName("index"), slice]) \ + Suppress(Literal("=")) \ + broadcast_mapping \ ).setResultsName("index_slice_mapping", listAllMatches=True) ## Mixed mapping of signal indices and slices separated by ';' # and an optional ';' in the end (to support generated index_slice_mapping strings) mixed_mapping = index_slice_mapping \ + ZeroOrMore(Suppress(Literal(";")) \ + index_slice_mapping) + Optional(";") ## Port mappings can be broadcast mapping in case of scalars and vectors # and mixed mapping in case of vectors port_mappings = mixed_mapping | broadcast_mapping ## Signal mapping of the form "signal=broadcast_maping|mixed_mapping"
from pyparsing import alphas, nums, restOfLine from pyparsing import Forward, Keyword, QuotedString, Word from pyparsing import OneOrMore, Optional, Or, ZeroOrMore from pyparsing import ParserElement from .data import Grammar, Expr, Token from .encoders import languages from .errors import InvalidQuantifier from .errors import MultipleEvalCode from .errors import MultipleStartSymbol, NoStartSymbol from .errors import MultipleTypeDefinition # Parser Elements ------------------------------------------------------------- _nsym = Word(alphas) _tsym = '"' + Word(alphas) + '"' _type = QuotedString('"') _ltag = '@' + Or(languages) _uops = '{' + Word(nums) + Optional(',' + Optional(Word(nums))) + '}' _uops = _uops | '*' | '+' | '?' _expr = Forward() _elem = _nsym | _tsym | '(' + _expr + ')' _fact = _elem + Optional(_uops) _term = OneOrMore(_fact) _expr << _term + ZeroOrMore('|' + _term) _sdef = Keyword('START') + '=' + _nsym _prod = _nsym + '->' + _expr _tdef = _nsym + Optional(_ltag) + ':' + _type _edef = Keyword('EVAL') + Optional(_ltag) + '=' + QuotedString('"', '\\') _cmnt = ('#' + restOfLine).suppress() _rule = (_sdef | _prod | _tdef | _edef) + ';' + Optional(_cmnt)
type_single = TYPE + EQUALITY + ASSET_NAME type_list = generate_in_expr(TYPE, ASSET_NAME) type_expr = type_list | type_single cond_single = COND + EQUALITY + CONDITIONS cond_list = generate_in_expr(COND, CONDITIONS) cond_expr = cond_list | cond_single path_single = PATH + EQUALITY + PATH_RE path_list = generate_in_expr(PATH, PATH_RE) path_expr = path_list | path_single label_expr = LABELLED + EQUALITY + Or((TRUE, FALSE)) assy_expr = ASSY + EQUALITY + Or((TRUE, FALSE)) tristate_expr = Or(TRI_STATE_KEYS) + EQUALITY + Or((TRUE, FALSE, UNSET)) or_expr = Forward() and_expr = Forward() not_expr = Forward() primary = Forward() func_expr = Forward() base_expr = (not_expr | code_expr | serial_expr | type_expr | cond_expr | path_expr | label_expr | assy_expr | tristate_expr) paren_expr = L_BRKT + or_expr + R_BRKT primary << (base_expr | paren_expr) func_expr << (FUNCTIONS + OF + func_expr | primary)
BOM = "\xef\xbb\xbf" bom = Regex(BOM) msg_utf8 = bom + utf_8_string msg_any = utf_8_string msg = Combine(Or([msg_utf8, msg_any])).setResultsName("MSG") sd_name = CharsNotIn('= ]"', 1, 32) param_name = sd_name.setResultsName("SD_PARAM_NAME") param_value = QuotedString(quoteChar='"', escChar="\\", multiline=True) param_value = param_value.setResultsName("SD_PARAM_VALUE") sd_id = sd_name.setResultsName("SD_ID") sd_param = Group(param_name + Regex("=") + param_value) sd_params = Group(ZeroOrMore(Group(sp + sd_param.setResultsName("SD_PARAM")))) sd_element = Group("[" + sd_id + sd_params.setResultsName("SD_PARAMS") + "]") sd_element = sd_element.setResultsName("SD_ELEMENT") sd_elements = Group(OneOrMore(sd_element)) structured_data = Or([nilvalue, sd_elements.setResultsName("SD_ELEMENTS")]) structured_data = structured_data.setResultsName("STRUCTURED_DATA") time_hour = Regex("0[0-9]|1[0-9]|2[0-3]") time_minute = Regex("[0-5][0-9]") time_second = time_minute time_secfrac = Regex("\.[0-9]{1,6}") time_numoffset = Or([Regex("\+"), Regex("-")]) + time_hour + ":" + time_minute time_offset = Or([Regex("Z"), time_numoffset]) partial_time = time_hour + ":" + time_minute + ":" + time_second + Optional(time_secfrac) full_time = partial_time + time_offset date_mday = Regex("[0-9]{2}") date_month = Regex("0[1-9]|1[0-2]") date_fullyear = Regex("[0-9]{4}") full_date = date_fullyear + "-" + date_month + "-" + date_mday timestamp = Combine(Or([nilvalue, full_date + "T" + full_time])) timestamp = timestamp.setResultsName("TIMESTAMP")
attrexpr << varexpr + Literal(".") + name valexpr << Group(QuotedString("\"") | nums | attrexpr) """ provides_decl = Keyword("provides") - Suppress(open_block) - Group(ZeroOrMore(Word(printables, excludeChars="{}"))) - Suppress(close_block) requires_decl = Keyword("requires") - Suppress(open_block) - Group(ZeroOrMore(Word(printables, excludeChars="{}"))) - Suppress(close_block) agent_decl = Keyword("agent") + Suppress(open_block) + Group(ZeroOrMore(Word(printables, excludeChars="{}"))) + Suppress(close_block) script_decl = Keyword("script") + Suppress(open_block) + QuotedString("\"", multiline=True) + Suppress(close_block) tool_decl = Keyword("tool") + Suppress(open_block) + QuotedString("\"", multiline=True) + Suppress(close_block) action_contents = Group(provides_decl ^ requires_decl ^ agent_decl ^ script_decl ^ tool_decl) action_decl = Group(Keyword("action") + name.setResultsName("actionName") + Optional(action_keywords) + Suppress(open_block) - ZeroOrMore(action_contents) - Suppress(close_block)).setParseAction(add_action) primary_decl = Forward() block = Forward() primary_keyword = Or([Keyword("branch"), Keyword("iteration"), Keyword("selection"), Keyword("sequence"), Keyword("task")]) primary_decl << Group(primary_keyword.setParseAction(add_primary) - Optional(name)("name") - Suppress(open_block).setParseAction(start_prim) - ZeroOrMore( (action_decl ^ primary_decl) ) - Suppress(close_block).setParseAction(stop_prim)) process_decl = (Keyword("process") + name.setResultsName("processName").setParseAction(process_name) + Suppress(open_block) - ZeroOrMore( (primary_decl ^ action_decl )).setResultsName("block") - Suppress(close_block)) def arr_to_json(orig_arr): arr = copy.deepcopy(orig_arr) global glob_cur_path global glob_d d = copy.deepcopy(glob_d) cur_path = d["process"]["contains"] glob_d = {"process": {"contains": {}, "type": "process", "name": ""}} glob_cur_path = glob_d["process"]["contains"] stack = [] count = -1
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ author = """Vincent Kraeutler, 2007""" from netaddress.rfc3986 import port, host, path_abempty from pyparsing import Combine, Literal, Optional, Or scheme = Or([Literal("http"), Literal("https")]).setResultsName("scheme") prefix = scheme + Literal("://") path = path_abempty.setResultsName("path") angelPort = Optional(Literal(":") + port) angelPath = Optional(Combine(path), default = "/") angelHost = Optional(Combine(host), default = "localhost").setResultsName("host") # URI grammar as used in angel-app angelURI = prefix + angelHost + angelPort + angelPath def parse(uri = ""): #try: return angelURI.parseString(uri) #except: # raise ValueError, "Failed to parse URI: " + `uri`
name = (LineStart() + Literal("Name:")).suppress() pre = Literal("Pre:").suppress() num = Combine(Optional(Literal("-")) + Word(nums)) assign = Literal("=").suppress() constant = Combine(Literal("C") + Optional(Word(nums))) skip = Literal("skip") to = Literal("to").suppress() percent = Literal("%").suppress() ident = Combine(percent + Word(alphanums)) funName = Word(srange("[a-zA-Z]"), alphanums) lpar = Literal( "(" ).suppress() rpar = Literal( ")" ).suppress() comma = Literal( "," ).suppress() # theory expression unops = ["~", "-"] lBvUnop = Or(mkLit(s) for s in unops) unops = ["!"] lUnop = Or(mkLit(s) for s in unops) binops = ["&", "|", "^", ">>", "u>>", "<<", "+", "-", "*", "/", "/u", "%", "%u"] lBvBinop = Or(mkLit(s) for s in binops) binops = ["&&", "||"] lBinop = Or(mkLit(s) for s in binops) cmps = ["==", "!=", "<", ">", "u<", "u>", "<=", ">=", "u<=", "u>="] lCmp = Or(mkLit(s) for s in cmps) funs = ["abs", "computeKnownZeroBits", "countLeadingZeros", \ "countTrailingZeros", "computeNumSignBits", "log2", "lshr", \ "max", "umax", "width"] xfuns = ["sext", "trunc", "zext", "ZExtOrTrunc"] lBvFun = Or(mkLit(s) for s in funs + xfuns) xFun = Or(mkLit(s) for s in xfuns) funs = ["equivalentAddressValues", "hasOneUse", "isPowerOf2", \
def property_grammar(): ParserElement.setDefaultWhitespaceChars(' ') dpi_setting = Group( Optional('*')('DEFAULT') + INTEGER('DPI') + Suppress('@') + INTEGER('HZ'))('SETTINGS*') mount_matrix_row = SIGNED_REAL + ',' + SIGNED_REAL + ',' + SIGNED_REAL mount_matrix = Group(mount_matrix_row + ';' + mount_matrix_row + ';' + mount_matrix_row)('MOUNT_MATRIX') xkb_setting = Optional(Word(alphanums + '+-/@._')) # Although this set doesn't cover all of characters in database entries, it's enough for test targets. name_literal = Word(printables + ' ') props = ( ('MOUSE_DPI', Group(OneOrMore(dpi_setting))), ('MOUSE_WHEEL_CLICK_ANGLE', INTEGER), ('MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL', INTEGER), ('MOUSE_WHEEL_CLICK_COUNT', INTEGER), ('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', INTEGER), ('ID_AUTOSUSPEND', Or((Literal('0'), Literal('1')))), ('ID_INPUT', Or((Literal('0'), Literal('1')))), ('ID_INPUT_ACCELEROMETER', Or((Literal('0'), Literal('1')))), ('ID_INPUT_JOYSTICK', Or((Literal('0'), Literal('1')))), ('ID_INPUT_KEY', Or((Literal('0'), Literal('1')))), ('ID_INPUT_KEYBOARD', Or((Literal('0'), Literal('1')))), ('ID_INPUT_MOUSE', Or((Literal('0'), Literal('1')))), ('ID_INPUT_POINTINGSTICK', Or((Literal('0'), Literal('1')))), ('ID_INPUT_SWITCH', Or((Literal('0'), Literal('1')))), ('ID_INPUT_TABLET', Or((Literal('0'), Literal('1')))), ('ID_INPUT_TABLET_PAD', Or((Literal('0'), Literal('1')))), ('ID_INPUT_TOUCHPAD', Or((Literal('0'), Literal('1')))), ('ID_INPUT_TOUCHSCREEN', Or((Literal('0'), Literal('1')))), ('ID_INPUT_TRACKBALL', Or((Literal('0'), Literal('1')))), ('POINTINGSTICK_SENSITIVITY', INTEGER), ('POINTINGSTICK_CONST_ACCEL', REAL), ('ID_INPUT_JOYSTICK_INTEGRATION', Or(('internal', 'external'))), ('ID_INPUT_TOUCHPAD_INTEGRATION', Or(('internal', 'external'))), ('XKB_FIXED_LAYOUT', xkb_setting), ('XKB_FIXED_VARIANT', xkb_setting), ('XKB_FIXED_MODEL', xkb_setting), ('KEYBOARD_LED_NUMLOCK', Literal('0')), ('KEYBOARD_LED_CAPSLOCK', Literal('0')), ('ACCEL_MOUNT_MATRIX', mount_matrix), ('ACCEL_LOCATION', Or(('display', 'base'))), ('PROXIMITY_NEAR_LEVEL', INTEGER), ('IEEE1394_UNIT_FUNCTION_MIDI', Or((Literal('0'), Literal('1')))), ('IEEE1394_UNIT_FUNCTION_AUDIO', Or((Literal('0'), Literal('1')))), ('IEEE1394_UNIT_FUNCTION_VIDEO', Or((Literal('0'), Literal('1')))), ('ID_VENDOR_FROM_DATABASE', name_literal), ('ID_MODEL_FROM_DATABASE', name_literal), ) fixed_props = [ Literal(name)('NAME') - Suppress('=') - val('VALUE') for name, val in props ] kbd_props = [ Regex(r'KEYBOARD_KEY_[0-9a-f]+')('NAME') - Suppress('=') - ('!' ^ (Optional('!') - Word(alphanums + '_')))('VALUE') ] abs_props = [ Regex(r'EVDEV_ABS_[0-9a-f]{2}')('NAME') - Suppress('=') - Word(nums + ':')('VALUE') ] grammar = Or(fixed_props + kbd_props + abs_props) + EOL return grammar
def _build_filter_parser(): field = _build_field_expr() negation = CaselessKeyword('not') negation.setParseAction(lambda x: Negation(x[0])) comparison_operator = Group( Keyword('=') ^ Keyword('!=') ^ Keyword('>=') ^ Keyword('<=') ^ Keyword('<') ^ Keyword('>') ^ Keyword('lte') # match before lt ^ Keyword('lt') ^ Keyword('gte') # match before gt ^ Keyword('gt') ^ (Optional(negation) + (CaselessKeyword('contains') ^ CaselessKeyword('icontains') ^ CaselessKeyword('startswith') ^ CaselessKeyword('istartswith') ^ CaselessKeyword('endswith') ^ CaselessKeyword('iendswith') ^ CaselessKeyword('eq')))) comparison_operator.setParseAction(lambda x: Operator(x)) single_value_operator = Group( CaselessKeyword('isnull') ^ (Optional(negation) + CaselessKeyword('isnull'))) single_value_operator.setParseAction(lambda x: Operator(x)) plusorminus = Literal('+') | Literal('-') num_integer = Combine(Optional(plusorminus) + Word(nums)) num_integer.setParseAction(lambda x: Integer(x[0])) num_float = Combine(Optional(plusorminus) + Word(nums) + '.' + Word(nums)) num_float.setParseAction(lambda x: Float(x[0])) quoted_string = (QuotedString("'") ^ QuotedString('"')) quoted_string.setParseAction(lambda x: String(x[0])) boolean = Or([ CaselessKeyword(v) for v in BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES ]) boolean.setParseAction(lambda x: Boolean(x[0])) value = (quoted_string ^ num_integer ^ num_float ^ boolean) comparison = Group((field + comparison_operator + value) ^ (value + comparison_operator + field) ^ (field + comparison_operator + field) ^ (field + single_value_operator)) comparison.setParseAction(lambda x: Comparison(x)) invalid_comparison = Group( (value + comparison_operator + value).setParseAction(lambda x: fail( "Value may not be compared with values: {0}".format(' '.join(x))))) logical_op = Group(CaselessKeyword("and") | CaselessKeyword("or")) logical_op.setParseAction(lambda x: LogicalOp(x[0][0])) expr = Forward() statement = (comparison | invalid_comparison | (Literal('(').suppress() + expr + Literal(')').suppress())) statement.setParseAction(lambda x: Statement(x)) expr << statement + ZeroOrMore(logical_op + statement) return expr
def np_uint(s, loc, tokens): return np_composite('uint', np_uint_dtypes) def np_int(s, loc, tokens): return np_composite('int', np_int_dtypes) def np_float(s, loc, tokens): return np_composite('float', np_float_dtypes) dtype_checks.append(Keyword('int').setParseAction(np_int)) dtype_checks.append(Keyword('uint').setParseAction(np_uint)) dtype_checks.append(Keyword('float').setParseAction(np_float)) composite = ['int', 'uint', 'float'] ndarray_simple_contract = Or(dtype_checks + array_constraints) ndarray_simple_contract.setName('numpy element contract') suggester = create_suggester(get_options=lambda: atomic + composite) baseExpr = ndarray_simple_contract | suggester baseExpr.setName('numpy contract (with recovery)') op = myOperatorPrecedence # op = operatorPrecedence ndarray_composite_contract = op(baseExpr, [ (',', 2, opAssoc.LEFT, ArrayAnd.parse_action), ('|', 2, opAssoc.LEFT, ArrayOR.parse_action), ]) def my_delim_list2(what, delim):
minus = Literal("-") comma = Literal(",") lbracket = Literal("[") rbracket = Literal("]") lbrace = Literal("{") rbrace = Literal("}") hashsign = Literal("#") exclamation = Literal("!") caret = Literal("^") hex_num = Combine(Literal("0x") + Word("0123456789abcdef")) dec_num = Word("0123456789") # Operand Parsing # ============================================================================ # sign = Optional(Or([plus, minus("minus")])) immediate = Group(Optional(Suppress(hashsign)) + (sign + Or([hex_num, dec_num]))("value")) register = Group(Or([ Combine(Literal("r") + Word(nums)("reg_num")), Combine(Literal("d") + Word(nums)("reg_num")), Combine(Literal("c") + Word(nums)("reg_num")), Combine(Literal("p") + Word(nums)("reg_num")), Literal("sp"), Literal("lr"), Literal("pc"), Literal("fp"), Literal("ip"), Literal("sl"), Literal("sb"),
this <<= operator_expression | last last = this # Set the full expression and return it expression <<= last return expression # An integer value integer = Word(nums) integer.setParseAction(Integer.parse) integer.setName("integer") dice_separators = RandomElement.DICE_MAP.keys() dice_element = Or( wrap_string(CaselessLiteral, x, suppress=False) for x in dice_separators ) special = wrap_string(Literal, "%", suppress=False) | wrap_string( CaselessLiteral, "f", suppress=False ) # An expression in dice notation expression = ( StringStart() + operatorPrecedence( integer, [ (dice_element, 2, opAssoc.LEFT, RandomElement.parse, special), (dice_element, 1, opAssoc.RIGHT, RandomElement.parse_unary, special), (wrap_string(CaselessLiteral, "x"), 2, opAssoc.LEFT, Explode.parse), (wrap_string(CaselessLiteral, "x"), 1, opAssoc.LEFT, Explode.parse),
from pyparsing import CaselessLiteral, Word, Optional, Group, nums, alphanums, oneOf, \ ZeroOrMore, StringEnd, Or, Suppress, Forward, ParseException as pyPE from dbp.db import Update, Insert, Delete from dbp.where import _combinators, _ops, ParseException # Example syntax # (SELECT, UPDATE) (values) WHERE (column operator value) # INSERT (values) # DELETE WHERE ... # INSERT (values) SELECT ? probs not. _operators = ("SELECT", "UPDATE") operators = Or(CaselessLiteral(o) for o in _operators) combinators = Or(CaselessLiteral(c) for c in _combinators) # comma separated values w = Word(alphanums) c = Suppress(",") values = w + ZeroOrMore(c + w) # Grammar: # B -> AND | OR # T -> f op f | ( C ) # C -> T B T | T # S -> WHERE C EOF stringLit = Group('"' + w + '"') intLit = Word(nums) C = Forward() T = ((stringLit | intLit | w) + oneOf(_ops) + (stringLit | intLit | w)) | (Suppress("(") + C + Suppress(")"))
expression = operatorPrecedence(term, [(oneOf("* /"), 2, opAssoc.LEFT, parse_expression), (oneOf("+ -"), 2, opAssoc.LEFT, parse_expression)] ) assignment_stmt = variable + equals + expression + StringEnd() assignment_stmt.setParseAction(parse_assignment_stmt) output_stmt = output_fn + expression + StringEnd() output_stmt.setParseAction(parse_output_stmt) input_stmt = input_fn + term + StringEnd() input_stmt.setParseAction(parse_input_stmt) stmt = Or([assignment_stmt, input_stmt, output_stmt]) stmt.setParseAction(print_tree) def parser_init(): global symbol_table symbol_table = {} def test_negative(): parser_init() test_negative_string("x=1 +") test_negative_string("x==1 ** x /") test_negative_string("z 1 3 + 4") test_negative_string("z =1 3 + 4") test_negative_string("z =1 3 + 4;") def test_negative_string(expr):
lbracket = Literal("[") rbracket = Literal("]") colon = Literal(":") hex_num = Combine(Literal("0x") + Word("0123456789abcdef")) dec_num = Word("0123456789") # Operand Parsing # ============================================================================ # modifier = (Optional( Or([ Literal("ymmword"), Literal("xmmword"), Literal("xword"), Literal("tword"), Literal("qword"), Literal("dword"), Literal("word"), Literal("byte"), Literal("far"), ])) + Optional(Literal("ptr"))) immediate = Optional("-") + Or([hex_num, dec_num]) segment = Or([ Literal("cs"), Literal("ds"), Literal("ss"), Literal("es"), Literal("fs"), Literal("gs"),
bullet = originalTextFor(number + Literal('.')) percent = originalTextFor(number + Literal('%')) hypen_word = Combine(word + Literal('-') + word) apos_word1 = Combine(Literal('\'') + word) apos_word2 = Combine(word + Literal('\'') + word) new_line = Literal('\n') tab = Literal('\t') punctuation = list(u"‘’.,;„“”()‘:\"\'`′!?-–—…") + ['...'] punkt = map(Literal, punctuation) all_ = [word, initial, bullet, hypen_word, apos_word1, apos_word2, \ number, percent, new_line, latin_word, tab] all_.extend(punkt) all_ = Or(all_).parseWithTabs() def tokenize(feed, include_junk = True, echo_junk = False): """Returns list of tokens""" tokens = [] start = 0 for t, s, e in all_.scanString(feed): if s != 0: junk = feed[start:s].strip() if junk != "": if include_junk: tokens.append(junk) if echo_junk: print(junk) if t[0]: tokens.append(t[0]) start = e
def np_int(s, loc, tokens): return np_composite('int', np_int_dtypes) def np_float(s, loc, tokens): return np_composite('float', np_float_dtypes) dtype_checks.append(Keyword('int').setParseAction(np_int)) dtype_checks.append(Keyword('uint').setParseAction(np_uint)) dtype_checks.append(Keyword('float').setParseAction(np_float)) composite = ['int', 'uint', 'float'] ndarray_simple_contract = Or(dtype_checks + array_constraints) ndarray_simple_contract.setName('numpy element contract') suggester = create_suggester(get_options=lambda: atomic + composite) baseExpr = ndarray_simple_contract | suggester baseExpr.setName('numpy contract (with recovery)') op = myOperatorPrecedence # op = operatorPrecedence ndarray_composite_contract = op(baseExpr, [ (',', 2, opAssoc.LEFT, ArrayAnd.parse_action), ('|', 2, opAssoc.LEFT, ArrayOR.parse_action), ]) def my_delim_list2(what, delim):
def someOf(*argv): return OneOrMore(Or(argv))
from pdfile import pdfile import sys """ A compiler from message definitions into Pd patches. """ # reserved words reserved_words = ['name', 'slots', 'constants', 'equations', 'output'] reserved_parsers = [] for reserved in reserved_words: literal = Literal(reserved.upper()) globals()[reserved] = literal reserved_parsers.append(literal) reserved = Or(reserved_parsers) word = ~reserved + Word(alphanums + "_" + "-") function = Literal("lexicon") equation = word + "=" + function + "(" + OneOrMore(word) + \ Optional(Literal(";") + OneOrMore(word + Optional(Literal("'")))) + ")" def parse_equation(toks): name = toks[0] function = toks[2] first_slot = [] second_slot = [] in_first = True position = 4 while position < len(toks) - 1:
def ruleParser(): global typedversion # ------------------------------------------------------ # Atomic # ------------------------------------------------------ # Tokens lbr = Literal("(") rbr = Literal(")") comma = Literal(",") hash = Literal("#") equ = Literal("=") implies = Literal("=>") dot = Literal(".") eol = Literal("\n").suppress() # Basic constructors Alfabet = alphas + nums + "_$" Number = Word(nums) Number.setParseAction(lambda s, l, t: ["number", Term.TermConstant(t[0])]) # Typeinfo/Constant TypeInfo = oneOf("mr nonce pk sk fu table") TypeInfo.setParseAction( lambda s, l, t: ["typeinfo", Term.TermConstant(t[0])]) Constant = Word(alphas, Alfabet) Constant.setParseAction(lambda s, l, t: [Term.TermConstant(t[0])]) # Time nTime = Number xTime = Literal("xTime") sTime = Literal("s").suppress() + lbr + Group(Number) + rbr Time = Or([nTime, xTime, sTime]) # Const Const = Forward() ConstC = Literal("c") + lbr + Constant + comma + Time + rbr ConstF = Literal("c(ni,ni)") Const << Or([Constant, ConstC, ConstF]) Const.setParseAction(lambda s, l, t: [If.Constant("".join(t))]) # Two versions Variable = Word("x", Alfabet) Variable.setParseAction( lambda s, l, t: [Term.TermVariable(t[0] + "V", None)]) if typedversion: Variable = TypeInfo + lbr + Variable + rbr # Optional prime optprime = Optional(Literal("'")) # Atomic ## DEVIANT : below there is an optprime after the atom. This ## is not in the BNF. Atomic = Or([TypeInfo + lbr + Const + rbr, Variable]) + optprime ### TEST #print Time.parseString("s(25)") #print Variable.parseString("xCas") #print Atomic.parseString("nonce(Koen)") # ------------------------------------------------------ # Messages # ------------------------------------------------------ # Base forward declaration Message = Forward() # Agents etc Agent = Or([Literal("mr") + lbr + Const + rbr, Variable]) KeyTable = Or([Literal("table") + lbr + Const + rbr, Variable]) KeyTableApp = Literal( "tb") + lbr + KeyTable + comma + Agent + rbr + optprime # Crypto pkterm = Literal("pk") + lbr + Const + rbr + optprime varterm = Variable + optprime Invertible = Or([pkterm, KeyTableApp, varterm]) PublicCypher = Literal("crypt") + lbr + Invertible + comma + Message + rbr PublicCypher.setParseAction(lambda s, l, t: [Term.TermEncrypt(t[2], t[1])]) XOR = Literal("rcrypt") + lbr + Message + comma + Message + rbr SymmetricCypher = Literal("scrypt") + lbr + Message + comma + Message + rbr futerm = Or([Literal("fu") + lbr + Const + rbr, Variable]) Function = Literal("funct") + lbr + futerm + comma + Message + rbr # Message composition Concatenation = Literal("c") + lbr + Message + comma + Message + rbr Concatenation.setParseAction(lambda s, l, t: [Term.TermTuple(t[1], t[2])]) Composed = Or([ Concatenation, SymmetricCypher, XOR, PublicCypher, Function, KeyTable, KeyTableApp ]) Message << Or([Composed, Atomic]) ### TEST #print Message.parseString("nonce(c(Na,xTime))") # ------------------------------------------------------ # Model of honest agents # ------------------------------------------------------ Boolean = Or([Literal("true"), Literal("false"), Variable]) Session = Forward() Session << Or([Literal("s") + lbr + Session + rbr, Number, Variable]) MsgEtc = Literal("etc") MsgList = Forward() MsgComp = Literal("c") + lbr + Message + comma + MsgList + rbr MsgList << Or([MsgEtc, Variable, MsgComp]) Step = Or([Number, Variable]) ### TEST #print Message.parseString("xKb") #print MsgList.parseString("etc") #print MsgList.parseString("c(xKb,etc)") #print MsgList.parseString("c(xA,c(xB,c(xKa,c(xKa',c(xKb,etc)))))") # Principal fact Principal = Literal( "w" ) + lbr + Step + comma + Agent + comma + Agent + comma + MsgList + comma + MsgList + comma + Boolean + comma + Session + rbr Principal.setParseAction(lambda s, l, t: ["Principal", t]) # Message fact MessageFact = Literal( "m" ) + lbr + Step + comma + Agent + comma + Agent + comma + Agent + comma + Message + comma + Session + rbr # Goal fact Correspondence = Principal + dot + Principal Secret = Literal("secret") + lbr + Message + Literal( "f") + lbr + Session + rbr + rbr Secrecy = Literal("secret") + lbr + Literal("xsecret") + comma + Literal( "f") + lbr + Session + rbr + rbr + dot + Literal("i") + lbr + Literal( "xsecret") + rbr Give = Literal("give") + lbr + Message + Literal( "f") + lbr + Session + rbr + rbr STSecrecy = Literal("give(xsecret,f(xc)).secret(xsecret,f(xc))" ) + implies + Literal("i(xsecret)") Witness = Literal( "witness" ) + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr Request = Literal( "request" ) + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr Authenticate = Literal( "request" ) + lbr + Agent + comma + Agent + comma + Constant + comma + Message + rbr GoalState = Or([Correspondence, Secrecy, STSecrecy, Authenticate]) GoalFact = Or([Secret, Give, Witness, Request]) # TimeFact TimeFact = Literal("h") + lbr + Message + rbr # Intruder knowledge IntruderKnowledge = Literal("i") + lbr + Message + rbr # Facts and states Fact = Or([ Principal, MessageFact, IntruderKnowledge, TimeFact, Secret, Give, Witness, Request ]) State = Group(delimitedList( Fact, ".")) ## From initial part of document, not in detailed BNF # Rules MFPrincipal = Or([MessageFact + dot + Principal, Principal]) mr1 = Literal("h") + lbr + Literal("s") + lbr + Literal( "xTime") + rbr + rbr + dot + MFPrincipal mr2 = implies mr3 = Literal("h") + lbr + Literal( "xTime") + rbr + dot + MFPrincipal + Optional( dot + delimitedList(GoalFact, ".")) MessageRule = Group(mr1) + mr2 + Group( mr3) ## DEVIANT : BNF requires newlines InitialState = Literal("h") + lbr + Literal( "xTime") + rbr + dot + State ## DEVIANT : BNF requires newlines # Intruder IntruderRule = Literal("nogniet") # Simplification f_simplif = Literal("f") + lbr + Literal("s") + lbr + Literal( "xc") + rbr + rbr + implies + Literal("f") + lbr + Literal( "xc") + rbr ## DEVIANT : EOL removed matching_request = Witness + dot + Request + implies no_auth_intruder = Request + implies SimplificationRule = Or([f_simplif, matching_request, no_auth_intruder]) # Compose all rules Rule = Or([ InitialState, MessageRule, IntruderRule, GoalState, SimplificationRule ]) return Rule
literal_kw = true_kw | false_kw # RESERVED reserved = statement_keyword | marker_keyword | operator_keyword | literal_kw # IDENTIFIER identifier_regex = r"(?:[a-zA-Z]|_[a-zA-Z])[a-zA-Z0-9_]*" identifier = ((~reserved + Regex(identifier_regex)).setName( "identifier").setParseAction(lambda r: Identifier(r[0]))) identifier_keyword = (Regex(identifier_regex).setName( "identifier keyword").setParseAction(lambda r: Identifier(r[0]))) # Types variable_type = Or(types) | identifier ############# # Grammar # ############# expression = Forward().setName("expression") statement = Forward().setName("statement") ################# # Expressions # ################# # Member access def __build_recursive_member_access(expr, pos, result):
message += cr ^ lf ^ crlf else: message += crlf message.leaveWhitespace() build_message() dispatcher.connect(build_message, 'parser.trailing_spaces', 'config') dispatcher.connect(build_message, 'parser.soft_eol', 'config') chanstring = charclass((0x01, 0x06), (0x08, 0x09), (0x0B, 0x0C), (0x0E, 0x1F), (0x21, 0x2B), (0x2D, 0x39), (0x3B, 0xFF)) channelid = 5 * (charclass((0x41, 0x5A)) ^ digit) channel = And([ Or([oneOf('# + &'), Literal('!') + Group(channelid)]), Group(OneOrMore(chanstring)), Optional(Suppress(Literal(':')) + Group(OneOrMore(chanstring))) ]) ### # Wildcard expressions ### wildone = Literal('?') wildmany = Literal('*') nowild = charclass((0x01, 0x29), (0x2B, 0x3E), (0x40, 0xFF)) noesc = charclass((0x01, 0x5B), (0x5D, 0xFF)) mask = Optional(wildone ^ wildmany) + \ ZeroOrMore(nowild ^ (noesc + wildone) ^ (noesc + wildmany)) # Fall back to regex for parsing wildcards
IS = CaselessKeyword("is") OR = CaselessKeyword("or") AND = CaselessKeyword("and") NOT = CaselessKeyword("not") OF = CaselessKeyword("of") EQUALS = Literal("=") COLON = Literal(":") L_C_BRKT = Literal("{") R_C_BRKT = Literal("}") L_BRKT = Literal("(") R_BRKT = Literal(")") COMMA = Literal(",") BANG = Literal("!") EQUALITY = Or((EQUALS, COLON, IS)) CODE = Literal("code") SERIAL = Literal("serial") TYPE = Literal("type") COND = Or((Literal("condition"), Literal("cond"))) LABELLED = Literal("labelled") PATH = Literal("path") ASSY = Literal("assy") _TRI_STATE_KEY_NAMES = ("development", "v-sense-move-1213", "motor-rail-mod-1360", "tested", "has_headers", "velcro-attached", "cased", "umbilical", "climit_disabled", "dremel-mod", "tvs-mod-698", "battery-aa-mod-1270") TRI_STATE_KEYS = [Literal(x) for x in _TRI_STATE_KEY_NAMES]
def __repr__(self): return 'CheckOrder(%r,%r,%r)' % (self.expr1, self.glyph, self.expr2) @staticmethod def parse_action(s, loc, tokens): expr1 = tokens.get('expr1', None) glyph = "".join(tokens['glyph']) expr2 = tokens['expr2'] where = W(s, loc) return CheckOrder(expr1, glyph, expr2, where=where) comparisons_expr = {} for glyph in CheckOrder.conditions: if glyph == '!=': # special case: ! must be followed by = glyph_expression = Literal('!') - Literal('=') glyph_expression.setName('!=') else: glyph_expression = Literal(glyph) # 2015-05: not sure why this doesn't work and the alternative with + does # expr = O(rvalue('expr1')) + glyph_expression('glyph') - rvalue('expr2') expr = O(rvalue('expr1')) + glyph_expression('glyph') + rvalue('expr2') expr.setParseAction(CheckOrder.parse_action) add_contract(expr) comparisons_expr[glyph] = expr comparison_expr = Or(exprs=list(comparisons_expr.values()))
# PARSER def ddlWord(string): return WordStart(alphanums + "_") + CaselessLiteral(string) + WordEnd( alphanums + "_") # This function should be refactored if we find some database function which needs parameters # Right now it works only for something like NOW() in MySQL default field value def ddlFunctionWord(string): return CaselessLiteral(string) + OneOrMore("(") + ZeroOrMore( " ") + OneOrMore(")") ddlString = Or([QuotedString("'"), QuotedString("\"", escQuote='""'), QuotedString( "`")]) negativeSign = Literal('-') ddlNum = Combine(Optional(negativeSign) + Word(nums + ".")) ddlTerm = Word(alphanums + "_$") ddlName = Or([ddlTerm, ddlString]) ddlMathOp = Word("+><=-") ddlBoolean = Or([ddlWord("AND"), ddlWord("OR"), ddlWord("NOT")]) ddlArguments = "(" + delimitedList(Or([ddlString, ddlTerm, ddlNum])) + ")" ddlMathCond = Optional("(") + delimitedList( Or([ Group(ddlName + ddlMathOp + ddlName), Group(ddlName + Optional(ddlWord("IS")) + ddlWord("NOT") + ddlWord( "NULL")), ]), delim=ddlBoolean) + Optional(")")
# http://en.wikipedia.org/wiki/Comparison_of_programming_languages_%28syntax%29#Libraries # Python: import # OpenSCAD: include (also use, but that's slightly different) # C etc: #include # LaTeX: \input, \include # Java: import # Modelica: import # OK, let's go for import then... importkeyword = Keyword("import").suppress() # Constants constantMap = { "pi": Constant("pi", math.pi), "e": Constant("e", math.e) } constant = Or([CaselessKeyword(constantName) for constantName in constantMap]) constant.setParseAction(lambda s, l, t: [constantMap[t[0]]]) # Parse numbers into floats straight away fNumber.setParseAction(lambda s, l, t: [float(t[0])]) # Binary operators binaryOperator = Word('+-*/^', exact=1) # N.B. To-the-power-of is ^ in this grammar at the moment - TODO make it accept Python syntax ** # Define mapping from binary operator symbols to composite expression classes binaryOperatorMap = { '+': SumExpression, '-': DifferenceExpression, '*': ProductExpression, '/': QuotientExpression, '^': PowerExpression }